1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2017 Intel Corporation 3 */ 4 5 #include <stdarg.h> 6 #include <stdio.h> 7 #include <stdlib.h> 8 #include <signal.h> 9 #include <string.h> 10 #include <time.h> 11 #include <fcntl.h> 12 #include <sys/mman.h> 13 #include <sys/types.h> 14 #include <errno.h> 15 #include <stdbool.h> 16 17 #include <sys/queue.h> 18 #include <sys/stat.h> 19 20 #include <stdint.h> 21 #include <unistd.h> 22 #include <inttypes.h> 23 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_byteorder.h> 27 #include <rte_log.h> 28 #include <rte_debug.h> 29 #include <rte_cycles.h> 30 #include <rte_memory.h> 31 #include <rte_memcpy.h> 32 #include <rte_launch.h> 33 #include <rte_eal.h> 34 #include <rte_alarm.h> 35 #include <rte_per_lcore.h> 36 #include <rte_lcore.h> 37 #include <rte_atomic.h> 38 #include <rte_branch_prediction.h> 39 #include <rte_mempool.h> 40 #include <rte_malloc.h> 41 #include <rte_mbuf.h> 42 #include <rte_mbuf_pool_ops.h> 43 #include <rte_interrupts.h> 44 #include <rte_pci.h> 45 #include <rte_ether.h> 46 #include <rte_ethdev.h> 47 #include <rte_dev.h> 48 #include <rte_string_fns.h> 49 #ifdef RTE_LIBRTE_IXGBE_PMD 50 #include <rte_pmd_ixgbe.h> 51 #endif 52 #ifdef RTE_LIBRTE_PDUMP 53 #include <rte_pdump.h> 54 #endif 55 #include <rte_flow.h> 56 #include <rte_metrics.h> 57 #ifdef RTE_LIBRTE_BITRATE 58 #include <rte_bitrate.h> 59 #endif 60 #ifdef RTE_LIBRTE_LATENCY_STATS 61 #include <rte_latencystats.h> 62 #endif 63 64 #include "testpmd.h" 65 66 #ifndef MAP_HUGETLB 67 /* FreeBSD may not have MAP_HUGETLB (in fact, it probably doesn't) */ 68 #define HUGE_FLAG (0x40000) 69 #else 70 #define HUGE_FLAG MAP_HUGETLB 71 #endif 72 73 #ifndef MAP_HUGE_SHIFT 74 /* older kernels (or FreeBSD) will not have this define */ 75 #define HUGE_SHIFT (26) 76 #else 77 #define HUGE_SHIFT MAP_HUGE_SHIFT 78 #endif 79 80 #define EXTMEM_HEAP_NAME "extmem" 81 82 uint16_t verbose_level = 0; /**< Silent by default. */ 83 int testpmd_logtype; /**< Log type for testpmd logs */ 84 85 /* use master core for command line ? */ 86 uint8_t interactive = 0; 87 uint8_t auto_start = 0; 88 uint8_t tx_first; 89 char cmdline_filename[PATH_MAX] = {0}; 90 91 /* 92 * NUMA support configuration. 93 * When set, the NUMA support attempts to dispatch the allocation of the 94 * RX and TX memory rings, and of the DMA memory buffers (mbufs) for the 95 * probed ports among the CPU sockets 0 and 1. 96 * Otherwise, all memory is allocated from CPU socket 0. 97 */ 98 uint8_t numa_support = 1; /**< numa enabled by default */ 99 100 /* 101 * In UMA mode,all memory is allocated from socket 0 if --socket-num is 102 * not configured. 103 */ 104 uint8_t socket_num = UMA_NO_CONFIG; 105 106 /* 107 * Select mempool allocation type: 108 * - native: use regular DPDK memory 109 * - anon: use regular DPDK memory to create mempool, but populate using 110 * anonymous memory (may not be IOVA-contiguous) 111 * - xmem: use externally allocated hugepage memory 112 */ 113 uint8_t mp_alloc_type = MP_ALLOC_NATIVE; 114 115 /* 116 * Store specified sockets on which memory pool to be used by ports 117 * is allocated. 118 */ 119 uint8_t port_numa[RTE_MAX_ETHPORTS]; 120 121 /* 122 * Store specified sockets on which RX ring to be used by ports 123 * is allocated. 124 */ 125 uint8_t rxring_numa[RTE_MAX_ETHPORTS]; 126 127 /* 128 * Store specified sockets on which TX ring to be used by ports 129 * is allocated. 130 */ 131 uint8_t txring_numa[RTE_MAX_ETHPORTS]; 132 133 /* 134 * Record the Ethernet address of peer target ports to which packets are 135 * forwarded. 136 * Must be instantiated with the ethernet addresses of peer traffic generator 137 * ports. 138 */ 139 struct rte_ether_addr peer_eth_addrs[RTE_MAX_ETHPORTS]; 140 portid_t nb_peer_eth_addrs = 0; 141 142 /* 143 * Probed Target Environment. 144 */ 145 struct rte_port *ports; /**< For all probed ethernet ports. */ 146 portid_t nb_ports; /**< Number of probed ethernet ports. */ 147 struct fwd_lcore **fwd_lcores; /**< For all probed logical cores. */ 148 lcoreid_t nb_lcores; /**< Number of probed logical cores. */ 149 150 portid_t ports_ids[RTE_MAX_ETHPORTS]; /**< Store all port ids. */ 151 152 /* 153 * Test Forwarding Configuration. 154 * nb_fwd_lcores <= nb_cfg_lcores <= nb_lcores 155 * nb_fwd_ports <= nb_cfg_ports <= nb_ports 156 */ 157 lcoreid_t nb_cfg_lcores; /**< Number of configured logical cores. */ 158 lcoreid_t nb_fwd_lcores; /**< Number of forwarding logical cores. */ 159 portid_t nb_cfg_ports; /**< Number of configured ports. */ 160 portid_t nb_fwd_ports; /**< Number of forwarding ports. */ 161 162 unsigned int fwd_lcores_cpuids[RTE_MAX_LCORE]; /**< CPU ids configuration. */ 163 portid_t fwd_ports_ids[RTE_MAX_ETHPORTS]; /**< Port ids configuration. */ 164 165 struct fwd_stream **fwd_streams; /**< For each RX queue of each port. */ 166 streamid_t nb_fwd_streams; /**< Is equal to (nb_ports * nb_rxq). */ 167 168 /* 169 * Forwarding engines. 170 */ 171 struct fwd_engine * fwd_engines[] = { 172 &io_fwd_engine, 173 &mac_fwd_engine, 174 &mac_swap_engine, 175 &flow_gen_engine, 176 &rx_only_engine, 177 &tx_only_engine, 178 &csum_fwd_engine, 179 &icmp_echo_engine, 180 &noisy_vnf_engine, 181 #if defined RTE_LIBRTE_PMD_SOFTNIC 182 &softnic_fwd_engine, 183 #endif 184 #ifdef RTE_LIBRTE_IEEE1588 185 &ieee1588_fwd_engine, 186 #endif 187 NULL, 188 }; 189 190 struct rte_mempool *mempools[RTE_MAX_NUMA_NODES]; 191 uint16_t mempool_flags; 192 193 struct fwd_config cur_fwd_config; 194 struct fwd_engine *cur_fwd_eng = &io_fwd_engine; /**< IO mode by default. */ 195 uint32_t retry_enabled; 196 uint32_t burst_tx_delay_time = BURST_TX_WAIT_US; 197 uint32_t burst_tx_retry_num = BURST_TX_RETRIES; 198 199 uint16_t mbuf_data_size = DEFAULT_MBUF_DATA_SIZE; /**< Mbuf data space size. */ 200 uint32_t param_total_num_mbufs = 0; /**< number of mbufs in all pools - if 201 * specified on command-line. */ 202 uint16_t stats_period; /**< Period to show statistics (disabled by default) */ 203 204 /* 205 * In container, it cannot terminate the process which running with 'stats-period' 206 * option. Set flag to exit stats period loop after received SIGINT/SIGTERM. 207 */ 208 uint8_t f_quit; 209 210 /* 211 * Configuration of packet segments used by the "txonly" processing engine. 212 */ 213 uint16_t tx_pkt_length = TXONLY_DEF_PACKET_LEN; /**< TXONLY packet length. */ 214 uint16_t tx_pkt_seg_lengths[RTE_MAX_SEGS_PER_PKT] = { 215 TXONLY_DEF_PACKET_LEN, 216 }; 217 uint8_t tx_pkt_nb_segs = 1; /**< Number of segments in TXONLY packets */ 218 219 enum tx_pkt_split tx_pkt_split = TX_PKT_SPLIT_OFF; 220 /**< Split policy for packets to TX. */ 221 222 uint8_t txonly_multi_flow; 223 /**< Whether multiple flows are generated in TXONLY mode. */ 224 225 uint16_t nb_pkt_per_burst = DEF_PKT_BURST; /**< Number of packets per burst. */ 226 uint16_t mb_mempool_cache = DEF_MBUF_CACHE; /**< Size of mbuf mempool cache. */ 227 228 /* current configuration is in DCB or not,0 means it is not in DCB mode */ 229 uint8_t dcb_config = 0; 230 231 /* Whether the dcb is in testing status */ 232 uint8_t dcb_test = 0; 233 234 /* 235 * Configurable number of RX/TX queues. 236 */ 237 queueid_t nb_hairpinq; /**< Number of hairpin queues per port. */ 238 queueid_t nb_rxq = 1; /**< Number of RX queues per port. */ 239 queueid_t nb_txq = 1; /**< Number of TX queues per port. */ 240 241 /* 242 * Configurable number of RX/TX ring descriptors. 243 * Defaults are supplied by drivers via ethdev. 244 */ 245 #define RTE_TEST_RX_DESC_DEFAULT 0 246 #define RTE_TEST_TX_DESC_DEFAULT 0 247 uint16_t nb_rxd = RTE_TEST_RX_DESC_DEFAULT; /**< Number of RX descriptors. */ 248 uint16_t nb_txd = RTE_TEST_TX_DESC_DEFAULT; /**< Number of TX descriptors. */ 249 250 #define RTE_PMD_PARAM_UNSET -1 251 /* 252 * Configurable values of RX and TX ring threshold registers. 253 */ 254 255 int8_t rx_pthresh = RTE_PMD_PARAM_UNSET; 256 int8_t rx_hthresh = RTE_PMD_PARAM_UNSET; 257 int8_t rx_wthresh = RTE_PMD_PARAM_UNSET; 258 259 int8_t tx_pthresh = RTE_PMD_PARAM_UNSET; 260 int8_t tx_hthresh = RTE_PMD_PARAM_UNSET; 261 int8_t tx_wthresh = RTE_PMD_PARAM_UNSET; 262 263 /* 264 * Configurable value of RX free threshold. 265 */ 266 int16_t rx_free_thresh = RTE_PMD_PARAM_UNSET; 267 268 /* 269 * Configurable value of RX drop enable. 270 */ 271 int8_t rx_drop_en = RTE_PMD_PARAM_UNSET; 272 273 /* 274 * Configurable value of TX free threshold. 275 */ 276 int16_t tx_free_thresh = RTE_PMD_PARAM_UNSET; 277 278 /* 279 * Configurable value of TX RS bit threshold. 280 */ 281 int16_t tx_rs_thresh = RTE_PMD_PARAM_UNSET; 282 283 /* 284 * Configurable value of buffered packets before sending. 285 */ 286 uint16_t noisy_tx_sw_bufsz; 287 288 /* 289 * Configurable value of packet buffer timeout. 290 */ 291 uint16_t noisy_tx_sw_buf_flush_time; 292 293 /* 294 * Configurable value for size of VNF internal memory area 295 * used for simulating noisy neighbour behaviour 296 */ 297 uint64_t noisy_lkup_mem_sz; 298 299 /* 300 * Configurable value of number of random writes done in 301 * VNF simulation memory area. 302 */ 303 uint64_t noisy_lkup_num_writes; 304 305 /* 306 * Configurable value of number of random reads done in 307 * VNF simulation memory area. 308 */ 309 uint64_t noisy_lkup_num_reads; 310 311 /* 312 * Configurable value of number of random reads/writes done in 313 * VNF simulation memory area. 314 */ 315 uint64_t noisy_lkup_num_reads_writes; 316 317 /* 318 * Receive Side Scaling (RSS) configuration. 319 */ 320 uint64_t rss_hf = ETH_RSS_IP; /* RSS IP by default. */ 321 322 /* 323 * Port topology configuration 324 */ 325 uint16_t port_topology = PORT_TOPOLOGY_PAIRED; /* Ports are paired by default */ 326 327 /* 328 * Avoids to flush all the RX streams before starts forwarding. 329 */ 330 uint8_t no_flush_rx = 0; /* flush by default */ 331 332 /* 333 * Flow API isolated mode. 334 */ 335 uint8_t flow_isolate_all; 336 337 /* 338 * Avoids to check link status when starting/stopping a port. 339 */ 340 uint8_t no_link_check = 0; /* check by default */ 341 342 /* 343 * Don't automatically start all ports in interactive mode. 344 */ 345 uint8_t no_device_start = 0; 346 347 /* 348 * Enable link status change notification 349 */ 350 uint8_t lsc_interrupt = 1; /* enabled by default */ 351 352 /* 353 * Enable device removal notification. 354 */ 355 uint8_t rmv_interrupt = 1; /* enabled by default */ 356 357 uint8_t hot_plug = 0; /**< hotplug disabled by default. */ 358 359 /* After attach, port setup is called on event or by iterator */ 360 bool setup_on_probe_event = true; 361 362 /* Clear ptypes on port initialization. */ 363 uint8_t clear_ptypes = true; 364 365 /* Pretty printing of ethdev events */ 366 static const char * const eth_event_desc[] = { 367 [RTE_ETH_EVENT_UNKNOWN] = "unknown", 368 [RTE_ETH_EVENT_INTR_LSC] = "link state change", 369 [RTE_ETH_EVENT_QUEUE_STATE] = "queue state", 370 [RTE_ETH_EVENT_INTR_RESET] = "reset", 371 [RTE_ETH_EVENT_VF_MBOX] = "VF mbox", 372 [RTE_ETH_EVENT_IPSEC] = "IPsec", 373 [RTE_ETH_EVENT_MACSEC] = "MACsec", 374 [RTE_ETH_EVENT_INTR_RMV] = "device removal", 375 [RTE_ETH_EVENT_NEW] = "device probed", 376 [RTE_ETH_EVENT_DESTROY] = "device released", 377 [RTE_ETH_EVENT_MAX] = NULL, 378 }; 379 380 /* 381 * Display or mask ether events 382 * Default to all events except VF_MBOX 383 */ 384 uint32_t event_print_mask = (UINT32_C(1) << RTE_ETH_EVENT_UNKNOWN) | 385 (UINT32_C(1) << RTE_ETH_EVENT_INTR_LSC) | 386 (UINT32_C(1) << RTE_ETH_EVENT_QUEUE_STATE) | 387 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RESET) | 388 (UINT32_C(1) << RTE_ETH_EVENT_IPSEC) | 389 (UINT32_C(1) << RTE_ETH_EVENT_MACSEC) | 390 (UINT32_C(1) << RTE_ETH_EVENT_INTR_RMV); 391 /* 392 * Decide if all memory are locked for performance. 393 */ 394 int do_mlockall = 0; 395 396 /* 397 * NIC bypass mode configuration options. 398 */ 399 400 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 401 /* The NIC bypass watchdog timeout. */ 402 uint32_t bypass_timeout = RTE_PMD_IXGBE_BYPASS_TMT_OFF; 403 #endif 404 405 406 #ifdef RTE_LIBRTE_LATENCY_STATS 407 408 /* 409 * Set when latency stats is enabled in the commandline 410 */ 411 uint8_t latencystats_enabled; 412 413 /* 414 * Lcore ID to serive latency statistics. 415 */ 416 lcoreid_t latencystats_lcore_id = -1; 417 418 #endif 419 420 /* 421 * Ethernet device configuration. 422 */ 423 struct rte_eth_rxmode rx_mode = { 424 .max_rx_pkt_len = RTE_ETHER_MAX_LEN, 425 /**< Default maximum frame length. */ 426 }; 427 428 struct rte_eth_txmode tx_mode = { 429 .offloads = DEV_TX_OFFLOAD_MBUF_FAST_FREE, 430 }; 431 432 struct rte_fdir_conf fdir_conf = { 433 .mode = RTE_FDIR_MODE_NONE, 434 .pballoc = RTE_FDIR_PBALLOC_64K, 435 .status = RTE_FDIR_REPORT_STATUS, 436 .mask = { 437 .vlan_tci_mask = 0xFFEF, 438 .ipv4_mask = { 439 .src_ip = 0xFFFFFFFF, 440 .dst_ip = 0xFFFFFFFF, 441 }, 442 .ipv6_mask = { 443 .src_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 444 .dst_ip = {0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, 445 }, 446 .src_port_mask = 0xFFFF, 447 .dst_port_mask = 0xFFFF, 448 .mac_addr_byte_mask = 0xFF, 449 .tunnel_type_mask = 1, 450 .tunnel_id_mask = 0xFFFFFFFF, 451 }, 452 .drop_queue = 127, 453 }; 454 455 volatile int test_done = 1; /* stop packet forwarding when set to 1. */ 456 457 struct queue_stats_mappings tx_queue_stats_mappings_array[MAX_TX_QUEUE_STATS_MAPPINGS]; 458 struct queue_stats_mappings rx_queue_stats_mappings_array[MAX_RX_QUEUE_STATS_MAPPINGS]; 459 460 struct queue_stats_mappings *tx_queue_stats_mappings = tx_queue_stats_mappings_array; 461 struct queue_stats_mappings *rx_queue_stats_mappings = rx_queue_stats_mappings_array; 462 463 uint16_t nb_tx_queue_stats_mappings = 0; 464 uint16_t nb_rx_queue_stats_mappings = 0; 465 466 /* 467 * Display zero values by default for xstats 468 */ 469 uint8_t xstats_hide_zero; 470 471 unsigned int num_sockets = 0; 472 unsigned int socket_ids[RTE_MAX_NUMA_NODES]; 473 474 #ifdef RTE_LIBRTE_BITRATE 475 /* Bitrate statistics */ 476 struct rte_stats_bitrates *bitrate_data; 477 lcoreid_t bitrate_lcore_id; 478 uint8_t bitrate_enabled; 479 #endif 480 481 struct gro_status gro_ports[RTE_MAX_ETHPORTS]; 482 uint8_t gro_flush_cycles = GRO_DEFAULT_FLUSH_CYCLES; 483 484 /* Forward function declarations */ 485 static void setup_attached_port(portid_t pi); 486 static void map_port_queue_stats_mapping_registers(portid_t pi, 487 struct rte_port *port); 488 static void check_all_ports_link_status(uint32_t port_mask); 489 static int eth_event_callback(portid_t port_id, 490 enum rte_eth_event_type type, 491 void *param, void *ret_param); 492 static void dev_event_callback(const char *device_name, 493 enum rte_dev_event_type type, 494 void *param); 495 496 /* 497 * Check if all the ports are started. 498 * If yes, return positive value. If not, return zero. 499 */ 500 static int all_ports_started(void); 501 502 struct gso_status gso_ports[RTE_MAX_ETHPORTS]; 503 uint16_t gso_max_segment_size = RTE_ETHER_MAX_LEN - RTE_ETHER_CRC_LEN; 504 505 /* Holds the registered mbuf dynamic flags names. */ 506 char dynf_names[64][RTE_MBUF_DYN_NAMESIZE]; 507 508 /* 509 * Helper function to check if socket is already discovered. 510 * If yes, return positive value. If not, return zero. 511 */ 512 int 513 new_socket_id(unsigned int socket_id) 514 { 515 unsigned int i; 516 517 for (i = 0; i < num_sockets; i++) { 518 if (socket_ids[i] == socket_id) 519 return 0; 520 } 521 return 1; 522 } 523 524 /* 525 * Setup default configuration. 526 */ 527 static void 528 set_default_fwd_lcores_config(void) 529 { 530 unsigned int i; 531 unsigned int nb_lc; 532 unsigned int sock_num; 533 534 nb_lc = 0; 535 for (i = 0; i < RTE_MAX_LCORE; i++) { 536 if (!rte_lcore_is_enabled(i)) 537 continue; 538 sock_num = rte_lcore_to_socket_id(i); 539 if (new_socket_id(sock_num)) { 540 if (num_sockets >= RTE_MAX_NUMA_NODES) { 541 rte_exit(EXIT_FAILURE, 542 "Total sockets greater than %u\n", 543 RTE_MAX_NUMA_NODES); 544 } 545 socket_ids[num_sockets++] = sock_num; 546 } 547 if (i == rte_get_master_lcore()) 548 continue; 549 fwd_lcores_cpuids[nb_lc++] = i; 550 } 551 nb_lcores = (lcoreid_t) nb_lc; 552 nb_cfg_lcores = nb_lcores; 553 nb_fwd_lcores = 1; 554 } 555 556 static void 557 set_def_peer_eth_addrs(void) 558 { 559 portid_t i; 560 561 for (i = 0; i < RTE_MAX_ETHPORTS; i++) { 562 peer_eth_addrs[i].addr_bytes[0] = RTE_ETHER_LOCAL_ADMIN_ADDR; 563 peer_eth_addrs[i].addr_bytes[5] = i; 564 } 565 } 566 567 static void 568 set_default_fwd_ports_config(void) 569 { 570 portid_t pt_id; 571 int i = 0; 572 573 RTE_ETH_FOREACH_DEV(pt_id) { 574 fwd_ports_ids[i++] = pt_id; 575 576 /* Update sockets info according to the attached device */ 577 int socket_id = rte_eth_dev_socket_id(pt_id); 578 if (socket_id >= 0 && new_socket_id(socket_id)) { 579 if (num_sockets >= RTE_MAX_NUMA_NODES) { 580 rte_exit(EXIT_FAILURE, 581 "Total sockets greater than %u\n", 582 RTE_MAX_NUMA_NODES); 583 } 584 socket_ids[num_sockets++] = socket_id; 585 } 586 } 587 588 nb_cfg_ports = nb_ports; 589 nb_fwd_ports = nb_ports; 590 } 591 592 void 593 set_def_fwd_config(void) 594 { 595 set_default_fwd_lcores_config(); 596 set_def_peer_eth_addrs(); 597 set_default_fwd_ports_config(); 598 } 599 600 /* extremely pessimistic estimation of memory required to create a mempool */ 601 static int 602 calc_mem_size(uint32_t nb_mbufs, uint32_t mbuf_sz, size_t pgsz, size_t *out) 603 { 604 unsigned int n_pages, mbuf_per_pg, leftover; 605 uint64_t total_mem, mbuf_mem, obj_sz; 606 607 /* there is no good way to predict how much space the mempool will 608 * occupy because it will allocate chunks on the fly, and some of those 609 * will come from default DPDK memory while some will come from our 610 * external memory, so just assume 128MB will be enough for everyone. 611 */ 612 uint64_t hdr_mem = 128 << 20; 613 614 /* account for possible non-contiguousness */ 615 obj_sz = rte_mempool_calc_obj_size(mbuf_sz, 0, NULL); 616 if (obj_sz > pgsz) { 617 TESTPMD_LOG(ERR, "Object size is bigger than page size\n"); 618 return -1; 619 } 620 621 mbuf_per_pg = pgsz / obj_sz; 622 leftover = (nb_mbufs % mbuf_per_pg) > 0; 623 n_pages = (nb_mbufs / mbuf_per_pg) + leftover; 624 625 mbuf_mem = n_pages * pgsz; 626 627 total_mem = RTE_ALIGN(hdr_mem + mbuf_mem, pgsz); 628 629 if (total_mem > SIZE_MAX) { 630 TESTPMD_LOG(ERR, "Memory size too big\n"); 631 return -1; 632 } 633 *out = (size_t)total_mem; 634 635 return 0; 636 } 637 638 static int 639 pagesz_flags(uint64_t page_sz) 640 { 641 /* as per mmap() manpage, all page sizes are log2 of page size 642 * shifted by MAP_HUGE_SHIFT 643 */ 644 int log2 = rte_log2_u64(page_sz); 645 646 return (log2 << HUGE_SHIFT); 647 } 648 649 static void * 650 alloc_mem(size_t memsz, size_t pgsz, bool huge) 651 { 652 void *addr; 653 int flags; 654 655 /* allocate anonymous hugepages */ 656 flags = MAP_ANONYMOUS | MAP_PRIVATE; 657 if (huge) 658 flags |= HUGE_FLAG | pagesz_flags(pgsz); 659 660 addr = mmap(NULL, memsz, PROT_READ | PROT_WRITE, flags, -1, 0); 661 if (addr == MAP_FAILED) 662 return NULL; 663 664 return addr; 665 } 666 667 struct extmem_param { 668 void *addr; 669 size_t len; 670 size_t pgsz; 671 rte_iova_t *iova_table; 672 unsigned int iova_table_len; 673 }; 674 675 static int 676 create_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, struct extmem_param *param, 677 bool huge) 678 { 679 uint64_t pgsizes[] = {RTE_PGSIZE_2M, RTE_PGSIZE_1G, /* x86_64, ARM */ 680 RTE_PGSIZE_16M, RTE_PGSIZE_16G}; /* POWER */ 681 unsigned int cur_page, n_pages, pgsz_idx; 682 size_t mem_sz, cur_pgsz; 683 rte_iova_t *iovas = NULL; 684 void *addr; 685 int ret; 686 687 for (pgsz_idx = 0; pgsz_idx < RTE_DIM(pgsizes); pgsz_idx++) { 688 /* skip anything that is too big */ 689 if (pgsizes[pgsz_idx] > SIZE_MAX) 690 continue; 691 692 cur_pgsz = pgsizes[pgsz_idx]; 693 694 /* if we were told not to allocate hugepages, override */ 695 if (!huge) 696 cur_pgsz = sysconf(_SC_PAGESIZE); 697 698 ret = calc_mem_size(nb_mbufs, mbuf_sz, cur_pgsz, &mem_sz); 699 if (ret < 0) { 700 TESTPMD_LOG(ERR, "Cannot calculate memory size\n"); 701 return -1; 702 } 703 704 /* allocate our memory */ 705 addr = alloc_mem(mem_sz, cur_pgsz, huge); 706 707 /* if we couldn't allocate memory with a specified page size, 708 * that doesn't mean we can't do it with other page sizes, so 709 * try another one. 710 */ 711 if (addr == NULL) 712 continue; 713 714 /* store IOVA addresses for every page in this memory area */ 715 n_pages = mem_sz / cur_pgsz; 716 717 iovas = malloc(sizeof(*iovas) * n_pages); 718 719 if (iovas == NULL) { 720 TESTPMD_LOG(ERR, "Cannot allocate memory for iova addresses\n"); 721 goto fail; 722 } 723 /* lock memory if it's not huge pages */ 724 if (!huge) 725 mlock(addr, mem_sz); 726 727 /* populate IOVA addresses */ 728 for (cur_page = 0; cur_page < n_pages; cur_page++) { 729 rte_iova_t iova; 730 size_t offset; 731 void *cur; 732 733 offset = cur_pgsz * cur_page; 734 cur = RTE_PTR_ADD(addr, offset); 735 736 /* touch the page before getting its IOVA */ 737 *(volatile char *)cur = 0; 738 739 iova = rte_mem_virt2iova(cur); 740 741 iovas[cur_page] = iova; 742 } 743 744 break; 745 } 746 /* if we couldn't allocate anything */ 747 if (iovas == NULL) 748 return -1; 749 750 param->addr = addr; 751 param->len = mem_sz; 752 param->pgsz = cur_pgsz; 753 param->iova_table = iovas; 754 param->iova_table_len = n_pages; 755 756 return 0; 757 fail: 758 if (iovas) 759 free(iovas); 760 if (addr) 761 munmap(addr, mem_sz); 762 763 return -1; 764 } 765 766 static int 767 setup_extmem(uint32_t nb_mbufs, uint32_t mbuf_sz, bool huge) 768 { 769 struct extmem_param param; 770 int socket_id, ret; 771 772 memset(¶m, 0, sizeof(param)); 773 774 /* check if our heap exists */ 775 socket_id = rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 776 if (socket_id < 0) { 777 /* create our heap */ 778 ret = rte_malloc_heap_create(EXTMEM_HEAP_NAME); 779 if (ret < 0) { 780 TESTPMD_LOG(ERR, "Cannot create heap\n"); 781 return -1; 782 } 783 } 784 785 ret = create_extmem(nb_mbufs, mbuf_sz, ¶m, huge); 786 if (ret < 0) { 787 TESTPMD_LOG(ERR, "Cannot create memory area\n"); 788 return -1; 789 } 790 791 /* we now have a valid memory area, so add it to heap */ 792 ret = rte_malloc_heap_memory_add(EXTMEM_HEAP_NAME, 793 param.addr, param.len, param.iova_table, 794 param.iova_table_len, param.pgsz); 795 796 /* when using VFIO, memory is automatically mapped for DMA by EAL */ 797 798 /* not needed any more */ 799 free(param.iova_table); 800 801 if (ret < 0) { 802 TESTPMD_LOG(ERR, "Cannot add memory to heap\n"); 803 munmap(param.addr, param.len); 804 return -1; 805 } 806 807 /* success */ 808 809 TESTPMD_LOG(DEBUG, "Allocated %zuMB of external memory\n", 810 param.len >> 20); 811 812 return 0; 813 } 814 static void 815 dma_unmap_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 816 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 817 { 818 uint16_t pid = 0; 819 int ret; 820 821 RTE_ETH_FOREACH_DEV(pid) { 822 struct rte_eth_dev *dev = 823 &rte_eth_devices[pid]; 824 825 ret = rte_dev_dma_unmap(dev->device, memhdr->addr, 0, 826 memhdr->len); 827 if (ret) { 828 TESTPMD_LOG(DEBUG, 829 "unable to DMA unmap addr 0x%p " 830 "for device %s\n", 831 memhdr->addr, dev->data->name); 832 } 833 } 834 ret = rte_extmem_unregister(memhdr->addr, memhdr->len); 835 if (ret) { 836 TESTPMD_LOG(DEBUG, 837 "unable to un-register addr 0x%p\n", memhdr->addr); 838 } 839 } 840 841 static void 842 dma_map_cb(struct rte_mempool *mp __rte_unused, void *opaque __rte_unused, 843 struct rte_mempool_memhdr *memhdr, unsigned mem_idx __rte_unused) 844 { 845 uint16_t pid = 0; 846 size_t page_size = sysconf(_SC_PAGESIZE); 847 int ret; 848 849 ret = rte_extmem_register(memhdr->addr, memhdr->len, NULL, 0, 850 page_size); 851 if (ret) { 852 TESTPMD_LOG(DEBUG, 853 "unable to register addr 0x%p\n", memhdr->addr); 854 return; 855 } 856 RTE_ETH_FOREACH_DEV(pid) { 857 struct rte_eth_dev *dev = 858 &rte_eth_devices[pid]; 859 860 ret = rte_dev_dma_map(dev->device, memhdr->addr, 0, 861 memhdr->len); 862 if (ret) { 863 TESTPMD_LOG(DEBUG, 864 "unable to DMA map addr 0x%p " 865 "for device %s\n", 866 memhdr->addr, dev->data->name); 867 } 868 } 869 } 870 871 /* 872 * Configuration initialisation done once at init time. 873 */ 874 static struct rte_mempool * 875 mbuf_pool_create(uint16_t mbuf_seg_size, unsigned nb_mbuf, 876 unsigned int socket_id) 877 { 878 char pool_name[RTE_MEMPOOL_NAMESIZE]; 879 struct rte_mempool *rte_mp = NULL; 880 uint32_t mb_size; 881 882 mb_size = sizeof(struct rte_mbuf) + mbuf_seg_size; 883 mbuf_poolname_build(socket_id, pool_name, sizeof(pool_name)); 884 885 TESTPMD_LOG(INFO, 886 "create a new mbuf pool <%s>: n=%u, size=%u, socket=%u\n", 887 pool_name, nb_mbuf, mbuf_seg_size, socket_id); 888 889 switch (mp_alloc_type) { 890 case MP_ALLOC_NATIVE: 891 { 892 /* wrapper to rte_mempool_create() */ 893 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 894 rte_mbuf_best_mempool_ops()); 895 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 896 mb_mempool_cache, 0, mbuf_seg_size, socket_id); 897 break; 898 } 899 case MP_ALLOC_ANON: 900 { 901 rte_mp = rte_mempool_create_empty(pool_name, nb_mbuf, 902 mb_size, (unsigned int) mb_mempool_cache, 903 sizeof(struct rte_pktmbuf_pool_private), 904 socket_id, mempool_flags); 905 if (rte_mp == NULL) 906 goto err; 907 908 if (rte_mempool_populate_anon(rte_mp) == 0) { 909 rte_mempool_free(rte_mp); 910 rte_mp = NULL; 911 goto err; 912 } 913 rte_pktmbuf_pool_init(rte_mp, NULL); 914 rte_mempool_obj_iter(rte_mp, rte_pktmbuf_init, NULL); 915 rte_mempool_mem_iter(rte_mp, dma_map_cb, NULL); 916 break; 917 } 918 case MP_ALLOC_XMEM: 919 case MP_ALLOC_XMEM_HUGE: 920 { 921 int heap_socket; 922 bool huge = mp_alloc_type == MP_ALLOC_XMEM_HUGE; 923 924 if (setup_extmem(nb_mbuf, mbuf_seg_size, huge) < 0) 925 rte_exit(EXIT_FAILURE, "Could not create external memory\n"); 926 927 heap_socket = 928 rte_malloc_heap_get_socket(EXTMEM_HEAP_NAME); 929 if (heap_socket < 0) 930 rte_exit(EXIT_FAILURE, "Could not get external memory socket ID\n"); 931 932 TESTPMD_LOG(INFO, "preferred mempool ops selected: %s\n", 933 rte_mbuf_best_mempool_ops()); 934 rte_mp = rte_pktmbuf_pool_create(pool_name, nb_mbuf, 935 mb_mempool_cache, 0, mbuf_seg_size, 936 heap_socket); 937 break; 938 } 939 default: 940 { 941 rte_exit(EXIT_FAILURE, "Invalid mempool creation mode\n"); 942 } 943 } 944 945 err: 946 if (rte_mp == NULL) { 947 rte_exit(EXIT_FAILURE, 948 "Creation of mbuf pool for socket %u failed: %s\n", 949 socket_id, rte_strerror(rte_errno)); 950 } else if (verbose_level > 0) { 951 rte_mempool_dump(stdout, rte_mp); 952 } 953 return rte_mp; 954 } 955 956 /* 957 * Check given socket id is valid or not with NUMA mode, 958 * if valid, return 0, else return -1 959 */ 960 static int 961 check_socket_id(const unsigned int socket_id) 962 { 963 static int warning_once = 0; 964 965 if (new_socket_id(socket_id)) { 966 if (!warning_once && numa_support) 967 printf("Warning: NUMA should be configured manually by" 968 " using --port-numa-config and" 969 " --ring-numa-config parameters along with" 970 " --numa.\n"); 971 warning_once = 1; 972 return -1; 973 } 974 return 0; 975 } 976 977 /* 978 * Get the allowed maximum number of RX queues. 979 * *pid return the port id which has minimal value of 980 * max_rx_queues in all ports. 981 */ 982 queueid_t 983 get_allowed_max_nb_rxq(portid_t *pid) 984 { 985 queueid_t allowed_max_rxq = RTE_MAX_QUEUES_PER_PORT; 986 bool max_rxq_valid = false; 987 portid_t pi; 988 struct rte_eth_dev_info dev_info; 989 990 RTE_ETH_FOREACH_DEV(pi) { 991 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 992 continue; 993 994 max_rxq_valid = true; 995 if (dev_info.max_rx_queues < allowed_max_rxq) { 996 allowed_max_rxq = dev_info.max_rx_queues; 997 *pid = pi; 998 } 999 } 1000 return max_rxq_valid ? allowed_max_rxq : 0; 1001 } 1002 1003 /* 1004 * Check input rxq is valid or not. 1005 * If input rxq is not greater than any of maximum number 1006 * of RX queues of all ports, it is valid. 1007 * if valid, return 0, else return -1 1008 */ 1009 int 1010 check_nb_rxq(queueid_t rxq) 1011 { 1012 queueid_t allowed_max_rxq; 1013 portid_t pid = 0; 1014 1015 allowed_max_rxq = get_allowed_max_nb_rxq(&pid); 1016 if (rxq > allowed_max_rxq) { 1017 printf("Fail: input rxq (%u) can't be greater " 1018 "than max_rx_queues (%u) of port %u\n", 1019 rxq, 1020 allowed_max_rxq, 1021 pid); 1022 return -1; 1023 } 1024 return 0; 1025 } 1026 1027 /* 1028 * Get the allowed maximum number of TX queues. 1029 * *pid return the port id which has minimal value of 1030 * max_tx_queues in all ports. 1031 */ 1032 queueid_t 1033 get_allowed_max_nb_txq(portid_t *pid) 1034 { 1035 queueid_t allowed_max_txq = RTE_MAX_QUEUES_PER_PORT; 1036 bool max_txq_valid = false; 1037 portid_t pi; 1038 struct rte_eth_dev_info dev_info; 1039 1040 RTE_ETH_FOREACH_DEV(pi) { 1041 if (eth_dev_info_get_print_err(pi, &dev_info) != 0) 1042 continue; 1043 1044 max_txq_valid = true; 1045 if (dev_info.max_tx_queues < allowed_max_txq) { 1046 allowed_max_txq = dev_info.max_tx_queues; 1047 *pid = pi; 1048 } 1049 } 1050 return max_txq_valid ? allowed_max_txq : 0; 1051 } 1052 1053 /* 1054 * Check input txq is valid or not. 1055 * If input txq is not greater than any of maximum number 1056 * of TX queues of all ports, it is valid. 1057 * if valid, return 0, else return -1 1058 */ 1059 int 1060 check_nb_txq(queueid_t txq) 1061 { 1062 queueid_t allowed_max_txq; 1063 portid_t pid = 0; 1064 1065 allowed_max_txq = get_allowed_max_nb_txq(&pid); 1066 if (txq > allowed_max_txq) { 1067 printf("Fail: input txq (%u) can't be greater " 1068 "than max_tx_queues (%u) of port %u\n", 1069 txq, 1070 allowed_max_txq, 1071 pid); 1072 return -1; 1073 } 1074 return 0; 1075 } 1076 1077 /* 1078 * Get the allowed maximum number of hairpin queues. 1079 * *pid return the port id which has minimal value of 1080 * max_hairpin_queues in all ports. 1081 */ 1082 queueid_t 1083 get_allowed_max_nb_hairpinq(portid_t *pid) 1084 { 1085 queueid_t allowed_max_hairpinq = RTE_MAX_QUEUES_PER_PORT; 1086 portid_t pi; 1087 struct rte_eth_hairpin_cap cap; 1088 1089 RTE_ETH_FOREACH_DEV(pi) { 1090 if (rte_eth_dev_hairpin_capability_get(pi, &cap) != 0) { 1091 *pid = pi; 1092 return 0; 1093 } 1094 if (cap.max_nb_queues < allowed_max_hairpinq) { 1095 allowed_max_hairpinq = cap.max_nb_queues; 1096 *pid = pi; 1097 } 1098 } 1099 return allowed_max_hairpinq; 1100 } 1101 1102 /* 1103 * Check input hairpin is valid or not. 1104 * If input hairpin is not greater than any of maximum number 1105 * of hairpin queues of all ports, it is valid. 1106 * if valid, return 0, else return -1 1107 */ 1108 int 1109 check_nb_hairpinq(queueid_t hairpinq) 1110 { 1111 queueid_t allowed_max_hairpinq; 1112 portid_t pid = 0; 1113 1114 allowed_max_hairpinq = get_allowed_max_nb_hairpinq(&pid); 1115 if (hairpinq > allowed_max_hairpinq) { 1116 printf("Fail: input hairpin (%u) can't be greater " 1117 "than max_hairpin_queues (%u) of port %u\n", 1118 hairpinq, allowed_max_hairpinq, pid); 1119 return -1; 1120 } 1121 return 0; 1122 } 1123 1124 static void 1125 init_config(void) 1126 { 1127 portid_t pid; 1128 struct rte_port *port; 1129 struct rte_mempool *mbp; 1130 unsigned int nb_mbuf_per_pool; 1131 lcoreid_t lc_id; 1132 uint8_t port_per_socket[RTE_MAX_NUMA_NODES]; 1133 struct rte_gro_param gro_param; 1134 uint32_t gso_types; 1135 uint16_t data_size; 1136 bool warning = 0; 1137 int k; 1138 int ret; 1139 1140 memset(port_per_socket,0,RTE_MAX_NUMA_NODES); 1141 1142 /* Configuration of logical cores. */ 1143 fwd_lcores = rte_zmalloc("testpmd: fwd_lcores", 1144 sizeof(struct fwd_lcore *) * nb_lcores, 1145 RTE_CACHE_LINE_SIZE); 1146 if (fwd_lcores == NULL) { 1147 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d (struct fwd_lcore *)) " 1148 "failed\n", nb_lcores); 1149 } 1150 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1151 fwd_lcores[lc_id] = rte_zmalloc("testpmd: struct fwd_lcore", 1152 sizeof(struct fwd_lcore), 1153 RTE_CACHE_LINE_SIZE); 1154 if (fwd_lcores[lc_id] == NULL) { 1155 rte_exit(EXIT_FAILURE, "rte_zmalloc(struct fwd_lcore) " 1156 "failed\n"); 1157 } 1158 fwd_lcores[lc_id]->cpuid_idx = lc_id; 1159 } 1160 1161 RTE_ETH_FOREACH_DEV(pid) { 1162 port = &ports[pid]; 1163 /* Apply default TxRx configuration for all ports */ 1164 port->dev_conf.txmode = tx_mode; 1165 port->dev_conf.rxmode = rx_mode; 1166 1167 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 1168 if (ret != 0) 1169 rte_exit(EXIT_FAILURE, 1170 "rte_eth_dev_info_get() failed\n"); 1171 1172 if (!(port->dev_info.tx_offload_capa & 1173 DEV_TX_OFFLOAD_MBUF_FAST_FREE)) 1174 port->dev_conf.txmode.offloads &= 1175 ~DEV_TX_OFFLOAD_MBUF_FAST_FREE; 1176 if (numa_support) { 1177 if (port_numa[pid] != NUMA_NO_CONFIG) 1178 port_per_socket[port_numa[pid]]++; 1179 else { 1180 uint32_t socket_id = rte_eth_dev_socket_id(pid); 1181 1182 /* 1183 * if socket_id is invalid, 1184 * set to the first available socket. 1185 */ 1186 if (check_socket_id(socket_id) < 0) 1187 socket_id = socket_ids[0]; 1188 port_per_socket[socket_id]++; 1189 } 1190 } 1191 1192 /* Apply Rx offloads configuration */ 1193 for (k = 0; k < port->dev_info.max_rx_queues; k++) 1194 port->rx_conf[k].offloads = 1195 port->dev_conf.rxmode.offloads; 1196 /* Apply Tx offloads configuration */ 1197 for (k = 0; k < port->dev_info.max_tx_queues; k++) 1198 port->tx_conf[k].offloads = 1199 port->dev_conf.txmode.offloads; 1200 1201 /* set flag to initialize port/queue */ 1202 port->need_reconfig = 1; 1203 port->need_reconfig_queues = 1; 1204 port->tx_metadata = 0; 1205 1206 /* Check for maximum number of segments per MTU. Accordingly 1207 * update the mbuf data size. 1208 */ 1209 if (port->dev_info.rx_desc_lim.nb_mtu_seg_max != UINT16_MAX && 1210 port->dev_info.rx_desc_lim.nb_mtu_seg_max != 0) { 1211 data_size = rx_mode.max_rx_pkt_len / 1212 port->dev_info.rx_desc_lim.nb_mtu_seg_max; 1213 1214 if ((data_size + RTE_PKTMBUF_HEADROOM) > 1215 mbuf_data_size) { 1216 mbuf_data_size = data_size + 1217 RTE_PKTMBUF_HEADROOM; 1218 warning = 1; 1219 } 1220 } 1221 } 1222 1223 if (warning) 1224 TESTPMD_LOG(WARNING, "Configured mbuf size %hu\n", 1225 mbuf_data_size); 1226 1227 /* 1228 * Create pools of mbuf. 1229 * If NUMA support is disabled, create a single pool of mbuf in 1230 * socket 0 memory by default. 1231 * Otherwise, create a pool of mbuf in the memory of sockets 0 and 1. 1232 * 1233 * Use the maximum value of nb_rxd and nb_txd here, then nb_rxd and 1234 * nb_txd can be configured at run time. 1235 */ 1236 if (param_total_num_mbufs) 1237 nb_mbuf_per_pool = param_total_num_mbufs; 1238 else { 1239 nb_mbuf_per_pool = RTE_TEST_RX_DESC_MAX + 1240 (nb_lcores * mb_mempool_cache) + 1241 RTE_TEST_TX_DESC_MAX + MAX_PKT_BURST; 1242 nb_mbuf_per_pool *= RTE_MAX_ETHPORTS; 1243 } 1244 1245 if (numa_support) { 1246 uint8_t i; 1247 1248 for (i = 0; i < num_sockets; i++) 1249 mempools[i] = mbuf_pool_create(mbuf_data_size, 1250 nb_mbuf_per_pool, 1251 socket_ids[i]); 1252 } else { 1253 if (socket_num == UMA_NO_CONFIG) 1254 mempools[0] = mbuf_pool_create(mbuf_data_size, 1255 nb_mbuf_per_pool, 0); 1256 else 1257 mempools[socket_num] = mbuf_pool_create 1258 (mbuf_data_size, 1259 nb_mbuf_per_pool, 1260 socket_num); 1261 } 1262 1263 init_port_config(); 1264 1265 gso_types = DEV_TX_OFFLOAD_TCP_TSO | DEV_TX_OFFLOAD_VXLAN_TNL_TSO | 1266 DEV_TX_OFFLOAD_GRE_TNL_TSO | DEV_TX_OFFLOAD_UDP_TSO; 1267 /* 1268 * Records which Mbuf pool to use by each logical core, if needed. 1269 */ 1270 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1271 mbp = mbuf_pool_find( 1272 rte_lcore_to_socket_id(fwd_lcores_cpuids[lc_id])); 1273 1274 if (mbp == NULL) 1275 mbp = mbuf_pool_find(0); 1276 fwd_lcores[lc_id]->mbp = mbp; 1277 /* initialize GSO context */ 1278 fwd_lcores[lc_id]->gso_ctx.direct_pool = mbp; 1279 fwd_lcores[lc_id]->gso_ctx.indirect_pool = mbp; 1280 fwd_lcores[lc_id]->gso_ctx.gso_types = gso_types; 1281 fwd_lcores[lc_id]->gso_ctx.gso_size = RTE_ETHER_MAX_LEN - 1282 RTE_ETHER_CRC_LEN; 1283 fwd_lcores[lc_id]->gso_ctx.flag = 0; 1284 } 1285 1286 /* Configuration of packet forwarding streams. */ 1287 if (init_fwd_streams() < 0) 1288 rte_exit(EXIT_FAILURE, "FAIL from init_fwd_streams()\n"); 1289 1290 fwd_config_setup(); 1291 1292 /* create a gro context for each lcore */ 1293 gro_param.gro_types = RTE_GRO_TCP_IPV4; 1294 gro_param.max_flow_num = GRO_MAX_FLUSH_CYCLES; 1295 gro_param.max_item_per_flow = MAX_PKT_BURST; 1296 for (lc_id = 0; lc_id < nb_lcores; lc_id++) { 1297 gro_param.socket_id = rte_lcore_to_socket_id( 1298 fwd_lcores_cpuids[lc_id]); 1299 fwd_lcores[lc_id]->gro_ctx = rte_gro_ctx_create(&gro_param); 1300 if (fwd_lcores[lc_id]->gro_ctx == NULL) { 1301 rte_exit(EXIT_FAILURE, 1302 "rte_gro_ctx_create() failed\n"); 1303 } 1304 } 1305 1306 #if defined RTE_LIBRTE_PMD_SOFTNIC 1307 if (strcmp(cur_fwd_eng->fwd_mode_name, "softnic") == 0) { 1308 RTE_ETH_FOREACH_DEV(pid) { 1309 port = &ports[pid]; 1310 const char *driver = port->dev_info.driver_name; 1311 1312 if (strcmp(driver, "net_softnic") == 0) 1313 port->softport.fwd_lcore_arg = fwd_lcores; 1314 } 1315 } 1316 #endif 1317 1318 } 1319 1320 1321 void 1322 reconfig(portid_t new_port_id, unsigned socket_id) 1323 { 1324 struct rte_port *port; 1325 int ret; 1326 1327 /* Reconfiguration of Ethernet ports. */ 1328 port = &ports[new_port_id]; 1329 1330 ret = eth_dev_info_get_print_err(new_port_id, &port->dev_info); 1331 if (ret != 0) 1332 return; 1333 1334 /* set flag to initialize port/queue */ 1335 port->need_reconfig = 1; 1336 port->need_reconfig_queues = 1; 1337 port->socket_id = socket_id; 1338 1339 init_port_config(); 1340 } 1341 1342 1343 int 1344 init_fwd_streams(void) 1345 { 1346 portid_t pid; 1347 struct rte_port *port; 1348 streamid_t sm_id, nb_fwd_streams_new; 1349 queueid_t q; 1350 1351 /* set socket id according to numa or not */ 1352 RTE_ETH_FOREACH_DEV(pid) { 1353 port = &ports[pid]; 1354 if (nb_rxq > port->dev_info.max_rx_queues) { 1355 printf("Fail: nb_rxq(%d) is greater than " 1356 "max_rx_queues(%d)\n", nb_rxq, 1357 port->dev_info.max_rx_queues); 1358 return -1; 1359 } 1360 if (nb_txq > port->dev_info.max_tx_queues) { 1361 printf("Fail: nb_txq(%d) is greater than " 1362 "max_tx_queues(%d)\n", nb_txq, 1363 port->dev_info.max_tx_queues); 1364 return -1; 1365 } 1366 if (numa_support) { 1367 if (port_numa[pid] != NUMA_NO_CONFIG) 1368 port->socket_id = port_numa[pid]; 1369 else { 1370 port->socket_id = rte_eth_dev_socket_id(pid); 1371 1372 /* 1373 * if socket_id is invalid, 1374 * set to the first available socket. 1375 */ 1376 if (check_socket_id(port->socket_id) < 0) 1377 port->socket_id = socket_ids[0]; 1378 } 1379 } 1380 else { 1381 if (socket_num == UMA_NO_CONFIG) 1382 port->socket_id = 0; 1383 else 1384 port->socket_id = socket_num; 1385 } 1386 } 1387 1388 q = RTE_MAX(nb_rxq, nb_txq); 1389 if (q == 0) { 1390 printf("Fail: Cannot allocate fwd streams as number of queues is 0\n"); 1391 return -1; 1392 } 1393 nb_fwd_streams_new = (streamid_t)(nb_ports * q); 1394 if (nb_fwd_streams_new == nb_fwd_streams) 1395 return 0; 1396 /* clear the old */ 1397 if (fwd_streams != NULL) { 1398 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1399 if (fwd_streams[sm_id] == NULL) 1400 continue; 1401 rte_free(fwd_streams[sm_id]); 1402 fwd_streams[sm_id] = NULL; 1403 } 1404 rte_free(fwd_streams); 1405 fwd_streams = NULL; 1406 } 1407 1408 /* init new */ 1409 nb_fwd_streams = nb_fwd_streams_new; 1410 if (nb_fwd_streams) { 1411 fwd_streams = rte_zmalloc("testpmd: fwd_streams", 1412 sizeof(struct fwd_stream *) * nb_fwd_streams, 1413 RTE_CACHE_LINE_SIZE); 1414 if (fwd_streams == NULL) 1415 rte_exit(EXIT_FAILURE, "rte_zmalloc(%d" 1416 " (struct fwd_stream *)) failed\n", 1417 nb_fwd_streams); 1418 1419 for (sm_id = 0; sm_id < nb_fwd_streams; sm_id++) { 1420 fwd_streams[sm_id] = rte_zmalloc("testpmd:" 1421 " struct fwd_stream", sizeof(struct fwd_stream), 1422 RTE_CACHE_LINE_SIZE); 1423 if (fwd_streams[sm_id] == NULL) 1424 rte_exit(EXIT_FAILURE, "rte_zmalloc" 1425 "(struct fwd_stream) failed\n"); 1426 } 1427 } 1428 1429 return 0; 1430 } 1431 1432 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1433 static void 1434 pkt_burst_stats_display(const char *rx_tx, struct pkt_burst_stats *pbs) 1435 { 1436 unsigned int total_burst; 1437 unsigned int nb_burst; 1438 unsigned int burst_stats[3]; 1439 uint16_t pktnb_stats[3]; 1440 uint16_t nb_pkt; 1441 int burst_percent[3]; 1442 1443 /* 1444 * First compute the total number of packet bursts and the 1445 * two highest numbers of bursts of the same number of packets. 1446 */ 1447 total_burst = 0; 1448 burst_stats[0] = burst_stats[1] = burst_stats[2] = 0; 1449 pktnb_stats[0] = pktnb_stats[1] = pktnb_stats[2] = 0; 1450 for (nb_pkt = 0; nb_pkt < MAX_PKT_BURST; nb_pkt++) { 1451 nb_burst = pbs->pkt_burst_spread[nb_pkt]; 1452 if (nb_burst == 0) 1453 continue; 1454 total_burst += nb_burst; 1455 if (nb_burst > burst_stats[0]) { 1456 burst_stats[1] = burst_stats[0]; 1457 pktnb_stats[1] = pktnb_stats[0]; 1458 burst_stats[0] = nb_burst; 1459 pktnb_stats[0] = nb_pkt; 1460 } else if (nb_burst > burst_stats[1]) { 1461 burst_stats[1] = nb_burst; 1462 pktnb_stats[1] = nb_pkt; 1463 } 1464 } 1465 if (total_burst == 0) 1466 return; 1467 burst_percent[0] = (burst_stats[0] * 100) / total_burst; 1468 printf(" %s-bursts : %u [%d%% of %d pkts", rx_tx, total_burst, 1469 burst_percent[0], (int) pktnb_stats[0]); 1470 if (burst_stats[0] == total_burst) { 1471 printf("]\n"); 1472 return; 1473 } 1474 if (burst_stats[0] + burst_stats[1] == total_burst) { 1475 printf(" + %d%% of %d pkts]\n", 1476 100 - burst_percent[0], pktnb_stats[1]); 1477 return; 1478 } 1479 burst_percent[1] = (burst_stats[1] * 100) / total_burst; 1480 burst_percent[2] = 100 - (burst_percent[0] + burst_percent[1]); 1481 if ((burst_percent[1] == 0) || (burst_percent[2] == 0)) { 1482 printf(" + %d%% of others]\n", 100 - burst_percent[0]); 1483 return; 1484 } 1485 printf(" + %d%% of %d pkts + %d%% of others]\n", 1486 burst_percent[1], (int) pktnb_stats[1], burst_percent[2]); 1487 } 1488 #endif /* RTE_TEST_PMD_RECORD_BURST_STATS */ 1489 1490 static void 1491 fwd_stream_stats_display(streamid_t stream_id) 1492 { 1493 struct fwd_stream *fs; 1494 static const char *fwd_top_stats_border = "-------"; 1495 1496 fs = fwd_streams[stream_id]; 1497 if ((fs->rx_packets == 0) && (fs->tx_packets == 0) && 1498 (fs->fwd_dropped == 0)) 1499 return; 1500 printf("\n %s Forward Stats for RX Port=%2d/Queue=%2d -> " 1501 "TX Port=%2d/Queue=%2d %s\n", 1502 fwd_top_stats_border, fs->rx_port, fs->rx_queue, 1503 fs->tx_port, fs->tx_queue, fwd_top_stats_border); 1504 printf(" RX-packets: %-14"PRIu64" TX-packets: %-14"PRIu64 1505 " TX-dropped: %-14"PRIu64, 1506 fs->rx_packets, fs->tx_packets, fs->fwd_dropped); 1507 1508 /* if checksum mode */ 1509 if (cur_fwd_eng == &csum_fwd_engine) { 1510 printf(" RX- bad IP checksum: %-14"PRIu64 1511 " Rx- bad L4 checksum: %-14"PRIu64 1512 " Rx- bad outer L4 checksum: %-14"PRIu64"\n", 1513 fs->rx_bad_ip_csum, fs->rx_bad_l4_csum, 1514 fs->rx_bad_outer_l4_csum); 1515 } else { 1516 printf("\n"); 1517 } 1518 1519 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1520 pkt_burst_stats_display("RX", &fs->rx_burst_stats); 1521 pkt_burst_stats_display("TX", &fs->tx_burst_stats); 1522 #endif 1523 } 1524 1525 void 1526 fwd_stats_display(void) 1527 { 1528 static const char *fwd_stats_border = "----------------------"; 1529 static const char *acc_stats_border = "+++++++++++++++"; 1530 struct { 1531 struct fwd_stream *rx_stream; 1532 struct fwd_stream *tx_stream; 1533 uint64_t tx_dropped; 1534 uint64_t rx_bad_ip_csum; 1535 uint64_t rx_bad_l4_csum; 1536 uint64_t rx_bad_outer_l4_csum; 1537 } ports_stats[RTE_MAX_ETHPORTS]; 1538 uint64_t total_rx_dropped = 0; 1539 uint64_t total_tx_dropped = 0; 1540 uint64_t total_rx_nombuf = 0; 1541 struct rte_eth_stats stats; 1542 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1543 uint64_t fwd_cycles = 0; 1544 #endif 1545 uint64_t total_recv = 0; 1546 uint64_t total_xmit = 0; 1547 struct rte_port *port; 1548 streamid_t sm_id; 1549 portid_t pt_id; 1550 int i; 1551 1552 memset(ports_stats, 0, sizeof(ports_stats)); 1553 1554 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1555 struct fwd_stream *fs = fwd_streams[sm_id]; 1556 1557 if (cur_fwd_config.nb_fwd_streams > 1558 cur_fwd_config.nb_fwd_ports) { 1559 fwd_stream_stats_display(sm_id); 1560 } else { 1561 ports_stats[fs->tx_port].tx_stream = fs; 1562 ports_stats[fs->rx_port].rx_stream = fs; 1563 } 1564 1565 ports_stats[fs->tx_port].tx_dropped += fs->fwd_dropped; 1566 1567 ports_stats[fs->rx_port].rx_bad_ip_csum += fs->rx_bad_ip_csum; 1568 ports_stats[fs->rx_port].rx_bad_l4_csum += fs->rx_bad_l4_csum; 1569 ports_stats[fs->rx_port].rx_bad_outer_l4_csum += 1570 fs->rx_bad_outer_l4_csum; 1571 1572 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1573 fwd_cycles += fs->core_cycles; 1574 #endif 1575 } 1576 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1577 uint8_t j; 1578 1579 pt_id = fwd_ports_ids[i]; 1580 port = &ports[pt_id]; 1581 1582 rte_eth_stats_get(pt_id, &stats); 1583 stats.ipackets -= port->stats.ipackets; 1584 stats.opackets -= port->stats.opackets; 1585 stats.ibytes -= port->stats.ibytes; 1586 stats.obytes -= port->stats.obytes; 1587 stats.imissed -= port->stats.imissed; 1588 stats.oerrors -= port->stats.oerrors; 1589 stats.rx_nombuf -= port->stats.rx_nombuf; 1590 1591 total_recv += stats.ipackets; 1592 total_xmit += stats.opackets; 1593 total_rx_dropped += stats.imissed; 1594 total_tx_dropped += ports_stats[pt_id].tx_dropped; 1595 total_tx_dropped += stats.oerrors; 1596 total_rx_nombuf += stats.rx_nombuf; 1597 1598 printf("\n %s Forward statistics for port %-2d %s\n", 1599 fwd_stats_border, pt_id, fwd_stats_border); 1600 1601 if (!port->rx_queue_stats_mapping_enabled && 1602 !port->tx_queue_stats_mapping_enabled) { 1603 printf(" RX-packets: %-14"PRIu64 1604 " RX-dropped: %-14"PRIu64 1605 "RX-total: %-"PRIu64"\n", 1606 stats.ipackets, stats.imissed, 1607 stats.ipackets + stats.imissed); 1608 1609 if (cur_fwd_eng == &csum_fwd_engine) 1610 printf(" Bad-ipcsum: %-14"PRIu64 1611 " Bad-l4csum: %-14"PRIu64 1612 "Bad-outer-l4csum: %-14"PRIu64"\n", 1613 ports_stats[pt_id].rx_bad_ip_csum, 1614 ports_stats[pt_id].rx_bad_l4_csum, 1615 ports_stats[pt_id].rx_bad_outer_l4_csum); 1616 if (stats.ierrors + stats.rx_nombuf > 0) { 1617 printf(" RX-error: %-"PRIu64"\n", 1618 stats.ierrors); 1619 printf(" RX-nombufs: %-14"PRIu64"\n", 1620 stats.rx_nombuf); 1621 } 1622 1623 printf(" TX-packets: %-14"PRIu64 1624 " TX-dropped: %-14"PRIu64 1625 "TX-total: %-"PRIu64"\n", 1626 stats.opackets, ports_stats[pt_id].tx_dropped, 1627 stats.opackets + ports_stats[pt_id].tx_dropped); 1628 } else { 1629 printf(" RX-packets: %14"PRIu64 1630 " RX-dropped:%14"PRIu64 1631 " RX-total:%14"PRIu64"\n", 1632 stats.ipackets, stats.imissed, 1633 stats.ipackets + stats.imissed); 1634 1635 if (cur_fwd_eng == &csum_fwd_engine) 1636 printf(" Bad-ipcsum:%14"PRIu64 1637 " Bad-l4csum:%14"PRIu64 1638 " Bad-outer-l4csum: %-14"PRIu64"\n", 1639 ports_stats[pt_id].rx_bad_ip_csum, 1640 ports_stats[pt_id].rx_bad_l4_csum, 1641 ports_stats[pt_id].rx_bad_outer_l4_csum); 1642 if ((stats.ierrors + stats.rx_nombuf) > 0) { 1643 printf(" RX-error:%"PRIu64"\n", stats.ierrors); 1644 printf(" RX-nombufs: %14"PRIu64"\n", 1645 stats.rx_nombuf); 1646 } 1647 1648 printf(" TX-packets: %14"PRIu64 1649 " TX-dropped:%14"PRIu64 1650 " TX-total:%14"PRIu64"\n", 1651 stats.opackets, ports_stats[pt_id].tx_dropped, 1652 stats.opackets + ports_stats[pt_id].tx_dropped); 1653 } 1654 1655 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1656 if (ports_stats[pt_id].rx_stream) 1657 pkt_burst_stats_display("RX", 1658 &ports_stats[pt_id].rx_stream->rx_burst_stats); 1659 if (ports_stats[pt_id].tx_stream) 1660 pkt_burst_stats_display("TX", 1661 &ports_stats[pt_id].tx_stream->tx_burst_stats); 1662 #endif 1663 1664 if (port->rx_queue_stats_mapping_enabled) { 1665 printf("\n"); 1666 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1667 printf(" Stats reg %2d RX-packets:%14"PRIu64 1668 " RX-errors:%14"PRIu64 1669 " RX-bytes:%14"PRIu64"\n", 1670 j, stats.q_ipackets[j], 1671 stats.q_errors[j], stats.q_ibytes[j]); 1672 } 1673 printf("\n"); 1674 } 1675 if (port->tx_queue_stats_mapping_enabled) { 1676 for (j = 0; j < RTE_ETHDEV_QUEUE_STAT_CNTRS; j++) { 1677 printf(" Stats reg %2d TX-packets:%14"PRIu64 1678 " TX-bytes:%14" 1679 PRIu64"\n", 1680 j, stats.q_opackets[j], 1681 stats.q_obytes[j]); 1682 } 1683 } 1684 1685 printf(" %s--------------------------------%s\n", 1686 fwd_stats_border, fwd_stats_border); 1687 } 1688 1689 printf("\n %s Accumulated forward statistics for all ports" 1690 "%s\n", 1691 acc_stats_border, acc_stats_border); 1692 printf(" RX-packets: %-14"PRIu64" RX-dropped: %-14"PRIu64"RX-total: " 1693 "%-"PRIu64"\n" 1694 " TX-packets: %-14"PRIu64" TX-dropped: %-14"PRIu64"TX-total: " 1695 "%-"PRIu64"\n", 1696 total_recv, total_rx_dropped, total_recv + total_rx_dropped, 1697 total_xmit, total_tx_dropped, total_xmit + total_tx_dropped); 1698 if (total_rx_nombuf > 0) 1699 printf(" RX-nombufs: %-14"PRIu64"\n", total_rx_nombuf); 1700 printf(" %s++++++++++++++++++++++++++++++++++++++++++++++" 1701 "%s\n", 1702 acc_stats_border, acc_stats_border); 1703 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1704 if (total_recv > 0) 1705 printf("\n CPU cycles/packet=%u (total cycles=" 1706 "%"PRIu64" / total RX packets=%"PRIu64")\n", 1707 (unsigned int)(fwd_cycles / total_recv), 1708 fwd_cycles, total_recv); 1709 #endif 1710 } 1711 1712 void 1713 fwd_stats_reset(void) 1714 { 1715 streamid_t sm_id; 1716 portid_t pt_id; 1717 int i; 1718 1719 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1720 pt_id = fwd_ports_ids[i]; 1721 rte_eth_stats_get(pt_id, &ports[pt_id].stats); 1722 } 1723 for (sm_id = 0; sm_id < cur_fwd_config.nb_fwd_streams; sm_id++) { 1724 struct fwd_stream *fs = fwd_streams[sm_id]; 1725 1726 fs->rx_packets = 0; 1727 fs->tx_packets = 0; 1728 fs->fwd_dropped = 0; 1729 fs->rx_bad_ip_csum = 0; 1730 fs->rx_bad_l4_csum = 0; 1731 fs->rx_bad_outer_l4_csum = 0; 1732 1733 #ifdef RTE_TEST_PMD_RECORD_BURST_STATS 1734 memset(&fs->rx_burst_stats, 0, sizeof(fs->rx_burst_stats)); 1735 memset(&fs->tx_burst_stats, 0, sizeof(fs->tx_burst_stats)); 1736 #endif 1737 #ifdef RTE_TEST_PMD_RECORD_CORE_CYCLES 1738 fs->core_cycles = 0; 1739 #endif 1740 } 1741 } 1742 1743 static void 1744 flush_fwd_rx_queues(void) 1745 { 1746 struct rte_mbuf *pkts_burst[MAX_PKT_BURST]; 1747 portid_t rxp; 1748 portid_t port_id; 1749 queueid_t rxq; 1750 uint16_t nb_rx; 1751 uint16_t i; 1752 uint8_t j; 1753 uint64_t prev_tsc = 0, diff_tsc, cur_tsc, timer_tsc = 0; 1754 uint64_t timer_period; 1755 1756 /* convert to number of cycles */ 1757 timer_period = rte_get_timer_hz(); /* 1 second timeout */ 1758 1759 for (j = 0; j < 2; j++) { 1760 for (rxp = 0; rxp < cur_fwd_config.nb_fwd_ports; rxp++) { 1761 for (rxq = 0; rxq < nb_rxq; rxq++) { 1762 port_id = fwd_ports_ids[rxp]; 1763 /** 1764 * testpmd can stuck in the below do while loop 1765 * if rte_eth_rx_burst() always returns nonzero 1766 * packets. So timer is added to exit this loop 1767 * after 1sec timer expiry. 1768 */ 1769 prev_tsc = rte_rdtsc(); 1770 do { 1771 nb_rx = rte_eth_rx_burst(port_id, rxq, 1772 pkts_burst, MAX_PKT_BURST); 1773 for (i = 0; i < nb_rx; i++) 1774 rte_pktmbuf_free(pkts_burst[i]); 1775 1776 cur_tsc = rte_rdtsc(); 1777 diff_tsc = cur_tsc - prev_tsc; 1778 timer_tsc += diff_tsc; 1779 } while ((nb_rx > 0) && 1780 (timer_tsc < timer_period)); 1781 timer_tsc = 0; 1782 } 1783 } 1784 rte_delay_ms(10); /* wait 10 milli-seconds before retrying */ 1785 } 1786 } 1787 1788 static void 1789 run_pkt_fwd_on_lcore(struct fwd_lcore *fc, packet_fwd_t pkt_fwd) 1790 { 1791 struct fwd_stream **fsm; 1792 streamid_t nb_fs; 1793 streamid_t sm_id; 1794 #ifdef RTE_LIBRTE_BITRATE 1795 uint64_t tics_per_1sec; 1796 uint64_t tics_datum; 1797 uint64_t tics_current; 1798 uint16_t i, cnt_ports; 1799 1800 cnt_ports = nb_ports; 1801 tics_datum = rte_rdtsc(); 1802 tics_per_1sec = rte_get_timer_hz(); 1803 #endif 1804 fsm = &fwd_streams[fc->stream_idx]; 1805 nb_fs = fc->stream_nb; 1806 do { 1807 for (sm_id = 0; sm_id < nb_fs; sm_id++) 1808 (*pkt_fwd)(fsm[sm_id]); 1809 #ifdef RTE_LIBRTE_BITRATE 1810 if (bitrate_enabled != 0 && 1811 bitrate_lcore_id == rte_lcore_id()) { 1812 tics_current = rte_rdtsc(); 1813 if (tics_current - tics_datum >= tics_per_1sec) { 1814 /* Periodic bitrate calculation */ 1815 for (i = 0; i < cnt_ports; i++) 1816 rte_stats_bitrate_calc(bitrate_data, 1817 ports_ids[i]); 1818 tics_datum = tics_current; 1819 } 1820 } 1821 #endif 1822 #ifdef RTE_LIBRTE_LATENCY_STATS 1823 if (latencystats_enabled != 0 && 1824 latencystats_lcore_id == rte_lcore_id()) 1825 rte_latencystats_update(); 1826 #endif 1827 1828 } while (! fc->stopped); 1829 } 1830 1831 static int 1832 start_pkt_forward_on_core(void *fwd_arg) 1833 { 1834 run_pkt_fwd_on_lcore((struct fwd_lcore *) fwd_arg, 1835 cur_fwd_config.fwd_eng->packet_fwd); 1836 return 0; 1837 } 1838 1839 /* 1840 * Run the TXONLY packet forwarding engine to send a single burst of packets. 1841 * Used to start communication flows in network loopback test configurations. 1842 */ 1843 static int 1844 run_one_txonly_burst_on_core(void *fwd_arg) 1845 { 1846 struct fwd_lcore *fwd_lc; 1847 struct fwd_lcore tmp_lcore; 1848 1849 fwd_lc = (struct fwd_lcore *) fwd_arg; 1850 tmp_lcore = *fwd_lc; 1851 tmp_lcore.stopped = 1; 1852 run_pkt_fwd_on_lcore(&tmp_lcore, tx_only_engine.packet_fwd); 1853 return 0; 1854 } 1855 1856 /* 1857 * Launch packet forwarding: 1858 * - Setup per-port forwarding context. 1859 * - launch logical cores with their forwarding configuration. 1860 */ 1861 static void 1862 launch_packet_forwarding(lcore_function_t *pkt_fwd_on_lcore) 1863 { 1864 port_fwd_begin_t port_fwd_begin; 1865 unsigned int i; 1866 unsigned int lc_id; 1867 int diag; 1868 1869 port_fwd_begin = cur_fwd_config.fwd_eng->port_fwd_begin; 1870 if (port_fwd_begin != NULL) { 1871 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1872 (*port_fwd_begin)(fwd_ports_ids[i]); 1873 } 1874 for (i = 0; i < cur_fwd_config.nb_fwd_lcores; i++) { 1875 lc_id = fwd_lcores_cpuids[i]; 1876 if ((interactive == 0) || (lc_id != rte_lcore_id())) { 1877 fwd_lcores[i]->stopped = 0; 1878 diag = rte_eal_remote_launch(pkt_fwd_on_lcore, 1879 fwd_lcores[i], lc_id); 1880 if (diag != 0) 1881 printf("launch lcore %u failed - diag=%d\n", 1882 lc_id, diag); 1883 } 1884 } 1885 } 1886 1887 /* 1888 * Launch packet forwarding configuration. 1889 */ 1890 void 1891 start_packet_forwarding(int with_tx_first) 1892 { 1893 port_fwd_begin_t port_fwd_begin; 1894 port_fwd_end_t port_fwd_end; 1895 struct rte_port *port; 1896 unsigned int i; 1897 portid_t pt_id; 1898 1899 if (strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") == 0 && !nb_rxq) 1900 rte_exit(EXIT_FAILURE, "rxq are 0, cannot use rxonly fwd mode\n"); 1901 1902 if (strcmp(cur_fwd_eng->fwd_mode_name, "txonly") == 0 && !nb_txq) 1903 rte_exit(EXIT_FAILURE, "txq are 0, cannot use txonly fwd mode\n"); 1904 1905 if ((strcmp(cur_fwd_eng->fwd_mode_name, "rxonly") != 0 && 1906 strcmp(cur_fwd_eng->fwd_mode_name, "txonly") != 0) && 1907 (!nb_rxq || !nb_txq)) 1908 rte_exit(EXIT_FAILURE, 1909 "Either rxq or txq are 0, cannot use %s fwd mode\n", 1910 cur_fwd_eng->fwd_mode_name); 1911 1912 if (all_ports_started() == 0) { 1913 printf("Not all ports were started\n"); 1914 return; 1915 } 1916 if (test_done == 0) { 1917 printf("Packet forwarding already started\n"); 1918 return; 1919 } 1920 1921 1922 if(dcb_test) { 1923 for (i = 0; i < nb_fwd_ports; i++) { 1924 pt_id = fwd_ports_ids[i]; 1925 port = &ports[pt_id]; 1926 if (!port->dcb_flag) { 1927 printf("In DCB mode, all forwarding ports must " 1928 "be configured in this mode.\n"); 1929 return; 1930 } 1931 } 1932 if (nb_fwd_lcores == 1) { 1933 printf("In DCB mode,the nb forwarding cores " 1934 "should be larger than 1.\n"); 1935 return; 1936 } 1937 } 1938 test_done = 0; 1939 1940 fwd_config_setup(); 1941 1942 if(!no_flush_rx) 1943 flush_fwd_rx_queues(); 1944 1945 pkt_fwd_config_display(&cur_fwd_config); 1946 rxtx_config_display(); 1947 1948 fwd_stats_reset(); 1949 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1950 pt_id = fwd_ports_ids[i]; 1951 port = &ports[pt_id]; 1952 map_port_queue_stats_mapping_registers(pt_id, port); 1953 } 1954 if (with_tx_first) { 1955 port_fwd_begin = tx_only_engine.port_fwd_begin; 1956 if (port_fwd_begin != NULL) { 1957 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1958 (*port_fwd_begin)(fwd_ports_ids[i]); 1959 } 1960 while (with_tx_first--) { 1961 launch_packet_forwarding( 1962 run_one_txonly_burst_on_core); 1963 rte_eal_mp_wait_lcore(); 1964 } 1965 port_fwd_end = tx_only_engine.port_fwd_end; 1966 if (port_fwd_end != NULL) { 1967 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 1968 (*port_fwd_end)(fwd_ports_ids[i]); 1969 } 1970 } 1971 launch_packet_forwarding(start_pkt_forward_on_core); 1972 } 1973 1974 void 1975 stop_packet_forwarding(void) 1976 { 1977 port_fwd_end_t port_fwd_end; 1978 lcoreid_t lc_id; 1979 portid_t pt_id; 1980 int i; 1981 1982 if (test_done) { 1983 printf("Packet forwarding not started\n"); 1984 return; 1985 } 1986 printf("Telling cores to stop..."); 1987 for (lc_id = 0; lc_id < cur_fwd_config.nb_fwd_lcores; lc_id++) 1988 fwd_lcores[lc_id]->stopped = 1; 1989 printf("\nWaiting for lcores to finish...\n"); 1990 rte_eal_mp_wait_lcore(); 1991 port_fwd_end = cur_fwd_config.fwd_eng->port_fwd_end; 1992 if (port_fwd_end != NULL) { 1993 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) { 1994 pt_id = fwd_ports_ids[i]; 1995 (*port_fwd_end)(pt_id); 1996 } 1997 } 1998 1999 fwd_stats_display(); 2000 2001 printf("\nDone.\n"); 2002 test_done = 1; 2003 } 2004 2005 void 2006 dev_set_link_up(portid_t pid) 2007 { 2008 if (rte_eth_dev_set_link_up(pid) < 0) 2009 printf("\nSet link up fail.\n"); 2010 } 2011 2012 void 2013 dev_set_link_down(portid_t pid) 2014 { 2015 if (rte_eth_dev_set_link_down(pid) < 0) 2016 printf("\nSet link down fail.\n"); 2017 } 2018 2019 static int 2020 all_ports_started(void) 2021 { 2022 portid_t pi; 2023 struct rte_port *port; 2024 2025 RTE_ETH_FOREACH_DEV(pi) { 2026 port = &ports[pi]; 2027 /* Check if there is a port which is not started */ 2028 if ((port->port_status != RTE_PORT_STARTED) && 2029 (port->slave_flag == 0)) 2030 return 0; 2031 } 2032 2033 /* No port is not started */ 2034 return 1; 2035 } 2036 2037 int 2038 port_is_stopped(portid_t port_id) 2039 { 2040 struct rte_port *port = &ports[port_id]; 2041 2042 if ((port->port_status != RTE_PORT_STOPPED) && 2043 (port->slave_flag == 0)) 2044 return 0; 2045 return 1; 2046 } 2047 2048 int 2049 all_ports_stopped(void) 2050 { 2051 portid_t pi; 2052 2053 RTE_ETH_FOREACH_DEV(pi) { 2054 if (!port_is_stopped(pi)) 2055 return 0; 2056 } 2057 2058 return 1; 2059 } 2060 2061 int 2062 port_is_started(portid_t port_id) 2063 { 2064 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2065 return 0; 2066 2067 if (ports[port_id].port_status != RTE_PORT_STARTED) 2068 return 0; 2069 2070 return 1; 2071 } 2072 2073 /* Configure the Rx and Tx hairpin queues for the selected port. */ 2074 static int 2075 setup_hairpin_queues(portid_t pi) 2076 { 2077 queueid_t qi; 2078 struct rte_eth_hairpin_conf hairpin_conf = { 2079 .peer_count = 1, 2080 }; 2081 int i; 2082 int diag; 2083 struct rte_port *port = &ports[pi]; 2084 2085 for (qi = nb_txq, i = 0; qi < nb_hairpinq + nb_txq; qi++) { 2086 hairpin_conf.peers[0].port = pi; 2087 hairpin_conf.peers[0].queue = i + nb_rxq; 2088 diag = rte_eth_tx_hairpin_queue_setup 2089 (pi, qi, nb_txd, &hairpin_conf); 2090 i++; 2091 if (diag == 0) 2092 continue; 2093 2094 /* Fail to setup rx queue, return */ 2095 if (rte_atomic16_cmpset(&(port->port_status), 2096 RTE_PORT_HANDLING, 2097 RTE_PORT_STOPPED) == 0) 2098 printf("Port %d can not be set back " 2099 "to stopped\n", pi); 2100 printf("Fail to configure port %d hairpin " 2101 "queues\n", pi); 2102 /* try to reconfigure queues next time */ 2103 port->need_reconfig_queues = 1; 2104 return -1; 2105 } 2106 for (qi = nb_rxq, i = 0; qi < nb_hairpinq + nb_rxq; qi++) { 2107 hairpin_conf.peers[0].port = pi; 2108 hairpin_conf.peers[0].queue = i + nb_txq; 2109 diag = rte_eth_rx_hairpin_queue_setup 2110 (pi, qi, nb_rxd, &hairpin_conf); 2111 i++; 2112 if (diag == 0) 2113 continue; 2114 2115 /* Fail to setup rx queue, return */ 2116 if (rte_atomic16_cmpset(&(port->port_status), 2117 RTE_PORT_HANDLING, 2118 RTE_PORT_STOPPED) == 0) 2119 printf("Port %d can not be set back " 2120 "to stopped\n", pi); 2121 printf("Fail to configure port %d hairpin " 2122 "queues\n", pi); 2123 /* try to reconfigure queues next time */ 2124 port->need_reconfig_queues = 1; 2125 return -1; 2126 } 2127 return 0; 2128 } 2129 2130 int 2131 start_port(portid_t pid) 2132 { 2133 int diag, need_check_link_status = -1; 2134 portid_t pi; 2135 queueid_t qi; 2136 struct rte_port *port; 2137 struct rte_ether_addr mac_addr; 2138 struct rte_eth_hairpin_cap cap; 2139 2140 if (port_id_is_invalid(pid, ENABLED_WARN)) 2141 return 0; 2142 2143 if(dcb_config) 2144 dcb_test = 1; 2145 RTE_ETH_FOREACH_DEV(pi) { 2146 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2147 continue; 2148 2149 need_check_link_status = 0; 2150 port = &ports[pi]; 2151 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STOPPED, 2152 RTE_PORT_HANDLING) == 0) { 2153 printf("Port %d is now not stopped\n", pi); 2154 continue; 2155 } 2156 2157 if (port->need_reconfig > 0) { 2158 port->need_reconfig = 0; 2159 2160 if (flow_isolate_all) { 2161 int ret = port_flow_isolate(pi, 1); 2162 if (ret) { 2163 printf("Failed to apply isolated" 2164 " mode on port %d\n", pi); 2165 return -1; 2166 } 2167 } 2168 configure_rxtx_dump_callbacks(0); 2169 printf("Configuring Port %d (socket %u)\n", pi, 2170 port->socket_id); 2171 if (nb_hairpinq > 0 && 2172 rte_eth_dev_hairpin_capability_get(pi, &cap)) { 2173 printf("Port %d doesn't support hairpin " 2174 "queues\n", pi); 2175 return -1; 2176 } 2177 /* configure port */ 2178 diag = rte_eth_dev_configure(pi, nb_rxq + nb_hairpinq, 2179 nb_txq + nb_hairpinq, 2180 &(port->dev_conf)); 2181 if (diag != 0) { 2182 if (rte_atomic16_cmpset(&(port->port_status), 2183 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2184 printf("Port %d can not be set back " 2185 "to stopped\n", pi); 2186 printf("Fail to configure port %d\n", pi); 2187 /* try to reconfigure port next time */ 2188 port->need_reconfig = 1; 2189 return -1; 2190 } 2191 } 2192 if (port->need_reconfig_queues > 0) { 2193 port->need_reconfig_queues = 0; 2194 /* setup tx queues */ 2195 for (qi = 0; qi < nb_txq; qi++) { 2196 if ((numa_support) && 2197 (txring_numa[pi] != NUMA_NO_CONFIG)) 2198 diag = rte_eth_tx_queue_setup(pi, qi, 2199 port->nb_tx_desc[qi], 2200 txring_numa[pi], 2201 &(port->tx_conf[qi])); 2202 else 2203 diag = rte_eth_tx_queue_setup(pi, qi, 2204 port->nb_tx_desc[qi], 2205 port->socket_id, 2206 &(port->tx_conf[qi])); 2207 2208 if (diag == 0) 2209 continue; 2210 2211 /* Fail to setup tx queue, return */ 2212 if (rte_atomic16_cmpset(&(port->port_status), 2213 RTE_PORT_HANDLING, 2214 RTE_PORT_STOPPED) == 0) 2215 printf("Port %d can not be set back " 2216 "to stopped\n", pi); 2217 printf("Fail to configure port %d tx queues\n", 2218 pi); 2219 /* try to reconfigure queues next time */ 2220 port->need_reconfig_queues = 1; 2221 return -1; 2222 } 2223 for (qi = 0; qi < nb_rxq; qi++) { 2224 /* setup rx queues */ 2225 if ((numa_support) && 2226 (rxring_numa[pi] != NUMA_NO_CONFIG)) { 2227 struct rte_mempool * mp = 2228 mbuf_pool_find(rxring_numa[pi]); 2229 if (mp == NULL) { 2230 printf("Failed to setup RX queue:" 2231 "No mempool allocation" 2232 " on the socket %d\n", 2233 rxring_numa[pi]); 2234 return -1; 2235 } 2236 2237 diag = rte_eth_rx_queue_setup(pi, qi, 2238 port->nb_rx_desc[qi], 2239 rxring_numa[pi], 2240 &(port->rx_conf[qi]), 2241 mp); 2242 } else { 2243 struct rte_mempool *mp = 2244 mbuf_pool_find(port->socket_id); 2245 if (mp == NULL) { 2246 printf("Failed to setup RX queue:" 2247 "No mempool allocation" 2248 " on the socket %d\n", 2249 port->socket_id); 2250 return -1; 2251 } 2252 diag = rte_eth_rx_queue_setup(pi, qi, 2253 port->nb_rx_desc[qi], 2254 port->socket_id, 2255 &(port->rx_conf[qi]), 2256 mp); 2257 } 2258 if (diag == 0) 2259 continue; 2260 2261 /* Fail to setup rx queue, return */ 2262 if (rte_atomic16_cmpset(&(port->port_status), 2263 RTE_PORT_HANDLING, 2264 RTE_PORT_STOPPED) == 0) 2265 printf("Port %d can not be set back " 2266 "to stopped\n", pi); 2267 printf("Fail to configure port %d rx queues\n", 2268 pi); 2269 /* try to reconfigure queues next time */ 2270 port->need_reconfig_queues = 1; 2271 return -1; 2272 } 2273 /* setup hairpin queues */ 2274 if (setup_hairpin_queues(pi) != 0) 2275 return -1; 2276 } 2277 configure_rxtx_dump_callbacks(verbose_level); 2278 if (clear_ptypes) { 2279 diag = rte_eth_dev_set_ptypes(pi, RTE_PTYPE_UNKNOWN, 2280 NULL, 0); 2281 if (diag < 0) 2282 printf( 2283 "Port %d: Failed to disable Ptype parsing\n", 2284 pi); 2285 } 2286 2287 /* start port */ 2288 if (rte_eth_dev_start(pi) < 0) { 2289 printf("Fail to start port %d\n", pi); 2290 2291 /* Fail to setup rx queue, return */ 2292 if (rte_atomic16_cmpset(&(port->port_status), 2293 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2294 printf("Port %d can not be set back to " 2295 "stopped\n", pi); 2296 continue; 2297 } 2298 2299 if (rte_atomic16_cmpset(&(port->port_status), 2300 RTE_PORT_HANDLING, RTE_PORT_STARTED) == 0) 2301 printf("Port %d can not be set into started\n", pi); 2302 2303 if (eth_macaddr_get_print_err(pi, &mac_addr) == 0) 2304 printf("Port %d: %02X:%02X:%02X:%02X:%02X:%02X\n", pi, 2305 mac_addr.addr_bytes[0], mac_addr.addr_bytes[1], 2306 mac_addr.addr_bytes[2], mac_addr.addr_bytes[3], 2307 mac_addr.addr_bytes[4], mac_addr.addr_bytes[5]); 2308 2309 /* at least one port started, need checking link status */ 2310 need_check_link_status = 1; 2311 } 2312 2313 if (need_check_link_status == 1 && !no_link_check) 2314 check_all_ports_link_status(RTE_PORT_ALL); 2315 else if (need_check_link_status == 0) 2316 printf("Please stop the ports first\n"); 2317 2318 printf("Done\n"); 2319 return 0; 2320 } 2321 2322 void 2323 stop_port(portid_t pid) 2324 { 2325 portid_t pi; 2326 struct rte_port *port; 2327 int need_check_link_status = 0; 2328 2329 if (dcb_test) { 2330 dcb_test = 0; 2331 dcb_config = 0; 2332 } 2333 2334 if (port_id_is_invalid(pid, ENABLED_WARN)) 2335 return; 2336 2337 printf("Stopping ports...\n"); 2338 2339 RTE_ETH_FOREACH_DEV(pi) { 2340 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2341 continue; 2342 2343 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2344 printf("Please remove port %d from forwarding configuration.\n", pi); 2345 continue; 2346 } 2347 2348 if (port_is_bonding_slave(pi)) { 2349 printf("Please remove port %d from bonded device.\n", pi); 2350 continue; 2351 } 2352 2353 port = &ports[pi]; 2354 if (rte_atomic16_cmpset(&(port->port_status), RTE_PORT_STARTED, 2355 RTE_PORT_HANDLING) == 0) 2356 continue; 2357 2358 rte_eth_dev_stop(pi); 2359 2360 if (rte_atomic16_cmpset(&(port->port_status), 2361 RTE_PORT_HANDLING, RTE_PORT_STOPPED) == 0) 2362 printf("Port %d can not be set into stopped\n", pi); 2363 need_check_link_status = 1; 2364 } 2365 if (need_check_link_status && !no_link_check) 2366 check_all_ports_link_status(RTE_PORT_ALL); 2367 2368 printf("Done\n"); 2369 } 2370 2371 static void 2372 remove_invalid_ports_in(portid_t *array, portid_t *total) 2373 { 2374 portid_t i; 2375 portid_t new_total = 0; 2376 2377 for (i = 0; i < *total; i++) 2378 if (!port_id_is_invalid(array[i], DISABLED_WARN)) { 2379 array[new_total] = array[i]; 2380 new_total++; 2381 } 2382 *total = new_total; 2383 } 2384 2385 static void 2386 remove_invalid_ports(void) 2387 { 2388 remove_invalid_ports_in(ports_ids, &nb_ports); 2389 remove_invalid_ports_in(fwd_ports_ids, &nb_fwd_ports); 2390 nb_cfg_ports = nb_fwd_ports; 2391 } 2392 2393 void 2394 close_port(portid_t pid) 2395 { 2396 portid_t pi; 2397 struct rte_port *port; 2398 2399 if (port_id_is_invalid(pid, ENABLED_WARN)) 2400 return; 2401 2402 printf("Closing ports...\n"); 2403 2404 RTE_ETH_FOREACH_DEV(pi) { 2405 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2406 continue; 2407 2408 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2409 printf("Please remove port %d from forwarding configuration.\n", pi); 2410 continue; 2411 } 2412 2413 if (port_is_bonding_slave(pi)) { 2414 printf("Please remove port %d from bonded device.\n", pi); 2415 continue; 2416 } 2417 2418 port = &ports[pi]; 2419 if (rte_atomic16_cmpset(&(port->port_status), 2420 RTE_PORT_CLOSED, RTE_PORT_CLOSED) == 1) { 2421 printf("Port %d is already closed\n", pi); 2422 continue; 2423 } 2424 2425 if (rte_atomic16_cmpset(&(port->port_status), 2426 RTE_PORT_STOPPED, RTE_PORT_HANDLING) == 0) { 2427 printf("Port %d is now not stopped\n", pi); 2428 continue; 2429 } 2430 2431 if (port->flow_list) 2432 port_flow_flush(pi); 2433 rte_eth_dev_close(pi); 2434 2435 remove_invalid_ports(); 2436 2437 if (rte_atomic16_cmpset(&(port->port_status), 2438 RTE_PORT_HANDLING, RTE_PORT_CLOSED) == 0) 2439 printf("Port %d cannot be set to closed\n", pi); 2440 } 2441 2442 printf("Done\n"); 2443 } 2444 2445 void 2446 reset_port(portid_t pid) 2447 { 2448 int diag; 2449 portid_t pi; 2450 struct rte_port *port; 2451 2452 if (port_id_is_invalid(pid, ENABLED_WARN)) 2453 return; 2454 2455 if ((pid == (portid_t)RTE_PORT_ALL && !all_ports_stopped()) || 2456 (pid != (portid_t)RTE_PORT_ALL && !port_is_stopped(pid))) { 2457 printf("Can not reset port(s), please stop port(s) first.\n"); 2458 return; 2459 } 2460 2461 printf("Resetting ports...\n"); 2462 2463 RTE_ETH_FOREACH_DEV(pi) { 2464 if (pid != pi && pid != (portid_t)RTE_PORT_ALL) 2465 continue; 2466 2467 if (port_is_forwarding(pi) != 0 && test_done == 0) { 2468 printf("Please remove port %d from forwarding " 2469 "configuration.\n", pi); 2470 continue; 2471 } 2472 2473 if (port_is_bonding_slave(pi)) { 2474 printf("Please remove port %d from bonded device.\n", 2475 pi); 2476 continue; 2477 } 2478 2479 diag = rte_eth_dev_reset(pi); 2480 if (diag == 0) { 2481 port = &ports[pi]; 2482 port->need_reconfig = 1; 2483 port->need_reconfig_queues = 1; 2484 } else { 2485 printf("Failed to reset port %d. diag=%d\n", pi, diag); 2486 } 2487 } 2488 2489 printf("Done\n"); 2490 } 2491 2492 void 2493 attach_port(char *identifier) 2494 { 2495 portid_t pi; 2496 struct rte_dev_iterator iterator; 2497 2498 printf("Attaching a new port...\n"); 2499 2500 if (identifier == NULL) { 2501 printf("Invalid parameters are specified\n"); 2502 return; 2503 } 2504 2505 if (rte_dev_probe(identifier) < 0) { 2506 TESTPMD_LOG(ERR, "Failed to attach port %s\n", identifier); 2507 return; 2508 } 2509 2510 /* first attach mode: event */ 2511 if (setup_on_probe_event) { 2512 /* new ports are detected on RTE_ETH_EVENT_NEW event */ 2513 for (pi = 0; pi < RTE_MAX_ETHPORTS; pi++) 2514 if (ports[pi].port_status == RTE_PORT_HANDLING && 2515 ports[pi].need_setup != 0) 2516 setup_attached_port(pi); 2517 return; 2518 } 2519 2520 /* second attach mode: iterator */ 2521 RTE_ETH_FOREACH_MATCHING_DEV(pi, identifier, &iterator) { 2522 /* setup ports matching the devargs used for probing */ 2523 if (port_is_forwarding(pi)) 2524 continue; /* port was already attached before */ 2525 setup_attached_port(pi); 2526 } 2527 } 2528 2529 static void 2530 setup_attached_port(portid_t pi) 2531 { 2532 unsigned int socket_id; 2533 int ret; 2534 2535 socket_id = (unsigned)rte_eth_dev_socket_id(pi); 2536 /* if socket_id is invalid, set to the first available socket. */ 2537 if (check_socket_id(socket_id) < 0) 2538 socket_id = socket_ids[0]; 2539 reconfig(pi, socket_id); 2540 ret = rte_eth_promiscuous_enable(pi); 2541 if (ret != 0) 2542 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 2543 pi, rte_strerror(-ret)); 2544 2545 ports_ids[nb_ports++] = pi; 2546 fwd_ports_ids[nb_fwd_ports++] = pi; 2547 nb_cfg_ports = nb_fwd_ports; 2548 ports[pi].need_setup = 0; 2549 ports[pi].port_status = RTE_PORT_STOPPED; 2550 2551 printf("Port %d is attached. Now total ports is %d\n", pi, nb_ports); 2552 printf("Done\n"); 2553 } 2554 2555 void 2556 detach_port_device(portid_t port_id) 2557 { 2558 struct rte_device *dev; 2559 portid_t sibling; 2560 2561 printf("Removing a device...\n"); 2562 2563 if (port_id_is_invalid(port_id, ENABLED_WARN)) 2564 return; 2565 2566 dev = rte_eth_devices[port_id].device; 2567 if (dev == NULL) { 2568 printf("Device already removed\n"); 2569 return; 2570 } 2571 2572 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2573 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2574 printf("Port not stopped\n"); 2575 return; 2576 } 2577 printf("Port was not closed\n"); 2578 if (ports[port_id].flow_list) 2579 port_flow_flush(port_id); 2580 } 2581 2582 if (rte_dev_remove(dev) < 0) { 2583 TESTPMD_LOG(ERR, "Failed to detach device %s\n", dev->name); 2584 return; 2585 } 2586 RTE_ETH_FOREACH_DEV_OF(sibling, dev) { 2587 /* reset mapping between old ports and removed device */ 2588 rte_eth_devices[sibling].device = NULL; 2589 if (ports[sibling].port_status != RTE_PORT_CLOSED) { 2590 /* sibling ports are forced to be closed */ 2591 ports[sibling].port_status = RTE_PORT_CLOSED; 2592 printf("Port %u is closed\n", sibling); 2593 } 2594 } 2595 2596 remove_invalid_ports(); 2597 2598 printf("Device of port %u is detached\n", port_id); 2599 printf("Now total ports is %d\n", nb_ports); 2600 printf("Done\n"); 2601 return; 2602 } 2603 2604 void 2605 detach_device(char *identifier) 2606 { 2607 struct rte_dev_iterator iterator; 2608 struct rte_devargs da; 2609 portid_t port_id; 2610 2611 printf("Removing a device...\n"); 2612 2613 memset(&da, 0, sizeof(da)); 2614 if (rte_devargs_parsef(&da, "%s", identifier)) { 2615 printf("cannot parse identifier\n"); 2616 if (da.args) 2617 free(da.args); 2618 return; 2619 } 2620 2621 RTE_ETH_FOREACH_MATCHING_DEV(port_id, identifier, &iterator) { 2622 if (ports[port_id].port_status != RTE_PORT_CLOSED) { 2623 if (ports[port_id].port_status != RTE_PORT_STOPPED) { 2624 printf("Port %u not stopped\n", port_id); 2625 rte_eth_iterator_cleanup(&iterator); 2626 return; 2627 } 2628 2629 /* sibling ports are forced to be closed */ 2630 if (ports[port_id].flow_list) 2631 port_flow_flush(port_id); 2632 ports[port_id].port_status = RTE_PORT_CLOSED; 2633 printf("Port %u is now closed\n", port_id); 2634 } 2635 } 2636 2637 if (rte_eal_hotplug_remove(da.bus->name, da.name) != 0) { 2638 TESTPMD_LOG(ERR, "Failed to detach device %s(%s)\n", 2639 da.name, da.bus->name); 2640 return; 2641 } 2642 2643 remove_invalid_ports(); 2644 2645 printf("Device %s is detached\n", identifier); 2646 printf("Now total ports is %d\n", nb_ports); 2647 printf("Done\n"); 2648 } 2649 2650 void 2651 pmd_test_exit(void) 2652 { 2653 portid_t pt_id; 2654 int ret; 2655 int i; 2656 2657 if (test_done == 0) 2658 stop_packet_forwarding(); 2659 2660 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2661 if (mempools[i]) { 2662 if (mp_alloc_type == MP_ALLOC_ANON) 2663 rte_mempool_mem_iter(mempools[i], dma_unmap_cb, 2664 NULL); 2665 } 2666 } 2667 if (ports != NULL) { 2668 no_link_check = 1; 2669 RTE_ETH_FOREACH_DEV(pt_id) { 2670 printf("\nStopping port %d...\n", pt_id); 2671 fflush(stdout); 2672 stop_port(pt_id); 2673 } 2674 RTE_ETH_FOREACH_DEV(pt_id) { 2675 printf("\nShutting down port %d...\n", pt_id); 2676 fflush(stdout); 2677 close_port(pt_id); 2678 } 2679 } 2680 2681 if (hot_plug) { 2682 ret = rte_dev_event_monitor_stop(); 2683 if (ret) { 2684 RTE_LOG(ERR, EAL, 2685 "fail to stop device event monitor."); 2686 return; 2687 } 2688 2689 ret = rte_dev_event_callback_unregister(NULL, 2690 dev_event_callback, NULL); 2691 if (ret < 0) { 2692 RTE_LOG(ERR, EAL, 2693 "fail to unregister device event callback.\n"); 2694 return; 2695 } 2696 2697 ret = rte_dev_hotplug_handle_disable(); 2698 if (ret) { 2699 RTE_LOG(ERR, EAL, 2700 "fail to disable hotplug handling.\n"); 2701 return; 2702 } 2703 } 2704 for (i = 0 ; i < RTE_MAX_NUMA_NODES ; i++) { 2705 if (mempools[i]) 2706 rte_mempool_free(mempools[i]); 2707 } 2708 2709 printf("\nBye...\n"); 2710 } 2711 2712 typedef void (*cmd_func_t)(void); 2713 struct pmd_test_command { 2714 const char *cmd_name; 2715 cmd_func_t cmd_func; 2716 }; 2717 2718 #define PMD_TEST_CMD_NB (sizeof(pmd_test_menu) / sizeof(pmd_test_menu[0])) 2719 2720 /* Check the link status of all ports in up to 9s, and print them finally */ 2721 static void 2722 check_all_ports_link_status(uint32_t port_mask) 2723 { 2724 #define CHECK_INTERVAL 100 /* 100ms */ 2725 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */ 2726 portid_t portid; 2727 uint8_t count, all_ports_up, print_flag = 0; 2728 struct rte_eth_link link; 2729 int ret; 2730 2731 printf("Checking link statuses...\n"); 2732 fflush(stdout); 2733 for (count = 0; count <= MAX_CHECK_TIME; count++) { 2734 all_ports_up = 1; 2735 RTE_ETH_FOREACH_DEV(portid) { 2736 if ((port_mask & (1 << portid)) == 0) 2737 continue; 2738 memset(&link, 0, sizeof(link)); 2739 ret = rte_eth_link_get_nowait(portid, &link); 2740 if (ret < 0) { 2741 all_ports_up = 0; 2742 if (print_flag == 1) 2743 printf("Port %u link get failed: %s\n", 2744 portid, rte_strerror(-ret)); 2745 continue; 2746 } 2747 /* print link status if flag set */ 2748 if (print_flag == 1) { 2749 if (link.link_status) 2750 printf( 2751 "Port%d Link Up. speed %u Mbps- %s\n", 2752 portid, link.link_speed, 2753 (link.link_duplex == ETH_LINK_FULL_DUPLEX) ? 2754 ("full-duplex") : ("half-duplex\n")); 2755 else 2756 printf("Port %d Link Down\n", portid); 2757 continue; 2758 } 2759 /* clear all_ports_up flag if any link down */ 2760 if (link.link_status == ETH_LINK_DOWN) { 2761 all_ports_up = 0; 2762 break; 2763 } 2764 } 2765 /* after finally printing all link status, get out */ 2766 if (print_flag == 1) 2767 break; 2768 2769 if (all_ports_up == 0) { 2770 fflush(stdout); 2771 rte_delay_ms(CHECK_INTERVAL); 2772 } 2773 2774 /* set the print_flag if all ports up or timeout */ 2775 if (all_ports_up == 1 || count == (MAX_CHECK_TIME - 1)) { 2776 print_flag = 1; 2777 } 2778 2779 if (lsc_interrupt) 2780 break; 2781 } 2782 } 2783 2784 /* 2785 * This callback is for remove a port for a device. It has limitation because 2786 * it is not for multiple port removal for a device. 2787 * TODO: the device detach invoke will plan to be removed from user side to 2788 * eal. And convert all PMDs to free port resources on ether device closing. 2789 */ 2790 static void 2791 rmv_port_callback(void *arg) 2792 { 2793 int need_to_start = 0; 2794 int org_no_link_check = no_link_check; 2795 portid_t port_id = (intptr_t)arg; 2796 2797 RTE_ETH_VALID_PORTID_OR_RET(port_id); 2798 2799 if (!test_done && port_is_forwarding(port_id)) { 2800 need_to_start = 1; 2801 stop_packet_forwarding(); 2802 } 2803 no_link_check = 1; 2804 stop_port(port_id); 2805 no_link_check = org_no_link_check; 2806 close_port(port_id); 2807 detach_port_device(port_id); 2808 if (need_to_start) 2809 start_packet_forwarding(0); 2810 } 2811 2812 /* This function is used by the interrupt thread */ 2813 static int 2814 eth_event_callback(portid_t port_id, enum rte_eth_event_type type, void *param, 2815 void *ret_param) 2816 { 2817 RTE_SET_USED(param); 2818 RTE_SET_USED(ret_param); 2819 2820 if (type >= RTE_ETH_EVENT_MAX) { 2821 fprintf(stderr, "\nPort %" PRIu16 ": %s called upon invalid event %d\n", 2822 port_id, __func__, type); 2823 fflush(stderr); 2824 } else if (event_print_mask & (UINT32_C(1) << type)) { 2825 printf("\nPort %" PRIu16 ": %s event\n", port_id, 2826 eth_event_desc[type]); 2827 fflush(stdout); 2828 } 2829 2830 switch (type) { 2831 case RTE_ETH_EVENT_NEW: 2832 ports[port_id].need_setup = 1; 2833 ports[port_id].port_status = RTE_PORT_HANDLING; 2834 break; 2835 case RTE_ETH_EVENT_INTR_RMV: 2836 if (port_id_is_invalid(port_id, DISABLED_WARN)) 2837 break; 2838 if (rte_eal_alarm_set(100000, 2839 rmv_port_callback, (void *)(intptr_t)port_id)) 2840 fprintf(stderr, "Could not set up deferred device removal\n"); 2841 break; 2842 default: 2843 break; 2844 } 2845 return 0; 2846 } 2847 2848 static int 2849 register_eth_event_callback(void) 2850 { 2851 int ret; 2852 enum rte_eth_event_type event; 2853 2854 for (event = RTE_ETH_EVENT_UNKNOWN; 2855 event < RTE_ETH_EVENT_MAX; event++) { 2856 ret = rte_eth_dev_callback_register(RTE_ETH_ALL, 2857 event, 2858 eth_event_callback, 2859 NULL); 2860 if (ret != 0) { 2861 TESTPMD_LOG(ERR, "Failed to register callback for " 2862 "%s event\n", eth_event_desc[event]); 2863 return -1; 2864 } 2865 } 2866 2867 return 0; 2868 } 2869 2870 /* This function is used by the interrupt thread */ 2871 static void 2872 dev_event_callback(const char *device_name, enum rte_dev_event_type type, 2873 __rte_unused void *arg) 2874 { 2875 uint16_t port_id; 2876 int ret; 2877 2878 if (type >= RTE_DEV_EVENT_MAX) { 2879 fprintf(stderr, "%s called upon invalid event %d\n", 2880 __func__, type); 2881 fflush(stderr); 2882 } 2883 2884 switch (type) { 2885 case RTE_DEV_EVENT_REMOVE: 2886 RTE_LOG(DEBUG, EAL, "The device: %s has been removed!\n", 2887 device_name); 2888 ret = rte_eth_dev_get_port_by_name(device_name, &port_id); 2889 if (ret) { 2890 RTE_LOG(ERR, EAL, "can not get port by device %s!\n", 2891 device_name); 2892 return; 2893 } 2894 /* 2895 * Because the user's callback is invoked in eal interrupt 2896 * callback, the interrupt callback need to be finished before 2897 * it can be unregistered when detaching device. So finish 2898 * callback soon and use a deferred removal to detach device 2899 * is need. It is a workaround, once the device detaching be 2900 * moved into the eal in the future, the deferred removal could 2901 * be deleted. 2902 */ 2903 if (rte_eal_alarm_set(100000, 2904 rmv_port_callback, (void *)(intptr_t)port_id)) 2905 RTE_LOG(ERR, EAL, 2906 "Could not set up deferred device removal\n"); 2907 break; 2908 case RTE_DEV_EVENT_ADD: 2909 RTE_LOG(ERR, EAL, "The device: %s has been added!\n", 2910 device_name); 2911 /* TODO: After finish kernel driver binding, 2912 * begin to attach port. 2913 */ 2914 break; 2915 default: 2916 break; 2917 } 2918 } 2919 2920 static int 2921 set_tx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 2922 { 2923 uint16_t i; 2924 int diag; 2925 uint8_t mapping_found = 0; 2926 2927 for (i = 0; i < nb_tx_queue_stats_mappings; i++) { 2928 if ((tx_queue_stats_mappings[i].port_id == port_id) && 2929 (tx_queue_stats_mappings[i].queue_id < nb_txq )) { 2930 diag = rte_eth_dev_set_tx_queue_stats_mapping(port_id, 2931 tx_queue_stats_mappings[i].queue_id, 2932 tx_queue_stats_mappings[i].stats_counter_id); 2933 if (diag != 0) 2934 return diag; 2935 mapping_found = 1; 2936 } 2937 } 2938 if (mapping_found) 2939 port->tx_queue_stats_mapping_enabled = 1; 2940 return 0; 2941 } 2942 2943 static int 2944 set_rx_queue_stats_mapping_registers(portid_t port_id, struct rte_port *port) 2945 { 2946 uint16_t i; 2947 int diag; 2948 uint8_t mapping_found = 0; 2949 2950 for (i = 0; i < nb_rx_queue_stats_mappings; i++) { 2951 if ((rx_queue_stats_mappings[i].port_id == port_id) && 2952 (rx_queue_stats_mappings[i].queue_id < nb_rxq )) { 2953 diag = rte_eth_dev_set_rx_queue_stats_mapping(port_id, 2954 rx_queue_stats_mappings[i].queue_id, 2955 rx_queue_stats_mappings[i].stats_counter_id); 2956 if (diag != 0) 2957 return diag; 2958 mapping_found = 1; 2959 } 2960 } 2961 if (mapping_found) 2962 port->rx_queue_stats_mapping_enabled = 1; 2963 return 0; 2964 } 2965 2966 static void 2967 map_port_queue_stats_mapping_registers(portid_t pi, struct rte_port *port) 2968 { 2969 int diag = 0; 2970 2971 diag = set_tx_queue_stats_mapping_registers(pi, port); 2972 if (diag != 0) { 2973 if (diag == -ENOTSUP) { 2974 port->tx_queue_stats_mapping_enabled = 0; 2975 printf("TX queue stats mapping not supported port id=%d\n", pi); 2976 } 2977 else 2978 rte_exit(EXIT_FAILURE, 2979 "set_tx_queue_stats_mapping_registers " 2980 "failed for port id=%d diag=%d\n", 2981 pi, diag); 2982 } 2983 2984 diag = set_rx_queue_stats_mapping_registers(pi, port); 2985 if (diag != 0) { 2986 if (diag == -ENOTSUP) { 2987 port->rx_queue_stats_mapping_enabled = 0; 2988 printf("RX queue stats mapping not supported port id=%d\n", pi); 2989 } 2990 else 2991 rte_exit(EXIT_FAILURE, 2992 "set_rx_queue_stats_mapping_registers " 2993 "failed for port id=%d diag=%d\n", 2994 pi, diag); 2995 } 2996 } 2997 2998 static void 2999 rxtx_port_config(struct rte_port *port) 3000 { 3001 uint16_t qid; 3002 uint64_t offloads; 3003 3004 for (qid = 0; qid < nb_rxq; qid++) { 3005 offloads = port->rx_conf[qid].offloads; 3006 port->rx_conf[qid] = port->dev_info.default_rxconf; 3007 if (offloads != 0) 3008 port->rx_conf[qid].offloads = offloads; 3009 3010 /* Check if any Rx parameters have been passed */ 3011 if (rx_pthresh != RTE_PMD_PARAM_UNSET) 3012 port->rx_conf[qid].rx_thresh.pthresh = rx_pthresh; 3013 3014 if (rx_hthresh != RTE_PMD_PARAM_UNSET) 3015 port->rx_conf[qid].rx_thresh.hthresh = rx_hthresh; 3016 3017 if (rx_wthresh != RTE_PMD_PARAM_UNSET) 3018 port->rx_conf[qid].rx_thresh.wthresh = rx_wthresh; 3019 3020 if (rx_free_thresh != RTE_PMD_PARAM_UNSET) 3021 port->rx_conf[qid].rx_free_thresh = rx_free_thresh; 3022 3023 if (rx_drop_en != RTE_PMD_PARAM_UNSET) 3024 port->rx_conf[qid].rx_drop_en = rx_drop_en; 3025 3026 port->nb_rx_desc[qid] = nb_rxd; 3027 } 3028 3029 for (qid = 0; qid < nb_txq; qid++) { 3030 offloads = port->tx_conf[qid].offloads; 3031 port->tx_conf[qid] = port->dev_info.default_txconf; 3032 if (offloads != 0) 3033 port->tx_conf[qid].offloads = offloads; 3034 3035 /* Check if any Tx parameters have been passed */ 3036 if (tx_pthresh != RTE_PMD_PARAM_UNSET) 3037 port->tx_conf[qid].tx_thresh.pthresh = tx_pthresh; 3038 3039 if (tx_hthresh != RTE_PMD_PARAM_UNSET) 3040 port->tx_conf[qid].tx_thresh.hthresh = tx_hthresh; 3041 3042 if (tx_wthresh != RTE_PMD_PARAM_UNSET) 3043 port->tx_conf[qid].tx_thresh.wthresh = tx_wthresh; 3044 3045 if (tx_rs_thresh != RTE_PMD_PARAM_UNSET) 3046 port->tx_conf[qid].tx_rs_thresh = tx_rs_thresh; 3047 3048 if (tx_free_thresh != RTE_PMD_PARAM_UNSET) 3049 port->tx_conf[qid].tx_free_thresh = tx_free_thresh; 3050 3051 port->nb_tx_desc[qid] = nb_txd; 3052 } 3053 } 3054 3055 void 3056 init_port_config(void) 3057 { 3058 portid_t pid; 3059 struct rte_port *port; 3060 int ret; 3061 3062 RTE_ETH_FOREACH_DEV(pid) { 3063 port = &ports[pid]; 3064 port->dev_conf.fdir_conf = fdir_conf; 3065 3066 ret = eth_dev_info_get_print_err(pid, &port->dev_info); 3067 if (ret != 0) 3068 return; 3069 3070 if (nb_rxq > 1) { 3071 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3072 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 3073 rss_hf & port->dev_info.flow_type_rss_offloads; 3074 } else { 3075 port->dev_conf.rx_adv_conf.rss_conf.rss_key = NULL; 3076 port->dev_conf.rx_adv_conf.rss_conf.rss_hf = 0; 3077 } 3078 3079 if (port->dcb_flag == 0) { 3080 if( port->dev_conf.rx_adv_conf.rss_conf.rss_hf != 0) 3081 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_RSS; 3082 else 3083 port->dev_conf.rxmode.mq_mode = ETH_MQ_RX_NONE; 3084 } 3085 3086 rxtx_port_config(port); 3087 3088 ret = eth_macaddr_get_print_err(pid, &port->eth_addr); 3089 if (ret != 0) 3090 return; 3091 3092 map_port_queue_stats_mapping_registers(pid, port); 3093 #if defined RTE_LIBRTE_IXGBE_PMD && defined RTE_LIBRTE_IXGBE_BYPASS 3094 rte_pmd_ixgbe_bypass_init(pid); 3095 #endif 3096 3097 if (lsc_interrupt && 3098 (rte_eth_devices[pid].data->dev_flags & 3099 RTE_ETH_DEV_INTR_LSC)) 3100 port->dev_conf.intr_conf.lsc = 1; 3101 if (rmv_interrupt && 3102 (rte_eth_devices[pid].data->dev_flags & 3103 RTE_ETH_DEV_INTR_RMV)) 3104 port->dev_conf.intr_conf.rmv = 1; 3105 } 3106 } 3107 3108 void set_port_slave_flag(portid_t slave_pid) 3109 { 3110 struct rte_port *port; 3111 3112 port = &ports[slave_pid]; 3113 port->slave_flag = 1; 3114 } 3115 3116 void clear_port_slave_flag(portid_t slave_pid) 3117 { 3118 struct rte_port *port; 3119 3120 port = &ports[slave_pid]; 3121 port->slave_flag = 0; 3122 } 3123 3124 uint8_t port_is_bonding_slave(portid_t slave_pid) 3125 { 3126 struct rte_port *port; 3127 3128 port = &ports[slave_pid]; 3129 if ((rte_eth_devices[slave_pid].data->dev_flags & 3130 RTE_ETH_DEV_BONDED_SLAVE) || (port->slave_flag == 1)) 3131 return 1; 3132 return 0; 3133 } 3134 3135 const uint16_t vlan_tags[] = { 3136 0, 1, 2, 3, 4, 5, 6, 7, 3137 8, 9, 10, 11, 12, 13, 14, 15, 3138 16, 17, 18, 19, 20, 21, 22, 23, 3139 24, 25, 26, 27, 28, 29, 30, 31 3140 }; 3141 3142 static int 3143 get_eth_dcb_conf(portid_t pid, struct rte_eth_conf *eth_conf, 3144 enum dcb_mode_enable dcb_mode, 3145 enum rte_eth_nb_tcs num_tcs, 3146 uint8_t pfc_en) 3147 { 3148 uint8_t i; 3149 int32_t rc; 3150 struct rte_eth_rss_conf rss_conf; 3151 3152 /* 3153 * Builds up the correct configuration for dcb+vt based on the vlan tags array 3154 * given above, and the number of traffic classes available for use. 3155 */ 3156 if (dcb_mode == DCB_VT_ENABLED) { 3157 struct rte_eth_vmdq_dcb_conf *vmdq_rx_conf = 3158 ð_conf->rx_adv_conf.vmdq_dcb_conf; 3159 struct rte_eth_vmdq_dcb_tx_conf *vmdq_tx_conf = 3160 ð_conf->tx_adv_conf.vmdq_dcb_tx_conf; 3161 3162 /* VMDQ+DCB RX and TX configurations */ 3163 vmdq_rx_conf->enable_default_pool = 0; 3164 vmdq_rx_conf->default_pool = 0; 3165 vmdq_rx_conf->nb_queue_pools = 3166 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3167 vmdq_tx_conf->nb_queue_pools = 3168 (num_tcs == ETH_4_TCS ? ETH_32_POOLS : ETH_16_POOLS); 3169 3170 vmdq_rx_conf->nb_pool_maps = vmdq_rx_conf->nb_queue_pools; 3171 for (i = 0; i < vmdq_rx_conf->nb_pool_maps; i++) { 3172 vmdq_rx_conf->pool_map[i].vlan_id = vlan_tags[i]; 3173 vmdq_rx_conf->pool_map[i].pools = 3174 1 << (i % vmdq_rx_conf->nb_queue_pools); 3175 } 3176 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3177 vmdq_rx_conf->dcb_tc[i] = i % num_tcs; 3178 vmdq_tx_conf->dcb_tc[i] = i % num_tcs; 3179 } 3180 3181 /* set DCB mode of RX and TX of multiple queues */ 3182 eth_conf->rxmode.mq_mode = ETH_MQ_RX_VMDQ_DCB; 3183 eth_conf->txmode.mq_mode = ETH_MQ_TX_VMDQ_DCB; 3184 } else { 3185 struct rte_eth_dcb_rx_conf *rx_conf = 3186 ð_conf->rx_adv_conf.dcb_rx_conf; 3187 struct rte_eth_dcb_tx_conf *tx_conf = 3188 ð_conf->tx_adv_conf.dcb_tx_conf; 3189 3190 rc = rte_eth_dev_rss_hash_conf_get(pid, &rss_conf); 3191 if (rc != 0) 3192 return rc; 3193 3194 rx_conf->nb_tcs = num_tcs; 3195 tx_conf->nb_tcs = num_tcs; 3196 3197 for (i = 0; i < ETH_DCB_NUM_USER_PRIORITIES; i++) { 3198 rx_conf->dcb_tc[i] = i % num_tcs; 3199 tx_conf->dcb_tc[i] = i % num_tcs; 3200 } 3201 3202 eth_conf->rxmode.mq_mode = ETH_MQ_RX_DCB_RSS; 3203 eth_conf->rx_adv_conf.rss_conf = rss_conf; 3204 eth_conf->txmode.mq_mode = ETH_MQ_TX_DCB; 3205 } 3206 3207 if (pfc_en) 3208 eth_conf->dcb_capability_en = 3209 ETH_DCB_PG_SUPPORT | ETH_DCB_PFC_SUPPORT; 3210 else 3211 eth_conf->dcb_capability_en = ETH_DCB_PG_SUPPORT; 3212 3213 return 0; 3214 } 3215 3216 int 3217 init_port_dcb_config(portid_t pid, 3218 enum dcb_mode_enable dcb_mode, 3219 enum rte_eth_nb_tcs num_tcs, 3220 uint8_t pfc_en) 3221 { 3222 struct rte_eth_conf port_conf; 3223 struct rte_port *rte_port; 3224 int retval; 3225 uint16_t i; 3226 3227 rte_port = &ports[pid]; 3228 3229 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 3230 /* Enter DCB configuration status */ 3231 dcb_config = 1; 3232 3233 port_conf.rxmode = rte_port->dev_conf.rxmode; 3234 port_conf.txmode = rte_port->dev_conf.txmode; 3235 3236 /*set configuration of DCB in vt mode and DCB in non-vt mode*/ 3237 retval = get_eth_dcb_conf(pid, &port_conf, dcb_mode, num_tcs, pfc_en); 3238 if (retval < 0) 3239 return retval; 3240 port_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3241 3242 /* re-configure the device . */ 3243 retval = rte_eth_dev_configure(pid, nb_rxq, nb_rxq, &port_conf); 3244 if (retval < 0) 3245 return retval; 3246 3247 retval = eth_dev_info_get_print_err(pid, &rte_port->dev_info); 3248 if (retval != 0) 3249 return retval; 3250 3251 /* If dev_info.vmdq_pool_base is greater than 0, 3252 * the queue id of vmdq pools is started after pf queues. 3253 */ 3254 if (dcb_mode == DCB_VT_ENABLED && 3255 rte_port->dev_info.vmdq_pool_base > 0) { 3256 printf("VMDQ_DCB multi-queue mode is nonsensical" 3257 " for port %d.", pid); 3258 return -1; 3259 } 3260 3261 /* Assume the ports in testpmd have the same dcb capability 3262 * and has the same number of rxq and txq in dcb mode 3263 */ 3264 if (dcb_mode == DCB_VT_ENABLED) { 3265 if (rte_port->dev_info.max_vfs > 0) { 3266 nb_rxq = rte_port->dev_info.nb_rx_queues; 3267 nb_txq = rte_port->dev_info.nb_tx_queues; 3268 } else { 3269 nb_rxq = rte_port->dev_info.max_rx_queues; 3270 nb_txq = rte_port->dev_info.max_tx_queues; 3271 } 3272 } else { 3273 /*if vt is disabled, use all pf queues */ 3274 if (rte_port->dev_info.vmdq_pool_base == 0) { 3275 nb_rxq = rte_port->dev_info.max_rx_queues; 3276 nb_txq = rte_port->dev_info.max_tx_queues; 3277 } else { 3278 nb_rxq = (queueid_t)num_tcs; 3279 nb_txq = (queueid_t)num_tcs; 3280 3281 } 3282 } 3283 rx_free_thresh = 64; 3284 3285 memcpy(&rte_port->dev_conf, &port_conf, sizeof(struct rte_eth_conf)); 3286 3287 rxtx_port_config(rte_port); 3288 /* VLAN filter */ 3289 rte_port->dev_conf.rxmode.offloads |= DEV_RX_OFFLOAD_VLAN_FILTER; 3290 for (i = 0; i < RTE_DIM(vlan_tags); i++) 3291 rx_vft_set(pid, vlan_tags[i], 1); 3292 3293 retval = eth_macaddr_get_print_err(pid, &rte_port->eth_addr); 3294 if (retval != 0) 3295 return retval; 3296 3297 map_port_queue_stats_mapping_registers(pid, rte_port); 3298 3299 rte_port->dcb_flag = 1; 3300 3301 return 0; 3302 } 3303 3304 static void 3305 init_port(void) 3306 { 3307 /* Configuration of Ethernet ports. */ 3308 ports = rte_zmalloc("testpmd: ports", 3309 sizeof(struct rte_port) * RTE_MAX_ETHPORTS, 3310 RTE_CACHE_LINE_SIZE); 3311 if (ports == NULL) { 3312 rte_exit(EXIT_FAILURE, 3313 "rte_zmalloc(%d struct rte_port) failed\n", 3314 RTE_MAX_ETHPORTS); 3315 } 3316 3317 /* Initialize ports NUMA structures */ 3318 memset(port_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3319 memset(rxring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3320 memset(txring_numa, NUMA_NO_CONFIG, RTE_MAX_ETHPORTS); 3321 } 3322 3323 static void 3324 force_quit(void) 3325 { 3326 pmd_test_exit(); 3327 prompt_exit(); 3328 } 3329 3330 static void 3331 print_stats(void) 3332 { 3333 uint8_t i; 3334 const char clr[] = { 27, '[', '2', 'J', '\0' }; 3335 const char top_left[] = { 27, '[', '1', ';', '1', 'H', '\0' }; 3336 3337 /* Clear screen and move to top left */ 3338 printf("%s%s", clr, top_left); 3339 3340 printf("\nPort statistics ===================================="); 3341 for (i = 0; i < cur_fwd_config.nb_fwd_ports; i++) 3342 nic_stats_display(fwd_ports_ids[i]); 3343 3344 fflush(stdout); 3345 } 3346 3347 static void 3348 signal_handler(int signum) 3349 { 3350 if (signum == SIGINT || signum == SIGTERM) { 3351 printf("\nSignal %d received, preparing to exit...\n", 3352 signum); 3353 #ifdef RTE_LIBRTE_PDUMP 3354 /* uninitialize packet capture framework */ 3355 rte_pdump_uninit(); 3356 #endif 3357 #ifdef RTE_LIBRTE_LATENCY_STATS 3358 if (latencystats_enabled != 0) 3359 rte_latencystats_uninit(); 3360 #endif 3361 force_quit(); 3362 /* Set flag to indicate the force termination. */ 3363 f_quit = 1; 3364 /* exit with the expected status */ 3365 signal(signum, SIG_DFL); 3366 kill(getpid(), signum); 3367 } 3368 } 3369 3370 int 3371 main(int argc, char** argv) 3372 { 3373 int diag; 3374 portid_t port_id; 3375 uint16_t count; 3376 int ret; 3377 3378 signal(SIGINT, signal_handler); 3379 signal(SIGTERM, signal_handler); 3380 3381 testpmd_logtype = rte_log_register("testpmd"); 3382 if (testpmd_logtype < 0) 3383 rte_exit(EXIT_FAILURE, "Cannot register log type"); 3384 rte_log_set_level(testpmd_logtype, RTE_LOG_DEBUG); 3385 3386 diag = rte_eal_init(argc, argv); 3387 if (diag < 0) 3388 rte_exit(EXIT_FAILURE, "Cannot init EAL: %s\n", 3389 rte_strerror(rte_errno)); 3390 3391 if (rte_eal_process_type() == RTE_PROC_SECONDARY) 3392 rte_exit(EXIT_FAILURE, 3393 "Secondary process type not supported.\n"); 3394 3395 ret = register_eth_event_callback(); 3396 if (ret != 0) 3397 rte_exit(EXIT_FAILURE, "Cannot register for ethdev events"); 3398 3399 #ifdef RTE_LIBRTE_PDUMP 3400 /* initialize packet capture framework */ 3401 rte_pdump_init(); 3402 #endif 3403 3404 count = 0; 3405 RTE_ETH_FOREACH_DEV(port_id) { 3406 ports_ids[count] = port_id; 3407 count++; 3408 } 3409 nb_ports = (portid_t) count; 3410 if (nb_ports == 0) 3411 TESTPMD_LOG(WARNING, "No probed ethernet devices\n"); 3412 3413 /* allocate port structures, and init them */ 3414 init_port(); 3415 3416 set_def_fwd_config(); 3417 if (nb_lcores == 0) 3418 rte_exit(EXIT_FAILURE, "No cores defined for forwarding\n" 3419 "Check the core mask argument\n"); 3420 3421 /* Bitrate/latency stats disabled by default */ 3422 #ifdef RTE_LIBRTE_BITRATE 3423 bitrate_enabled = 0; 3424 #endif 3425 #ifdef RTE_LIBRTE_LATENCY_STATS 3426 latencystats_enabled = 0; 3427 #endif 3428 3429 /* on FreeBSD, mlockall() is disabled by default */ 3430 #ifdef RTE_EXEC_ENV_FREEBSD 3431 do_mlockall = 0; 3432 #else 3433 do_mlockall = 1; 3434 #endif 3435 3436 argc -= diag; 3437 argv += diag; 3438 if (argc > 1) 3439 launch_args_parse(argc, argv); 3440 3441 if (do_mlockall && mlockall(MCL_CURRENT | MCL_FUTURE)) { 3442 TESTPMD_LOG(NOTICE, "mlockall() failed with error \"%s\"\n", 3443 strerror(errno)); 3444 } 3445 3446 if (tx_first && interactive) 3447 rte_exit(EXIT_FAILURE, "--tx-first cannot be used on " 3448 "interactive mode.\n"); 3449 3450 if (tx_first && lsc_interrupt) { 3451 printf("Warning: lsc_interrupt needs to be off when " 3452 " using tx_first. Disabling.\n"); 3453 lsc_interrupt = 0; 3454 } 3455 3456 if (!nb_rxq && !nb_txq) 3457 printf("Warning: Either rx or tx queues should be non-zero\n"); 3458 3459 if (nb_rxq > 1 && nb_rxq > nb_txq) 3460 printf("Warning: nb_rxq=%d enables RSS configuration, " 3461 "but nb_txq=%d will prevent to fully test it.\n", 3462 nb_rxq, nb_txq); 3463 3464 init_config(); 3465 3466 if (hot_plug) { 3467 ret = rte_dev_hotplug_handle_enable(); 3468 if (ret) { 3469 RTE_LOG(ERR, EAL, 3470 "fail to enable hotplug handling."); 3471 return -1; 3472 } 3473 3474 ret = rte_dev_event_monitor_start(); 3475 if (ret) { 3476 RTE_LOG(ERR, EAL, 3477 "fail to start device event monitoring."); 3478 return -1; 3479 } 3480 3481 ret = rte_dev_event_callback_register(NULL, 3482 dev_event_callback, NULL); 3483 if (ret) { 3484 RTE_LOG(ERR, EAL, 3485 "fail to register device event callback\n"); 3486 return -1; 3487 } 3488 } 3489 3490 if (!no_device_start && start_port(RTE_PORT_ALL) != 0) 3491 rte_exit(EXIT_FAILURE, "Start ports failed\n"); 3492 3493 /* set all ports to promiscuous mode by default */ 3494 RTE_ETH_FOREACH_DEV(port_id) { 3495 ret = rte_eth_promiscuous_enable(port_id); 3496 if (ret != 0) 3497 printf("Error during enabling promiscuous mode for port %u: %s - ignore\n", 3498 port_id, rte_strerror(-ret)); 3499 } 3500 3501 /* Init metrics library */ 3502 rte_metrics_init(rte_socket_id()); 3503 3504 #ifdef RTE_LIBRTE_LATENCY_STATS 3505 if (latencystats_enabled != 0) { 3506 int ret = rte_latencystats_init(1, NULL); 3507 if (ret) 3508 printf("Warning: latencystats init()" 3509 " returned error %d\n", ret); 3510 printf("Latencystats running on lcore %d\n", 3511 latencystats_lcore_id); 3512 } 3513 #endif 3514 3515 /* Setup bitrate stats */ 3516 #ifdef RTE_LIBRTE_BITRATE 3517 if (bitrate_enabled != 0) { 3518 bitrate_data = rte_stats_bitrate_create(); 3519 if (bitrate_data == NULL) 3520 rte_exit(EXIT_FAILURE, 3521 "Could not allocate bitrate data.\n"); 3522 rte_stats_bitrate_reg(bitrate_data); 3523 } 3524 #endif 3525 3526 #ifdef RTE_LIBRTE_CMDLINE 3527 if (strlen(cmdline_filename) != 0) 3528 cmdline_read_from_file(cmdline_filename); 3529 3530 if (interactive == 1) { 3531 if (auto_start) { 3532 printf("Start automatic packet forwarding\n"); 3533 start_packet_forwarding(0); 3534 } 3535 prompt(); 3536 pmd_test_exit(); 3537 } else 3538 #endif 3539 { 3540 char c; 3541 int rc; 3542 3543 f_quit = 0; 3544 3545 printf("No commandline core given, start packet forwarding\n"); 3546 start_packet_forwarding(tx_first); 3547 if (stats_period != 0) { 3548 uint64_t prev_time = 0, cur_time, diff_time = 0; 3549 uint64_t timer_period; 3550 3551 /* Convert to number of cycles */ 3552 timer_period = stats_period * rte_get_timer_hz(); 3553 3554 while (f_quit == 0) { 3555 cur_time = rte_get_timer_cycles(); 3556 diff_time += cur_time - prev_time; 3557 3558 if (diff_time >= timer_period) { 3559 print_stats(); 3560 /* Reset the timer */ 3561 diff_time = 0; 3562 } 3563 /* Sleep to avoid unnecessary checks */ 3564 prev_time = cur_time; 3565 sleep(1); 3566 } 3567 } 3568 3569 printf("Press enter to exit\n"); 3570 rc = read(0, &c, 1); 3571 pmd_test_exit(); 3572 if (rc < 0) 3573 return 1; 3574 } 3575 3576 ret = rte_eal_cleanup(); 3577 if (ret != 0) 3578 rte_exit(EXIT_FAILURE, 3579 "EAL cleanup failed: %s\n", strerror(-ret)); 3580 3581 return EXIT_SUCCESS; 3582 } 3583