1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2010-2015 Intel Corporation 3 */ 4 5 #include <stdint.h> 6 #include <stdlib.h> 7 #include <inttypes.h> 8 #include <getopt.h> 9 #include <rte_eal.h> 10 #include <rte_ethdev.h> 11 #include <rte_cycles.h> 12 #include <rte_lcore.h> 13 #include <rte_mbuf.h> 14 #include <rte_mbuf_dyn.h> 15 16 #define RX_RING_SIZE 1024 17 #define TX_RING_SIZE 1024 18 19 #define NUM_MBUFS 8191 20 #define MBUF_CACHE_SIZE 250 21 #define BURST_SIZE 32 22 23 static int hwts_dynfield_offset = -1; 24 25 static inline rte_mbuf_timestamp_t * 26 hwts_field(struct rte_mbuf *mbuf) 27 { 28 return RTE_MBUF_DYNFIELD(mbuf, 29 hwts_dynfield_offset, rte_mbuf_timestamp_t *); 30 } 31 32 typedef uint64_t tsc_t; 33 static int tsc_dynfield_offset = -1; 34 35 static inline tsc_t * 36 tsc_field(struct rte_mbuf *mbuf) 37 { 38 return RTE_MBUF_DYNFIELD(mbuf, tsc_dynfield_offset, tsc_t *); 39 } 40 41 static const char usage[] = 42 "%s EAL_ARGS -- [-t]\n"; 43 44 static struct { 45 uint64_t total_cycles; 46 uint64_t total_queue_cycles; 47 uint64_t total_pkts; 48 } latency_numbers; 49 50 int hw_timestamping; 51 52 #define TICKS_PER_CYCLE_SHIFT 16 53 static uint64_t ticks_per_cycle_mult; 54 55 /* Callback added to the RX port and applied to packets. 8< */ 56 static uint16_t 57 add_timestamps(uint16_t port __rte_unused, uint16_t qidx __rte_unused, 58 struct rte_mbuf **pkts, uint16_t nb_pkts, 59 uint16_t max_pkts __rte_unused, void *_ __rte_unused) 60 { 61 unsigned i; 62 uint64_t now = rte_rdtsc(); 63 64 for (i = 0; i < nb_pkts; i++) 65 *tsc_field(pkts[i]) = now; 66 return nb_pkts; 67 } 68 /* >8 End of callback addition and application. */ 69 70 /* Callback is added to the TX port. 8< */ 71 static uint16_t 72 calc_latency(uint16_t port, uint16_t qidx __rte_unused, 73 struct rte_mbuf **pkts, uint16_t nb_pkts, void *_ __rte_unused) 74 { 75 uint64_t cycles = 0; 76 uint64_t queue_ticks = 0; 77 uint64_t now = rte_rdtsc(); 78 uint64_t ticks; 79 unsigned i; 80 81 if (hw_timestamping) 82 rte_eth_read_clock(port, &ticks); 83 84 for (i = 0; i < nb_pkts; i++) { 85 cycles += now - *tsc_field(pkts[i]); 86 if (hw_timestamping) 87 queue_ticks += ticks - *hwts_field(pkts[i]); 88 } 89 90 latency_numbers.total_cycles += cycles; 91 if (hw_timestamping) 92 latency_numbers.total_queue_cycles += (queue_ticks 93 * ticks_per_cycle_mult) >> TICKS_PER_CYCLE_SHIFT; 94 95 latency_numbers.total_pkts += nb_pkts; 96 97 if (latency_numbers.total_pkts > (100 * 1000 * 1000ULL)) { 98 printf("Latency = %"PRIu64" cycles\n", 99 latency_numbers.total_cycles / latency_numbers.total_pkts); 100 if (hw_timestamping) { 101 printf("Latency from HW = %"PRIu64" cycles\n", 102 latency_numbers.total_queue_cycles 103 / latency_numbers.total_pkts); 104 } 105 latency_numbers.total_cycles = 0; 106 latency_numbers.total_queue_cycles = 0; 107 latency_numbers.total_pkts = 0; 108 } 109 return nb_pkts; 110 } 111 /* >8 End of callback addition. */ 112 113 /* 114 * Initialises a given port using global settings and with the rx buffers 115 * coming from the mbuf_pool passed as parameter 116 */ 117 118 /* Port initialization. 8< */ 119 static inline int 120 port_init(uint16_t port, struct rte_mempool *mbuf_pool) 121 { 122 struct rte_eth_conf port_conf; 123 const uint16_t rx_rings = 1, tx_rings = 1; 124 uint16_t nb_rxd = RX_RING_SIZE; 125 uint16_t nb_txd = TX_RING_SIZE; 126 int retval; 127 uint16_t q; 128 struct rte_eth_dev_info dev_info; 129 struct rte_eth_rxconf rxconf; 130 struct rte_eth_txconf txconf; 131 132 if (!rte_eth_dev_is_valid_port(port)) 133 return -1; 134 135 memset(&port_conf, 0, sizeof(struct rte_eth_conf)); 136 137 retval = rte_eth_dev_info_get(port, &dev_info); 138 if (retval != 0) { 139 printf("Error during getting device (port %u) info: %s\n", 140 port, strerror(-retval)); 141 142 return retval; 143 } 144 145 if (dev_info.tx_offload_capa & RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE) 146 port_conf.txmode.offloads |= 147 RTE_ETH_TX_OFFLOAD_MBUF_FAST_FREE; 148 149 if (hw_timestamping) { 150 if (!(dev_info.rx_offload_capa & RTE_ETH_RX_OFFLOAD_TIMESTAMP)) { 151 printf("\nERROR: Port %u does not support hardware timestamping\n" 152 , port); 153 return -1; 154 } 155 port_conf.rxmode.offloads |= RTE_ETH_RX_OFFLOAD_TIMESTAMP; 156 rte_mbuf_dyn_rx_timestamp_register(&hwts_dynfield_offset, NULL); 157 if (hwts_dynfield_offset < 0) { 158 printf("ERROR: Failed to register timestamp field\n"); 159 return -rte_errno; 160 } 161 } 162 163 retval = rte_eth_dev_configure(port, rx_rings, tx_rings, &port_conf); 164 if (retval != 0) 165 return retval; 166 167 retval = rte_eth_dev_adjust_nb_rx_tx_desc(port, &nb_rxd, &nb_txd); 168 if (retval != 0) 169 return retval; 170 171 rxconf = dev_info.default_rxconf; 172 173 for (q = 0; q < rx_rings; q++) { 174 retval = rte_eth_rx_queue_setup(port, q, nb_rxd, 175 rte_eth_dev_socket_id(port), &rxconf, mbuf_pool); 176 if (retval < 0) 177 return retval; 178 } 179 180 txconf = dev_info.default_txconf; 181 txconf.offloads = port_conf.txmode.offloads; 182 for (q = 0; q < tx_rings; q++) { 183 retval = rte_eth_tx_queue_setup(port, q, nb_txd, 184 rte_eth_dev_socket_id(port), &txconf); 185 if (retval < 0) 186 return retval; 187 } 188 189 retval = rte_eth_dev_start(port); 190 if (retval < 0) 191 return retval; 192 193 if (hw_timestamping && ticks_per_cycle_mult == 0) { 194 uint64_t cycles_base = rte_rdtsc(); 195 uint64_t ticks_base; 196 retval = rte_eth_read_clock(port, &ticks_base); 197 if (retval != 0) 198 return retval; 199 rte_delay_ms(100); 200 uint64_t cycles = rte_rdtsc(); 201 uint64_t ticks; 202 rte_eth_read_clock(port, &ticks); 203 uint64_t c_freq = cycles - cycles_base; 204 uint64_t t_freq = ticks - ticks_base; 205 double freq_mult = (double)c_freq / t_freq; 206 printf("TSC Freq ~= %" PRIu64 207 "\nHW Freq ~= %" PRIu64 208 "\nRatio : %f\n", 209 c_freq * 10, t_freq * 10, freq_mult); 210 /* TSC will be faster than internal ticks so freq_mult is > 0 211 * We convert the multiplication to an integer shift & mult 212 */ 213 ticks_per_cycle_mult = (1 << TICKS_PER_CYCLE_SHIFT) / freq_mult; 214 } 215 216 struct rte_ether_addr addr; 217 218 retval = rte_eth_macaddr_get(port, &addr); 219 if (retval < 0) { 220 printf("Failed to get MAC address on port %u: %s\n", 221 port, rte_strerror(-retval)); 222 return retval; 223 } 224 printf("Port %u MAC: %02"PRIx8" %02"PRIx8" %02"PRIx8 225 " %02"PRIx8" %02"PRIx8" %02"PRIx8"\n", 226 (unsigned)port, 227 RTE_ETHER_ADDR_BYTES(&addr)); 228 229 retval = rte_eth_promiscuous_enable(port); 230 if (retval != 0) 231 return retval; 232 233 /* RX and TX callbacks are added to the ports. 8< */ 234 rte_eth_add_rx_callback(port, 0, add_timestamps, NULL); 235 rte_eth_add_tx_callback(port, 0, calc_latency, NULL); 236 /* >8 End of RX and TX callbacks. */ 237 238 return 0; 239 } 240 /* >8 End of port initialization. */ 241 242 /* 243 * Main thread that does the work, reading from INPUT_PORT 244 * and writing to OUTPUT_PORT 245 */ 246 static __rte_noreturn void 247 lcore_main(void) 248 { 249 uint16_t port; 250 251 RTE_ETH_FOREACH_DEV(port) 252 if (rte_eth_dev_socket_id(port) > 0 && 253 rte_eth_dev_socket_id(port) != 254 (int)rte_socket_id()) 255 printf("WARNING, port %u is on remote NUMA node to " 256 "polling thread.\n\tPerformance will " 257 "not be optimal.\n", port); 258 259 printf("\nCore %u forwarding packets. [Ctrl+C to quit]\n", 260 rte_lcore_id()); 261 for (;;) { 262 RTE_ETH_FOREACH_DEV(port) { 263 struct rte_mbuf *bufs[BURST_SIZE]; 264 const uint16_t nb_rx = rte_eth_rx_burst(port, 0, 265 bufs, BURST_SIZE); 266 if (unlikely(nb_rx == 0)) 267 continue; 268 const uint16_t nb_tx = rte_eth_tx_burst(port ^ 1, 0, 269 bufs, nb_rx); 270 if (unlikely(nb_tx < nb_rx)) { 271 uint16_t buf; 272 273 for (buf = nb_tx; buf < nb_rx; buf++) 274 rte_pktmbuf_free(bufs[buf]); 275 } 276 } 277 } 278 } 279 280 /* Main function, does initialisation and calls the per-lcore functions */ 281 int 282 main(int argc, char *argv[]) 283 { 284 struct rte_mempool *mbuf_pool; 285 uint16_t nb_ports; 286 uint16_t portid; 287 struct option lgopts[] = { 288 { NULL, 0, 0, 0 } 289 }; 290 int opt, option_index; 291 292 static const struct rte_mbuf_dynfield tsc_dynfield_desc = { 293 .name = "example_bbdev_dynfield_tsc", 294 .size = sizeof(tsc_t), 295 .align = __alignof__(tsc_t), 296 }; 297 298 /* init EAL */ 299 int ret = rte_eal_init(argc, argv); 300 301 if (ret < 0) 302 rte_exit(EXIT_FAILURE, "Error with EAL initialization\n"); 303 argc -= ret; 304 argv += ret; 305 306 while ((opt = getopt_long(argc, argv, "t", lgopts, &option_index)) 307 != EOF) 308 switch (opt) { 309 case 't': 310 hw_timestamping = 1; 311 break; 312 default: 313 printf(usage, argv[0]); 314 return -1; 315 } 316 optind = 1; /* reset getopt lib */ 317 318 nb_ports = rte_eth_dev_count_avail(); 319 if (nb_ports < 2 || (nb_ports & 1)) 320 rte_exit(EXIT_FAILURE, "Error: number of ports must be even\n"); 321 322 mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", 323 NUM_MBUFS * nb_ports, MBUF_CACHE_SIZE, 0, 324 RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id()); 325 if (mbuf_pool == NULL) 326 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n"); 327 328 tsc_dynfield_offset = 329 rte_mbuf_dynfield_register(&tsc_dynfield_desc); 330 if (tsc_dynfield_offset < 0) 331 rte_exit(EXIT_FAILURE, "Cannot register mbuf field\n"); 332 333 /* initialize all ports */ 334 RTE_ETH_FOREACH_DEV(portid) 335 if (port_init(portid, mbuf_pool) != 0) 336 rte_exit(EXIT_FAILURE, "Cannot init port %"PRIu16"\n", 337 portid); 338 339 if (rte_lcore_count() > 1) 340 printf("\nWARNING: Too much enabled lcores - " 341 "App uses only 1 lcore\n"); 342 343 /* call lcore_main on main core only */ 344 lcore_main(); 345 346 /* clean up the EAL */ 347 rte_eal_cleanup(); 348 349 return 0; 350 } 351