1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include <stdio.h> 35 #include <unistd.h> 36 #include <stdlib.h> 37 #include <stdbool.h> 38 #include <string.h> 39 40 #include <rte_config.h> 41 #include <rte_lcore.h> 42 43 #include "spdk/ioat.h" 44 #include "spdk/env.h" 45 #include "spdk/queue.h" 46 #include "spdk/string.h" 47 48 struct user_config { 49 int xfer_size_bytes; 50 int queue_depth; 51 int time_in_sec; 52 bool verify; 53 char *core_mask; 54 }; 55 56 struct ioat_device { 57 struct spdk_ioat_chan *ioat; 58 TAILQ_ENTRY(ioat_device) tailq; 59 }; 60 61 static TAILQ_HEAD(, ioat_device) g_devices; 62 static struct ioat_device *g_next_device; 63 64 static struct user_config g_user_config; 65 66 struct thread_entry { 67 struct spdk_ioat_chan *chan; 68 uint64_t xfer_completed; 69 uint64_t xfer_failed; 70 uint64_t current_queue_depth; 71 unsigned lcore_id; 72 bool is_draining; 73 struct spdk_mempool *data_pool; 74 struct spdk_mempool *task_pool; 75 }; 76 77 struct ioat_task { 78 struct thread_entry *thread_entry; 79 void *src; 80 void *dst; 81 }; 82 83 static void submit_single_xfer(struct thread_entry *thread_entry, struct ioat_task *ioat_task, 84 void *dst, void *src); 85 86 static void 87 construct_user_config(struct user_config *self) 88 { 89 self->xfer_size_bytes = 4096; 90 self->queue_depth = 256; 91 self->time_in_sec = 10; 92 self->verify = false; 93 self->core_mask = "0x1"; 94 } 95 96 static void 97 dump_user_config(struct user_config *self) 98 { 99 printf("User configuration:\n"); 100 printf("Transfer size: %u bytes\n", self->xfer_size_bytes); 101 printf("Queue depth: %u\n", self->queue_depth); 102 printf("Run time: %u seconds\n", self->time_in_sec); 103 printf("Core mask: %s\n", self->core_mask); 104 printf("Verify: %s\n\n", self->verify ? "Yes" : "No"); 105 } 106 107 static void 108 ioat_exit(void) 109 { 110 struct ioat_device *dev; 111 112 while (!TAILQ_EMPTY(&g_devices)) { 113 dev = TAILQ_FIRST(&g_devices); 114 TAILQ_REMOVE(&g_devices, dev, tailq); 115 if (dev->ioat) { 116 spdk_ioat_detach(dev->ioat); 117 } 118 spdk_free(dev); 119 } 120 } 121 122 static void 123 ioat_done(void *cb_arg) 124 { 125 struct ioat_task *ioat_task = (struct ioat_task *)cb_arg; 126 struct thread_entry *thread_entry = ioat_task->thread_entry; 127 128 if (g_user_config.verify && memcmp(ioat_task->src, ioat_task->dst, g_user_config.xfer_size_bytes)) { 129 thread_entry->xfer_failed++; 130 } else { 131 thread_entry->xfer_completed++; 132 } 133 134 thread_entry->current_queue_depth--; 135 136 if (thread_entry->is_draining) { 137 spdk_mempool_put(thread_entry->data_pool, ioat_task->src); 138 spdk_mempool_put(thread_entry->data_pool, ioat_task->dst); 139 spdk_mempool_put(thread_entry->task_pool, ioat_task); 140 } else { 141 submit_single_xfer(thread_entry, ioat_task, ioat_task->dst, ioat_task->src); 142 } 143 } 144 145 static bool 146 probe_cb(void *cb_ctx, struct spdk_pci_device *pci_dev) 147 { 148 printf(" Found matching device at %d:%d:%d " 149 "vendor:0x%04x device:0x%04x\n", 150 spdk_pci_device_get_bus(pci_dev), spdk_pci_device_get_dev(pci_dev), 151 spdk_pci_device_get_func(pci_dev), 152 spdk_pci_device_get_vendor_id(pci_dev), spdk_pci_device_get_device_id(pci_dev)); 153 154 return true; 155 } 156 157 static void 158 attach_cb(void *cb_ctx, struct spdk_pci_device *pci_dev, struct spdk_ioat_chan *ioat) 159 { 160 struct ioat_device *dev; 161 162 dev = spdk_zmalloc(sizeof(*dev), 0, NULL); 163 if (dev == NULL) { 164 printf("Failed to allocate device struct\n"); 165 return; 166 } 167 168 dev->ioat = ioat; 169 TAILQ_INSERT_TAIL(&g_devices, dev, tailq); 170 } 171 172 static int 173 ioat_init(void) 174 { 175 TAILQ_INIT(&g_devices); 176 177 if (spdk_ioat_probe(NULL, probe_cb, attach_cb) != 0) { 178 fprintf(stderr, "ioat_probe() failed\n"); 179 return 1; 180 } 181 182 return 0; 183 } 184 185 static void 186 usage(char *program_name) 187 { 188 printf("%s options\n", program_name); 189 printf("\t[-h help message]\n"); 190 printf("\t[-c core mask for distributing I/O submission/completion work]\n"); 191 printf("\t[-q queue depth]\n"); 192 printf("\t[-s transfer size in bytes]\n"); 193 printf("\t[-t time in seconds]\n"); 194 printf("\t[-v verify copy result if this switch is on]\n"); 195 } 196 197 static int 198 parse_args(int argc, char **argv) 199 { 200 int op; 201 202 construct_user_config(&g_user_config); 203 while ((op = getopt(argc, argv, "c:hq:s:t:v")) != -1) { 204 switch (op) { 205 case 's': 206 g_user_config.xfer_size_bytes = atoi(optarg); 207 break; 208 case 'q': 209 g_user_config.queue_depth = atoi(optarg); 210 break; 211 case 't': 212 g_user_config.time_in_sec = atoi(optarg); 213 break; 214 case 'c': 215 g_user_config.core_mask = optarg; 216 break; 217 case 'v': 218 g_user_config.verify = true; 219 break; 220 case 'h': 221 usage(argv[0]); 222 exit(0); 223 default: 224 usage(argv[0]); 225 return 1; 226 } 227 } 228 if (!g_user_config.xfer_size_bytes || !g_user_config.queue_depth || 229 !g_user_config.time_in_sec || !g_user_config.core_mask) { 230 usage(argv[0]); 231 return 1; 232 } 233 optind = 1; 234 return 0; 235 } 236 237 static void 238 drain_io(struct thread_entry *thread_entry) 239 { 240 while (thread_entry->current_queue_depth > 0) { 241 spdk_ioat_process_events(thread_entry->chan); 242 } 243 } 244 245 static void 246 submit_single_xfer(struct thread_entry *thread_entry, struct ioat_task *ioat_task, void *dst, 247 void *src) 248 { 249 ioat_task->thread_entry = thread_entry; 250 ioat_task->src = src; 251 ioat_task->dst = dst; 252 253 spdk_ioat_submit_copy(thread_entry->chan, ioat_task, ioat_done, dst, src, 254 g_user_config.xfer_size_bytes); 255 256 thread_entry->current_queue_depth++; 257 } 258 259 static void 260 submit_xfers(struct thread_entry *thread_entry, uint64_t queue_depth) 261 { 262 while (queue_depth-- > 0) { 263 void *src = NULL, *dst = NULL; 264 struct ioat_task *ioat_task = NULL; 265 266 src = spdk_mempool_get(thread_entry->data_pool); 267 dst = spdk_mempool_get(thread_entry->data_pool); 268 ioat_task = spdk_mempool_get(thread_entry->task_pool); 269 270 submit_single_xfer(thread_entry, ioat_task, dst, src); 271 } 272 } 273 274 static int 275 work_fn(void *arg) 276 { 277 char buf_pool_name[20], task_pool_name[20]; 278 uint64_t tsc_end; 279 struct thread_entry *t = (struct thread_entry *)arg; 280 281 if (!t->chan) { 282 return 0; 283 } 284 285 t->lcore_id = rte_lcore_id(); 286 287 snprintf(buf_pool_name, sizeof(buf_pool_name), "buf_pool_%d", rte_lcore_id()); 288 snprintf(task_pool_name, sizeof(task_pool_name), "task_pool_%d", rte_lcore_id()); 289 t->data_pool = spdk_mempool_create(buf_pool_name, 512, g_user_config.xfer_size_bytes, -1); 290 t->task_pool = spdk_mempool_create(task_pool_name, 512, sizeof(struct ioat_task), -1); 291 if (!t->data_pool || !t->task_pool) { 292 fprintf(stderr, "Could not allocate buffer pool.\n"); 293 return 1; 294 } 295 296 tsc_end = spdk_get_ticks() + g_user_config.time_in_sec * spdk_get_ticks_hz(); 297 298 // begin to submit transfers 299 submit_xfers(t, g_user_config.queue_depth); 300 while (spdk_get_ticks() < tsc_end) { 301 spdk_ioat_process_events(t->chan); 302 } 303 304 // begin to drain io 305 t->is_draining = true; 306 drain_io(t); 307 308 return 0; 309 } 310 311 static int 312 init(void) 313 { 314 char *core_mask_conf; 315 316 core_mask_conf = spdk_sprintf_alloc("-c %s", g_user_config.core_mask); 317 if (!core_mask_conf) { 318 return 1; 319 } 320 321 char *ealargs[] = {"perf", core_mask_conf, "-n 4"}; 322 323 if (rte_eal_init(sizeof(ealargs) / sizeof(ealargs[0]), ealargs) < 0) { 324 free(core_mask_conf); 325 fprintf(stderr, "Could not init eal\n"); 326 return 1; 327 } 328 329 free(core_mask_conf); 330 331 if (ioat_init() != 0) { 332 fprintf(stderr, "Could not init ioat\n"); 333 return 1; 334 } 335 336 return 0; 337 } 338 339 static int 340 dump_result(struct thread_entry *threads, int len) 341 { 342 int i; 343 uint64_t total_completed = 0; 344 uint64_t total_failed = 0; 345 uint64_t total_xfer_per_sec, total_bw_in_MBps; 346 347 printf("lcore Transfers Bandwidth Failed\n"); 348 printf("--------------------------------------------\n"); 349 for (i = 0; i < len; i++) { 350 struct thread_entry *t = &threads[i]; 351 352 uint64_t xfer_per_sec = t->xfer_completed / g_user_config.time_in_sec; 353 uint64_t bw_in_MBps = (t->xfer_completed * g_user_config.xfer_size_bytes) / 354 (g_user_config.time_in_sec * 1024 * 1024); 355 356 total_completed += t->xfer_completed; 357 total_failed += t->xfer_failed; 358 359 if (xfer_per_sec) { 360 printf("%5d %10" PRIu64 "/s %10" PRIu64 " MB/s %6" PRIu64 "\n", 361 t->lcore_id, xfer_per_sec, bw_in_MBps, t->xfer_failed); 362 } 363 } 364 365 total_xfer_per_sec = total_completed / g_user_config.time_in_sec; 366 total_bw_in_MBps = (total_completed * g_user_config.xfer_size_bytes) / 367 (g_user_config.time_in_sec * 1024 * 1024); 368 369 printf("============================================\n"); 370 printf("Total: %10" PRIu64 "/s %10" PRIu64 " MB/s %6" PRIu64 "\n", 371 total_xfer_per_sec, total_bw_in_MBps, total_failed); 372 return total_failed ? 1 : 0; 373 } 374 375 static struct spdk_ioat_chan * 376 get_next_chan(void) 377 { 378 struct spdk_ioat_chan *chan; 379 380 if (g_next_device == NULL) { 381 fprintf(stderr, "Not enough ioat channels found. Check that ioatdma driver is unloaded.\n"); 382 return NULL; 383 } 384 385 chan = g_next_device->ioat; 386 387 g_next_device = TAILQ_NEXT(g_next_device, tailq); 388 389 return chan; 390 } 391 392 int 393 main(int argc, char **argv) 394 { 395 unsigned lcore_id; 396 struct thread_entry threads[RTE_MAX_LCORE] = {}; 397 int rc; 398 399 if (parse_args(argc, argv) != 0) { 400 return 1; 401 } 402 403 if (init() != 0) { 404 return 1; 405 } 406 407 dump_user_config(&g_user_config); 408 409 g_next_device = TAILQ_FIRST(&g_devices); 410 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 411 threads[lcore_id].chan = get_next_chan(); 412 rte_eal_remote_launch(work_fn, &threads[lcore_id], lcore_id); 413 } 414 415 threads[rte_get_master_lcore()].chan = get_next_chan(); 416 if (work_fn(&threads[rte_get_master_lcore()]) != 0) { 417 rc = 1; 418 goto cleanup; 419 } 420 421 RTE_LCORE_FOREACH_SLAVE(lcore_id) { 422 if (rte_eal_wait_lcore(lcore_id) != 0) { 423 rc = 1; 424 goto cleanup; 425 } 426 } 427 428 rc = dump_result(threads, RTE_MAX_LCORE); 429 430 cleanup: 431 ioat_exit(); 432 433 return rc; 434 } 435