1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include "spdk/ioat.h" 37 #include "spdk/env.h" 38 #include "spdk/queue.h" 39 #include "spdk/string.h" 40 41 struct user_config { 42 int xfer_size_bytes; 43 int queue_depth; 44 int time_in_sec; 45 bool verify; 46 char *core_mask; 47 int ioat_chan_num; 48 }; 49 50 struct ioat_device { 51 struct spdk_ioat_chan *ioat; 52 TAILQ_ENTRY(ioat_device) tailq; 53 }; 54 55 static TAILQ_HEAD(, ioat_device) g_devices = TAILQ_HEAD_INITIALIZER(g_devices); 56 static struct ioat_device *g_next_device; 57 58 static struct user_config g_user_config; 59 60 struct ioat_chan_entry { 61 struct spdk_ioat_chan *chan; 62 int ioat_chan_id; 63 uint64_t xfer_completed; 64 uint64_t xfer_failed; 65 uint64_t current_queue_depth; 66 uint64_t waiting_for_flush; 67 uint64_t flush_threshold; 68 bool is_draining; 69 struct spdk_mempool *data_pool; 70 struct spdk_mempool *task_pool; 71 struct ioat_chan_entry *next; 72 }; 73 74 struct worker_thread { 75 struct ioat_chan_entry *ctx; 76 struct worker_thread *next; 77 unsigned core; 78 }; 79 80 struct ioat_task { 81 struct ioat_chan_entry *ioat_chan_entry; 82 void *src; 83 void *dst; 84 }; 85 86 static struct worker_thread *g_workers = NULL; 87 static int g_num_workers = 0; 88 static int g_ioat_chan_num = 0; 89 90 static void submit_single_xfer(struct ioat_chan_entry *ioat_chan_entry, struct ioat_task *ioat_task, 91 void *dst, void *src); 92 93 static void 94 construct_user_config(struct user_config *self) 95 { 96 self->xfer_size_bytes = 4096; 97 self->ioat_chan_num = 1; 98 self->queue_depth = 256; 99 self->time_in_sec = 10; 100 self->verify = false; 101 self->core_mask = "0x1"; 102 } 103 104 static void 105 dump_user_config(struct user_config *self) 106 { 107 printf("User configuration:\n"); 108 printf("Number of channels: %u\n", self->ioat_chan_num); 109 printf("Transfer size: %u bytes\n", self->xfer_size_bytes); 110 printf("Queue depth: %u\n", self->queue_depth); 111 printf("Run time: %u seconds\n", self->time_in_sec); 112 printf("Core mask: %s\n", self->core_mask); 113 printf("Verify: %s\n\n", self->verify ? "Yes" : "No"); 114 } 115 116 static void 117 ioat_exit(void) 118 { 119 struct ioat_device *dev; 120 121 while (!TAILQ_EMPTY(&g_devices)) { 122 dev = TAILQ_FIRST(&g_devices); 123 TAILQ_REMOVE(&g_devices, dev, tailq); 124 if (dev->ioat) { 125 spdk_ioat_detach(dev->ioat); 126 } 127 spdk_dma_free(dev); 128 } 129 } 130 131 static void 132 ioat_done(void *cb_arg) 133 { 134 struct ioat_task *ioat_task = (struct ioat_task *)cb_arg; 135 struct ioat_chan_entry *ioat_chan_entry = ioat_task->ioat_chan_entry; 136 137 if (g_user_config.verify && memcmp(ioat_task->src, ioat_task->dst, g_user_config.xfer_size_bytes)) { 138 ioat_chan_entry->xfer_failed++; 139 } else { 140 ioat_chan_entry->xfer_completed++; 141 } 142 143 ioat_chan_entry->current_queue_depth--; 144 145 if (ioat_chan_entry->is_draining) { 146 spdk_mempool_put(ioat_chan_entry->data_pool, ioat_task->src); 147 spdk_mempool_put(ioat_chan_entry->data_pool, ioat_task->dst); 148 spdk_mempool_put(ioat_chan_entry->task_pool, ioat_task); 149 } else { 150 submit_single_xfer(ioat_chan_entry, ioat_task, ioat_task->dst, ioat_task->src); 151 } 152 } 153 154 static int 155 register_workers(void) 156 { 157 uint32_t i; 158 struct worker_thread *worker; 159 160 g_workers = NULL; 161 g_num_workers = 0; 162 163 SPDK_ENV_FOREACH_CORE(i) { 164 worker = calloc(1, sizeof(*worker)); 165 if (worker == NULL) { 166 fprintf(stderr, "Unable to allocate worker\n"); 167 return 1; 168 } 169 170 worker->core = i; 171 worker->next = g_workers; 172 g_workers = worker; 173 g_num_workers++; 174 } 175 176 return 0; 177 } 178 179 static void 180 unregister_workers(void) 181 { 182 struct worker_thread *worker = g_workers; 183 struct ioat_chan_entry *entry, *entry1; 184 185 /* Free ioat_chan_entry and worker thread */ 186 while (worker) { 187 struct worker_thread *next_worker = worker->next; 188 entry = worker->ctx; 189 while (entry) { 190 entry1 = entry->next; 191 spdk_mempool_free(entry->data_pool); 192 spdk_mempool_free(entry->task_pool); 193 free(entry); 194 entry = entry1; 195 } 196 free(worker); 197 worker = next_worker; 198 } 199 } 200 201 static bool 202 probe_cb(void *cb_ctx, struct spdk_pci_device *pci_dev) 203 { 204 printf(" Found matching device at %04x:%02x:%02x.%x " 205 "vendor:0x%04x device:0x%04x\n", 206 spdk_pci_device_get_domain(pci_dev), 207 spdk_pci_device_get_bus(pci_dev), spdk_pci_device_get_dev(pci_dev), 208 spdk_pci_device_get_func(pci_dev), 209 spdk_pci_device_get_vendor_id(pci_dev), spdk_pci_device_get_device_id(pci_dev)); 210 211 return true; 212 } 213 214 static void 215 attach_cb(void *cb_ctx, struct spdk_pci_device *pci_dev, struct spdk_ioat_chan *ioat) 216 { 217 struct ioat_device *dev; 218 219 if (g_ioat_chan_num >= g_user_config.ioat_chan_num) { 220 return; 221 } 222 223 dev = spdk_dma_zmalloc(sizeof(*dev), 0, NULL); 224 if (dev == NULL) { 225 printf("Failed to allocate device struct\n"); 226 return; 227 } 228 229 dev->ioat = ioat; 230 g_ioat_chan_num++; 231 TAILQ_INSERT_TAIL(&g_devices, dev, tailq); 232 } 233 234 static int 235 ioat_init(void) 236 { 237 if (spdk_ioat_probe(NULL, probe_cb, attach_cb) != 0) { 238 fprintf(stderr, "ioat_probe() failed\n"); 239 return 1; 240 } 241 242 return 0; 243 } 244 245 static void 246 usage(char *program_name) 247 { 248 printf("%s options\n", program_name); 249 printf("\t[-h help message]\n"); 250 printf("\t[-c core mask for distributing I/O submission/completion work]\n"); 251 printf("\t[-q queue depth]\n"); 252 printf("\t[-n number of channels]\n"); 253 printf("\t[-o transfer size in bytes]\n"); 254 printf("\t[-t time in seconds]\n"); 255 printf("\t[-v verify copy result if this switch is on]\n"); 256 } 257 258 static int 259 parse_args(int argc, char **argv) 260 { 261 int op; 262 263 construct_user_config(&g_user_config); 264 while ((op = getopt(argc, argv, "c:hn:o:q:t:v")) != -1) { 265 switch (op) { 266 case 'o': 267 g_user_config.xfer_size_bytes = spdk_strtol(optarg, 10); 268 break; 269 case 'n': 270 g_user_config.ioat_chan_num = spdk_strtol(optarg, 10); 271 break; 272 case 'q': 273 g_user_config.queue_depth = spdk_strtol(optarg, 10); 274 break; 275 case 't': 276 g_user_config.time_in_sec = spdk_strtol(optarg, 10); 277 break; 278 case 'c': 279 g_user_config.core_mask = optarg; 280 break; 281 case 'v': 282 g_user_config.verify = true; 283 break; 284 case 'h': 285 usage(argv[0]); 286 exit(0); 287 default: 288 usage(argv[0]); 289 return 1; 290 } 291 } 292 if (g_user_config.xfer_size_bytes <= 0 || g_user_config.queue_depth <= 0 || 293 g_user_config.time_in_sec <= 0 || !g_user_config.core_mask || 294 g_user_config.ioat_chan_num <= 0) { 295 usage(argv[0]); 296 return 1; 297 } 298 299 return 0; 300 } 301 302 static void 303 drain_io(struct ioat_chan_entry *ioat_chan_entry) 304 { 305 spdk_ioat_flush(ioat_chan_entry->chan); 306 while (ioat_chan_entry->current_queue_depth > 0) { 307 spdk_ioat_process_events(ioat_chan_entry->chan); 308 } 309 } 310 311 static void 312 submit_single_xfer(struct ioat_chan_entry *ioat_chan_entry, struct ioat_task *ioat_task, void *dst, 313 void *src) 314 { 315 ioat_task->ioat_chan_entry = ioat_chan_entry; 316 ioat_task->src = src; 317 ioat_task->dst = dst; 318 319 spdk_ioat_build_copy(ioat_chan_entry->chan, ioat_task, ioat_done, dst, src, 320 g_user_config.xfer_size_bytes); 321 ioat_chan_entry->waiting_for_flush++; 322 if (ioat_chan_entry->waiting_for_flush >= ioat_chan_entry->flush_threshold) { 323 spdk_ioat_flush(ioat_chan_entry->chan); 324 ioat_chan_entry->waiting_for_flush = 0; 325 } 326 327 ioat_chan_entry->current_queue_depth++; 328 } 329 330 static int 331 submit_xfers(struct ioat_chan_entry *ioat_chan_entry, uint64_t queue_depth) 332 { 333 while (queue_depth-- > 0) { 334 void *src = NULL, *dst = NULL; 335 struct ioat_task *ioat_task = NULL; 336 337 src = spdk_mempool_get(ioat_chan_entry->data_pool); 338 dst = spdk_mempool_get(ioat_chan_entry->data_pool); 339 ioat_task = spdk_mempool_get(ioat_chan_entry->task_pool); 340 if (!ioat_task) { 341 fprintf(stderr, "Unable to get ioat_task\n"); 342 return 1; 343 } 344 345 submit_single_xfer(ioat_chan_entry, ioat_task, dst, src); 346 } 347 return 0; 348 } 349 350 static int 351 work_fn(void *arg) 352 { 353 uint64_t tsc_end; 354 struct worker_thread *worker = (struct worker_thread *)arg; 355 struct ioat_chan_entry *t = NULL; 356 357 printf("Starting thread on core %u\n", worker->core); 358 359 tsc_end = spdk_get_ticks() + g_user_config.time_in_sec * spdk_get_ticks_hz(); 360 361 t = worker->ctx; 362 while (t != NULL) { 363 /* begin to submit transfers */ 364 t->waiting_for_flush = 0; 365 t->flush_threshold = g_user_config.queue_depth / 2; 366 if (submit_xfers(t, g_user_config.queue_depth) != 0) { 367 return 1; 368 } 369 t = t->next; 370 } 371 372 while (1) { 373 t = worker->ctx; 374 while (t != NULL) { 375 spdk_ioat_process_events(t->chan); 376 t = t->next; 377 } 378 379 if (spdk_get_ticks() > tsc_end) { 380 break; 381 } 382 } 383 384 t = worker->ctx; 385 while (t != NULL) { 386 /* begin to drain io */ 387 t->is_draining = true; 388 drain_io(t); 389 t = t->next; 390 } 391 392 return 0; 393 } 394 395 static int 396 init(void) 397 { 398 struct spdk_env_opts opts; 399 400 spdk_env_opts_init(&opts); 401 opts.name = "ioat_perf"; 402 opts.core_mask = g_user_config.core_mask; 403 if (spdk_env_init(&opts) < 0) { 404 return 1; 405 } 406 407 return 0; 408 } 409 410 static int 411 dump_result(void) 412 { 413 uint64_t total_completed = 0; 414 uint64_t total_failed = 0; 415 uint64_t total_xfer_per_sec, total_bw_in_MiBps; 416 struct worker_thread *worker = g_workers; 417 418 printf("Channel_ID Core Transfers Bandwidth Failed\n"); 419 printf("-----------------------------------------------------------\n"); 420 while (worker != NULL) { 421 struct ioat_chan_entry *t = worker->ctx; 422 while (t) { 423 uint64_t xfer_per_sec = t->xfer_completed / g_user_config.time_in_sec; 424 uint64_t bw_in_MiBps = (t->xfer_completed * g_user_config.xfer_size_bytes) / 425 (g_user_config.time_in_sec * 1024 * 1024); 426 427 total_completed += t->xfer_completed; 428 total_failed += t->xfer_failed; 429 430 if (xfer_per_sec) { 431 printf("%10d%10d%12" PRIu64 "/s%8" PRIu64 " MiB/s%11" PRIu64 "\n", 432 t->ioat_chan_id, worker->core, xfer_per_sec, 433 bw_in_MiBps, t->xfer_failed); 434 } 435 t = t->next; 436 } 437 worker = worker->next; 438 } 439 440 total_xfer_per_sec = total_completed / g_user_config.time_in_sec; 441 total_bw_in_MiBps = (total_completed * g_user_config.xfer_size_bytes) / 442 (g_user_config.time_in_sec * 1024 * 1024); 443 444 printf("===========================================================\n"); 445 printf("Total:%26" PRIu64 "/s%8" PRIu64 " MiB/s%11" PRIu64 "\n", 446 total_xfer_per_sec, total_bw_in_MiBps, total_failed); 447 448 return total_failed ? 1 : 0; 449 } 450 451 static struct spdk_ioat_chan * 452 get_next_chan(void) 453 { 454 struct spdk_ioat_chan *chan; 455 456 if (g_next_device == NULL) { 457 return NULL; 458 } 459 460 chan = g_next_device->ioat; 461 462 g_next_device = TAILQ_NEXT(g_next_device, tailq); 463 464 return chan; 465 } 466 467 static int 468 associate_workers_with_chan(void) 469 { 470 struct spdk_ioat_chan *chan = get_next_chan(); 471 struct worker_thread *worker = g_workers; 472 struct ioat_chan_entry *t; 473 char buf_pool_name[30], task_pool_name[30]; 474 int i = 0; 475 476 while (chan != NULL) { 477 t = calloc(1, sizeof(struct ioat_chan_entry)); 478 if (!t) { 479 return 1; 480 } 481 482 t->ioat_chan_id = i; 483 snprintf(buf_pool_name, sizeof(buf_pool_name), "buf_pool_%d", i); 484 snprintf(task_pool_name, sizeof(task_pool_name), "task_pool_%d", i); 485 t->data_pool = spdk_mempool_create(buf_pool_name, 486 g_user_config.queue_depth * 2, /* src + dst */ 487 g_user_config.xfer_size_bytes, 488 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 489 SPDK_ENV_SOCKET_ID_ANY); 490 t->task_pool = spdk_mempool_create(task_pool_name, 491 g_user_config.queue_depth, 492 sizeof(struct ioat_task), 493 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 494 SPDK_ENV_SOCKET_ID_ANY); 495 if (!t->data_pool || !t->task_pool) { 496 fprintf(stderr, "Could not allocate buffer pool.\n"); 497 spdk_mempool_free(t->data_pool); 498 spdk_mempool_free(t->task_pool); 499 free(t); 500 return 1; 501 } 502 printf("Associating ioat_channel %d with core %d\n", i, worker->core); 503 t->chan = chan; 504 t->next = worker->ctx; 505 worker->ctx = t; 506 507 worker = worker->next; 508 if (worker == NULL) { 509 worker = g_workers; 510 } 511 512 chan = get_next_chan(); 513 i++; 514 } 515 516 return 0; 517 } 518 519 int 520 main(int argc, char **argv) 521 { 522 int rc; 523 struct worker_thread *worker, *main_worker; 524 unsigned main_core; 525 526 if (parse_args(argc, argv) != 0) { 527 return 1; 528 } 529 530 if (init() != 0) { 531 return 1; 532 } 533 534 if (register_workers() != 0) { 535 rc = 1; 536 goto cleanup; 537 } 538 539 if (ioat_init() != 0) { 540 rc = 1; 541 goto cleanup; 542 } 543 544 if (g_ioat_chan_num == 0) { 545 printf("No channels found\n"); 546 rc = 1; 547 goto cleanup; 548 } 549 550 if (g_user_config.ioat_chan_num > g_ioat_chan_num) { 551 printf("%d channels are requested, but only %d are found," 552 "so only test %d channels\n", g_user_config.ioat_chan_num, 553 g_ioat_chan_num, g_ioat_chan_num); 554 g_user_config.ioat_chan_num = g_ioat_chan_num; 555 } 556 557 g_next_device = TAILQ_FIRST(&g_devices); 558 dump_user_config(&g_user_config); 559 560 if (associate_workers_with_chan() != 0) { 561 rc = 1; 562 goto cleanup; 563 } 564 565 /* Launch all of the secondary workers */ 566 main_core = spdk_env_get_current_core(); 567 main_worker = NULL; 568 worker = g_workers; 569 while (worker != NULL) { 570 if (worker->core != main_core) { 571 spdk_env_thread_launch_pinned(worker->core, work_fn, worker); 572 } else { 573 assert(main_worker == NULL); 574 main_worker = worker; 575 } 576 worker = worker->next; 577 } 578 579 assert(main_worker != NULL); 580 rc = work_fn(main_worker); 581 if (rc != 0) { 582 goto cleanup; 583 } 584 585 spdk_env_thread_wait_all(); 586 587 rc = dump_result(); 588 589 cleanup: 590 unregister_workers(); 591 ioat_exit(); 592 593 return rc; 594 } 595