1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include "spdk/ioat.h" 37 #include "spdk/env.h" 38 #include "spdk/queue.h" 39 #include "spdk/string.h" 40 41 struct user_config { 42 int xfer_size_bytes; 43 int queue_depth; 44 int time_in_sec; 45 bool verify; 46 char *core_mask; 47 int ioat_chan_num; 48 }; 49 50 struct ioat_device { 51 struct spdk_ioat_chan *ioat; 52 TAILQ_ENTRY(ioat_device) tailq; 53 }; 54 55 static TAILQ_HEAD(, ioat_device) g_devices; 56 static struct ioat_device *g_next_device; 57 58 static struct user_config g_user_config; 59 60 struct ioat_chan_entry { 61 struct spdk_ioat_chan *chan; 62 int ioat_chan_id; 63 uint64_t xfer_completed; 64 uint64_t xfer_failed; 65 uint64_t current_queue_depth; 66 uint64_t waiting_for_flush; 67 uint64_t flush_threshold; 68 bool is_draining; 69 struct spdk_mempool *data_pool; 70 struct spdk_mempool *task_pool; 71 struct ioat_chan_entry *next; 72 }; 73 74 struct worker_thread { 75 struct ioat_chan_entry *ctx; 76 struct worker_thread *next; 77 unsigned core; 78 }; 79 80 struct ioat_task { 81 struct ioat_chan_entry *ioat_chan_entry; 82 void *src; 83 void *dst; 84 }; 85 86 static struct worker_thread *g_workers = NULL; 87 static int g_num_workers = 0; 88 static int g_ioat_chan_num = 0; 89 90 static void submit_single_xfer(struct ioat_chan_entry *ioat_chan_entry, struct ioat_task *ioat_task, 91 void *dst, void *src); 92 93 static void 94 construct_user_config(struct user_config *self) 95 { 96 self->xfer_size_bytes = 4096; 97 self->ioat_chan_num = 1; 98 self->queue_depth = 256; 99 self->time_in_sec = 10; 100 self->verify = false; 101 self->core_mask = "0x1"; 102 } 103 104 static void 105 dump_user_config(struct user_config *self) 106 { 107 printf("User configuration:\n"); 108 printf("Number of channels: %u\n", self->ioat_chan_num); 109 printf("Transfer size: %u bytes\n", self->xfer_size_bytes); 110 printf("Queue depth: %u\n", self->queue_depth); 111 printf("Run time: %u seconds\n", self->time_in_sec); 112 printf("Core mask: %s\n", self->core_mask); 113 printf("Verify: %s\n\n", self->verify ? "Yes" : "No"); 114 } 115 116 static void 117 ioat_exit(void) 118 { 119 struct ioat_device *dev; 120 121 while (!TAILQ_EMPTY(&g_devices)) { 122 dev = TAILQ_FIRST(&g_devices); 123 TAILQ_REMOVE(&g_devices, dev, tailq); 124 if (dev->ioat) { 125 spdk_ioat_detach(dev->ioat); 126 } 127 spdk_dma_free(dev); 128 } 129 } 130 131 static void 132 ioat_done(void *cb_arg) 133 { 134 struct ioat_task *ioat_task = (struct ioat_task *)cb_arg; 135 struct ioat_chan_entry *ioat_chan_entry = ioat_task->ioat_chan_entry; 136 137 if (g_user_config.verify && memcmp(ioat_task->src, ioat_task->dst, g_user_config.xfer_size_bytes)) { 138 ioat_chan_entry->xfer_failed++; 139 } else { 140 ioat_chan_entry->xfer_completed++; 141 } 142 143 ioat_chan_entry->current_queue_depth--; 144 145 if (ioat_chan_entry->is_draining) { 146 spdk_mempool_put(ioat_chan_entry->data_pool, ioat_task->src); 147 spdk_mempool_put(ioat_chan_entry->data_pool, ioat_task->dst); 148 spdk_mempool_put(ioat_chan_entry->task_pool, ioat_task); 149 } else { 150 submit_single_xfer(ioat_chan_entry, ioat_task, ioat_task->dst, ioat_task->src); 151 } 152 } 153 154 static int 155 register_workers(void) 156 { 157 uint32_t i; 158 struct worker_thread *worker; 159 160 g_workers = NULL; 161 g_num_workers = 0; 162 163 SPDK_ENV_FOREACH_CORE(i) { 164 worker = calloc(1, sizeof(*worker)); 165 if (worker == NULL) { 166 fprintf(stderr, "Unable to allocate worker\n"); 167 return -1; 168 } 169 170 worker->core = i; 171 worker->next = g_workers; 172 g_workers = worker; 173 g_num_workers++; 174 } 175 176 return 0; 177 } 178 179 static void 180 unregister_workers(void) 181 { 182 struct worker_thread *worker = g_workers; 183 struct ioat_chan_entry *entry, *entry1; 184 185 /* Free ioat_chan_entry and worker thread */ 186 while (worker) { 187 struct worker_thread *next_worker = worker->next; 188 entry = worker->ctx; 189 while (entry) { 190 entry1 = entry->next; 191 spdk_mempool_free(entry->data_pool); 192 spdk_mempool_free(entry->task_pool); 193 free(entry); 194 entry = entry1; 195 } 196 free(worker); 197 worker = next_worker; 198 } 199 } 200 201 static bool 202 probe_cb(void *cb_ctx, struct spdk_pci_device *pci_dev) 203 { 204 printf(" Found matching device at %04x:%02x:%02x.%x " 205 "vendor:0x%04x device:0x%04x\n", 206 spdk_pci_device_get_domain(pci_dev), 207 spdk_pci_device_get_bus(pci_dev), spdk_pci_device_get_dev(pci_dev), 208 spdk_pci_device_get_func(pci_dev), 209 spdk_pci_device_get_vendor_id(pci_dev), spdk_pci_device_get_device_id(pci_dev)); 210 211 return true; 212 } 213 214 static void 215 attach_cb(void *cb_ctx, struct spdk_pci_device *pci_dev, struct spdk_ioat_chan *ioat) 216 { 217 struct ioat_device *dev; 218 219 if (g_ioat_chan_num >= g_user_config.ioat_chan_num) { 220 return; 221 } 222 223 dev = spdk_dma_zmalloc(sizeof(*dev), 0, NULL); 224 if (dev == NULL) { 225 printf("Failed to allocate device struct\n"); 226 return; 227 } 228 229 dev->ioat = ioat; 230 g_ioat_chan_num++; 231 TAILQ_INSERT_TAIL(&g_devices, dev, tailq); 232 } 233 234 static int 235 ioat_init(void) 236 { 237 TAILQ_INIT(&g_devices); 238 239 if (spdk_ioat_probe(NULL, probe_cb, attach_cb) != 0) { 240 fprintf(stderr, "ioat_probe() failed\n"); 241 return 1; 242 } 243 244 return 0; 245 } 246 247 static void 248 usage(char *program_name) 249 { 250 printf("%s options\n", program_name); 251 printf("\t[-h help message]\n"); 252 printf("\t[-c core mask for distributing I/O submission/completion work]\n"); 253 printf("\t[-q queue depth]\n"); 254 printf("\t[-n number of channels]\n"); 255 printf("\t[-o transfer size in bytes]\n"); 256 printf("\t[-t time in seconds]\n"); 257 printf("\t[-v verify copy result if this switch is on]\n"); 258 } 259 260 static int 261 parse_args(int argc, char **argv) 262 { 263 int op; 264 265 construct_user_config(&g_user_config); 266 while ((op = getopt(argc, argv, "c:hn:o:q:t:v")) != -1) { 267 switch (op) { 268 case 'o': 269 g_user_config.xfer_size_bytes = spdk_strtol(optarg, 10); 270 break; 271 case 'n': 272 g_user_config.ioat_chan_num = spdk_strtol(optarg, 10); 273 break; 274 case 'q': 275 g_user_config.queue_depth = spdk_strtol(optarg, 10); 276 break; 277 case 't': 278 g_user_config.time_in_sec = spdk_strtol(optarg, 10); 279 break; 280 case 'c': 281 g_user_config.core_mask = optarg; 282 break; 283 case 'v': 284 g_user_config.verify = true; 285 break; 286 case 'h': 287 usage(argv[0]); 288 exit(0); 289 default: 290 usage(argv[0]); 291 return 1; 292 } 293 } 294 if (g_user_config.xfer_size_bytes <= 0 || g_user_config.queue_depth <= 0 || 295 g_user_config.time_in_sec <= 0 || !g_user_config.core_mask || 296 g_user_config.ioat_chan_num <= 0) { 297 usage(argv[0]); 298 return 1; 299 } 300 301 return 0; 302 } 303 304 static void 305 drain_io(struct ioat_chan_entry *ioat_chan_entry) 306 { 307 spdk_ioat_flush(ioat_chan_entry->chan); 308 while (ioat_chan_entry->current_queue_depth > 0) { 309 spdk_ioat_process_events(ioat_chan_entry->chan); 310 } 311 } 312 313 static void 314 submit_single_xfer(struct ioat_chan_entry *ioat_chan_entry, struct ioat_task *ioat_task, void *dst, 315 void *src) 316 { 317 ioat_task->ioat_chan_entry = ioat_chan_entry; 318 ioat_task->src = src; 319 ioat_task->dst = dst; 320 321 spdk_ioat_build_copy(ioat_chan_entry->chan, ioat_task, ioat_done, dst, src, 322 g_user_config.xfer_size_bytes); 323 ioat_chan_entry->waiting_for_flush++; 324 if (ioat_chan_entry->waiting_for_flush >= ioat_chan_entry->flush_threshold) { 325 spdk_ioat_flush(ioat_chan_entry->chan); 326 ioat_chan_entry->waiting_for_flush = 0; 327 } 328 329 ioat_chan_entry->current_queue_depth++; 330 } 331 332 static int 333 submit_xfers(struct ioat_chan_entry *ioat_chan_entry, uint64_t queue_depth) 334 { 335 while (queue_depth-- > 0) { 336 void *src = NULL, *dst = NULL; 337 struct ioat_task *ioat_task = NULL; 338 339 src = spdk_mempool_get(ioat_chan_entry->data_pool); 340 dst = spdk_mempool_get(ioat_chan_entry->data_pool); 341 ioat_task = spdk_mempool_get(ioat_chan_entry->task_pool); 342 if (!ioat_task) { 343 fprintf(stderr, "Unable to get ioat_task\n"); 344 return -1; 345 } 346 347 submit_single_xfer(ioat_chan_entry, ioat_task, dst, src); 348 } 349 return 0; 350 } 351 352 static int 353 work_fn(void *arg) 354 { 355 uint64_t tsc_end; 356 struct worker_thread *worker = (struct worker_thread *)arg; 357 struct ioat_chan_entry *t = NULL; 358 359 printf("Starting thread on core %u\n", worker->core); 360 361 tsc_end = spdk_get_ticks() + g_user_config.time_in_sec * spdk_get_ticks_hz(); 362 363 t = worker->ctx; 364 while (t != NULL) { 365 /* begin to submit transfers */ 366 t->waiting_for_flush = 0; 367 t->flush_threshold = g_user_config.queue_depth / 2; 368 if (submit_xfers(t, g_user_config.queue_depth) < 0) { 369 return -1; 370 } 371 t = t->next; 372 } 373 374 while (1) { 375 t = worker->ctx; 376 while (t != NULL) { 377 spdk_ioat_process_events(t->chan); 378 t = t->next; 379 } 380 381 if (spdk_get_ticks() > tsc_end) { 382 break; 383 } 384 } 385 386 t = worker->ctx; 387 while (t != NULL) { 388 /* begin to drain io */ 389 t->is_draining = true; 390 drain_io(t); 391 t = t->next; 392 } 393 394 return 0; 395 } 396 397 static int 398 init(void) 399 { 400 struct spdk_env_opts opts; 401 402 spdk_env_opts_init(&opts); 403 opts.name = "perf"; 404 opts.core_mask = g_user_config.core_mask; 405 if (spdk_env_init(&opts) < 0) { 406 return -1; 407 } 408 409 return 0; 410 } 411 412 static int 413 dump_result(void) 414 { 415 uint64_t total_completed = 0; 416 uint64_t total_failed = 0; 417 uint64_t total_xfer_per_sec, total_bw_in_MiBps; 418 struct worker_thread *worker = g_workers; 419 420 printf("Channel_ID Core Transfers Bandwidth Failed\n"); 421 printf("-----------------------------------------------------------\n"); 422 while (worker != NULL) { 423 struct ioat_chan_entry *t = worker->ctx; 424 while (t) { 425 uint64_t xfer_per_sec = t->xfer_completed / g_user_config.time_in_sec; 426 uint64_t bw_in_MiBps = (t->xfer_completed * g_user_config.xfer_size_bytes) / 427 (g_user_config.time_in_sec * 1024 * 1024); 428 429 total_completed += t->xfer_completed; 430 total_failed += t->xfer_failed; 431 432 if (xfer_per_sec) { 433 printf("%10d%10d%12" PRIu64 "/s%8" PRIu64 " MiB/s%11" PRIu64 "\n", 434 t->ioat_chan_id, worker->core, xfer_per_sec, 435 bw_in_MiBps, t->xfer_failed); 436 } 437 t = t->next; 438 } 439 worker = worker->next; 440 } 441 442 total_xfer_per_sec = total_completed / g_user_config.time_in_sec; 443 total_bw_in_MiBps = (total_completed * g_user_config.xfer_size_bytes) / 444 (g_user_config.time_in_sec * 1024 * 1024); 445 446 printf("===========================================================\n"); 447 printf("Total:%26" PRIu64 "/s%8" PRIu64 " MiB/s%11" PRIu64 "\n", 448 total_xfer_per_sec, total_bw_in_MiBps, total_failed); 449 450 return total_failed ? 1 : 0; 451 } 452 453 static struct spdk_ioat_chan * 454 get_next_chan(void) 455 { 456 struct spdk_ioat_chan *chan; 457 458 if (g_next_device == NULL) { 459 return NULL; 460 } 461 462 chan = g_next_device->ioat; 463 464 g_next_device = TAILQ_NEXT(g_next_device, tailq); 465 466 return chan; 467 } 468 469 static int 470 associate_workers_with_chan(void) 471 { 472 struct spdk_ioat_chan *chan = get_next_chan(); 473 struct worker_thread *worker = g_workers; 474 struct ioat_chan_entry *t; 475 char buf_pool_name[30], task_pool_name[30]; 476 int i = 0; 477 478 while (chan != NULL) { 479 t = calloc(1, sizeof(struct ioat_chan_entry)); 480 if (!t) { 481 return -1; 482 } 483 484 t->ioat_chan_id = i; 485 snprintf(buf_pool_name, sizeof(buf_pool_name), "buf_pool_%d", i); 486 snprintf(task_pool_name, sizeof(task_pool_name), "task_pool_%d", i); 487 t->data_pool = spdk_mempool_create(buf_pool_name, 488 g_user_config.queue_depth * 2, /* src + dst */ 489 g_user_config.xfer_size_bytes, 490 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 491 SPDK_ENV_SOCKET_ID_ANY); 492 t->task_pool = spdk_mempool_create(task_pool_name, 493 g_user_config.queue_depth, 494 sizeof(struct ioat_task), 495 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 496 SPDK_ENV_SOCKET_ID_ANY); 497 if (!t->data_pool || !t->task_pool) { 498 fprintf(stderr, "Could not allocate buffer pool.\n"); 499 spdk_mempool_free(t->data_pool); 500 spdk_mempool_free(t->task_pool); 501 free(t); 502 return 1; 503 } 504 printf("Associating ioat_channel %d with core %d\n", i, worker->core); 505 t->chan = chan; 506 t->next = worker->ctx; 507 worker->ctx = t; 508 509 worker = worker->next; 510 if (worker == NULL) { 511 worker = g_workers; 512 } 513 514 chan = get_next_chan(); 515 i++; 516 } 517 518 return 0; 519 } 520 521 int 522 main(int argc, char **argv) 523 { 524 int rc; 525 struct worker_thread *worker, *master_worker; 526 unsigned master_core; 527 528 if (parse_args(argc, argv) != 0) { 529 return 1; 530 } 531 532 if (init() != 0) { 533 return 1; 534 } 535 536 if (register_workers() != 0) { 537 rc = -1; 538 goto cleanup; 539 } 540 541 if (ioat_init() != 0) { 542 rc = -1; 543 goto cleanup; 544 } 545 546 if (g_ioat_chan_num == 0) { 547 printf("No channels found\n"); 548 rc = 0; 549 goto cleanup; 550 } 551 552 if (g_user_config.ioat_chan_num > g_ioat_chan_num) { 553 printf("%d channels are requested, but only %d are found," 554 "so only test %d channels\n", g_user_config.ioat_chan_num, 555 g_ioat_chan_num, g_ioat_chan_num); 556 g_user_config.ioat_chan_num = g_ioat_chan_num; 557 } 558 559 g_next_device = TAILQ_FIRST(&g_devices); 560 dump_user_config(&g_user_config); 561 562 if (associate_workers_with_chan() != 0) { 563 rc = -1; 564 goto cleanup; 565 } 566 567 /* Launch all of the slave workers */ 568 master_core = spdk_env_get_current_core(); 569 master_worker = NULL; 570 worker = g_workers; 571 while (worker != NULL) { 572 if (worker->core != master_core) { 573 spdk_env_thread_launch_pinned(worker->core, work_fn, worker); 574 } else { 575 assert(master_worker == NULL); 576 master_worker = worker; 577 } 578 worker = worker->next; 579 } 580 581 assert(master_worker != NULL); 582 rc = work_fn(master_worker); 583 if (rc < 0) { 584 goto cleanup; 585 } 586 587 spdk_env_thread_wait_all(); 588 589 rc = dump_result(); 590 591 cleanup: 592 unregister_workers(); 593 ioat_exit(); 594 595 return rc; 596 } 597