1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include "spdk/ioat.h" 37 #include "spdk/env.h" 38 #include "spdk/queue.h" 39 #include "spdk/string.h" 40 41 struct user_config { 42 int xfer_size_bytes; 43 int queue_depth; 44 int time_in_sec; 45 bool verify; 46 char *core_mask; 47 int ioat_chan_num; 48 }; 49 50 struct ioat_device { 51 struct spdk_ioat_chan *ioat; 52 TAILQ_ENTRY(ioat_device) tailq; 53 }; 54 55 static TAILQ_HEAD(, ioat_device) g_devices; 56 static struct ioat_device *g_next_device; 57 58 static struct user_config g_user_config; 59 60 struct ioat_chan_entry { 61 struct spdk_ioat_chan *chan; 62 int ioat_chan_id; 63 uint64_t xfer_completed; 64 uint64_t xfer_failed; 65 uint64_t current_queue_depth; 66 bool is_draining; 67 struct spdk_mempool *data_pool; 68 struct spdk_mempool *task_pool; 69 struct ioat_chan_entry *next; 70 }; 71 72 struct worker_thread { 73 struct ioat_chan_entry *ctx; 74 struct worker_thread *next; 75 unsigned core; 76 }; 77 78 struct ioat_task { 79 struct ioat_chan_entry *ioat_chan_entry; 80 void *src; 81 void *dst; 82 }; 83 84 static struct worker_thread *g_workers = NULL; 85 static int g_num_workers = 0; 86 static int g_ioat_chan_num = 0; 87 88 static void submit_single_xfer(struct ioat_chan_entry *ioat_chan_entry, struct ioat_task *ioat_task, 89 void *dst, void *src); 90 91 static void 92 construct_user_config(struct user_config *self) 93 { 94 self->xfer_size_bytes = 4096; 95 self->ioat_chan_num = 1; 96 self->queue_depth = 256; 97 self->time_in_sec = 10; 98 self->verify = false; 99 self->core_mask = "0x1"; 100 } 101 102 static void 103 dump_user_config(struct user_config *self) 104 { 105 printf("User configuration:\n"); 106 printf("Number of channels: %u\n", self->ioat_chan_num); 107 printf("Transfer size: %u bytes\n", self->xfer_size_bytes); 108 printf("Queue depth: %u\n", self->queue_depth); 109 printf("Run time: %u seconds\n", self->time_in_sec); 110 printf("Core mask: %s\n", self->core_mask); 111 printf("Verify: %s\n\n", self->verify ? "Yes" : "No"); 112 } 113 114 static void 115 ioat_exit(void) 116 { 117 struct ioat_device *dev; 118 119 while (!TAILQ_EMPTY(&g_devices)) { 120 dev = TAILQ_FIRST(&g_devices); 121 TAILQ_REMOVE(&g_devices, dev, tailq); 122 if (dev->ioat) { 123 spdk_ioat_detach(dev->ioat); 124 } 125 spdk_dma_free(dev); 126 } 127 } 128 129 static void 130 ioat_done(void *cb_arg) 131 { 132 struct ioat_task *ioat_task = (struct ioat_task *)cb_arg; 133 struct ioat_chan_entry *ioat_chan_entry = ioat_task->ioat_chan_entry; 134 135 if (g_user_config.verify && memcmp(ioat_task->src, ioat_task->dst, g_user_config.xfer_size_bytes)) { 136 ioat_chan_entry->xfer_failed++; 137 } else { 138 ioat_chan_entry->xfer_completed++; 139 } 140 141 ioat_chan_entry->current_queue_depth--; 142 143 if (ioat_chan_entry->is_draining) { 144 spdk_mempool_put(ioat_chan_entry->data_pool, ioat_task->src); 145 spdk_mempool_put(ioat_chan_entry->data_pool, ioat_task->dst); 146 spdk_mempool_put(ioat_chan_entry->task_pool, ioat_task); 147 } else { 148 submit_single_xfer(ioat_chan_entry, ioat_task, ioat_task->dst, ioat_task->src); 149 } 150 } 151 152 static int 153 register_workers(void) 154 { 155 uint32_t i; 156 struct worker_thread *worker; 157 158 g_workers = NULL; 159 g_num_workers = 0; 160 161 SPDK_ENV_FOREACH_CORE(i) { 162 worker = calloc(1, sizeof(*worker)); 163 if (worker == NULL) { 164 fprintf(stderr, "Unable to allocate worker\n"); 165 return -1; 166 } 167 168 worker->core = i; 169 worker->next = g_workers; 170 g_workers = worker; 171 g_num_workers++; 172 } 173 174 return 0; 175 } 176 177 static void 178 unregister_workers(void) 179 { 180 struct worker_thread *worker = g_workers; 181 struct ioat_chan_entry *entry, *entry1; 182 183 /* Free ioat_chan_entry and worker thread */ 184 while (worker) { 185 struct worker_thread *next_worker = worker->next; 186 entry = worker->ctx; 187 while (entry) { 188 entry1 = entry->next; 189 spdk_mempool_free(entry->data_pool); 190 spdk_mempool_free(entry->task_pool); 191 free(entry); 192 entry = entry1; 193 } 194 free(worker); 195 worker = next_worker; 196 } 197 } 198 199 static bool 200 probe_cb(void *cb_ctx, struct spdk_pci_device *pci_dev) 201 { 202 printf(" Found matching device at %04x:%02x:%02x.%x " 203 "vendor:0x%04x device:0x%04x\n", 204 spdk_pci_device_get_domain(pci_dev), 205 spdk_pci_device_get_bus(pci_dev), spdk_pci_device_get_dev(pci_dev), 206 spdk_pci_device_get_func(pci_dev), 207 spdk_pci_device_get_vendor_id(pci_dev), spdk_pci_device_get_device_id(pci_dev)); 208 209 return true; 210 } 211 212 static void 213 attach_cb(void *cb_ctx, struct spdk_pci_device *pci_dev, struct spdk_ioat_chan *ioat) 214 { 215 struct ioat_device *dev; 216 217 if (g_ioat_chan_num >= g_user_config.ioat_chan_num) { 218 return; 219 } 220 221 dev = spdk_dma_zmalloc(sizeof(*dev), 0, NULL); 222 if (dev == NULL) { 223 printf("Failed to allocate device struct\n"); 224 return; 225 } 226 227 dev->ioat = ioat; 228 g_ioat_chan_num++; 229 TAILQ_INSERT_TAIL(&g_devices, dev, tailq); 230 } 231 232 static int 233 ioat_init(void) 234 { 235 TAILQ_INIT(&g_devices); 236 237 if (spdk_ioat_probe(NULL, probe_cb, attach_cb) != 0) { 238 fprintf(stderr, "ioat_probe() failed\n"); 239 return 1; 240 } 241 242 return 0; 243 } 244 245 static void 246 usage(char *program_name) 247 { 248 printf("%s options\n", program_name); 249 printf("\t[-h help message]\n"); 250 printf("\t[-c core mask for distributing I/O submission/completion work]\n"); 251 printf("\t[-q queue depth]\n"); 252 printf("\t[-n number of channels]\n"); 253 printf("\t[-s transfer size in bytes]\n"); 254 printf("\t[-t time in seconds]\n"); 255 printf("\t[-v verify copy result if this switch is on]\n"); 256 } 257 258 static int 259 parse_args(int argc, char **argv) 260 { 261 int op; 262 263 construct_user_config(&g_user_config); 264 while ((op = getopt(argc, argv, "c:hn:q:s:t:v")) != -1) { 265 switch (op) { 266 case 's': 267 g_user_config.xfer_size_bytes = atoi(optarg); 268 break; 269 case 'n': 270 g_user_config.ioat_chan_num = atoi(optarg); 271 break; 272 case 'q': 273 g_user_config.queue_depth = atoi(optarg); 274 break; 275 case 't': 276 g_user_config.time_in_sec = atoi(optarg); 277 break; 278 case 'c': 279 g_user_config.core_mask = optarg; 280 break; 281 case 'v': 282 g_user_config.verify = true; 283 break; 284 case 'h': 285 usage(argv[0]); 286 exit(0); 287 default: 288 usage(argv[0]); 289 return 1; 290 } 291 } 292 if (!g_user_config.xfer_size_bytes || !g_user_config.queue_depth || 293 !g_user_config.time_in_sec || !g_user_config.core_mask || 294 !g_user_config.ioat_chan_num) { 295 usage(argv[0]); 296 return 1; 297 } 298 299 return 0; 300 } 301 302 static void 303 drain_io(struct ioat_chan_entry *ioat_chan_entry) 304 { 305 while (ioat_chan_entry->current_queue_depth > 0) { 306 spdk_ioat_process_events(ioat_chan_entry->chan); 307 } 308 } 309 310 static void 311 submit_single_xfer(struct ioat_chan_entry *ioat_chan_entry, struct ioat_task *ioat_task, void *dst, 312 void *src) 313 { 314 ioat_task->ioat_chan_entry = ioat_chan_entry; 315 ioat_task->src = src; 316 ioat_task->dst = dst; 317 318 spdk_ioat_submit_copy(ioat_chan_entry->chan, ioat_task, ioat_done, dst, src, 319 g_user_config.xfer_size_bytes); 320 321 ioat_chan_entry->current_queue_depth++; 322 } 323 324 static void 325 submit_xfers(struct ioat_chan_entry *ioat_chan_entry, uint64_t queue_depth) 326 { 327 while (queue_depth-- > 0) { 328 void *src = NULL, *dst = NULL; 329 struct ioat_task *ioat_task = NULL; 330 331 src = spdk_mempool_get(ioat_chan_entry->data_pool); 332 dst = spdk_mempool_get(ioat_chan_entry->data_pool); 333 ioat_task = spdk_mempool_get(ioat_chan_entry->task_pool); 334 335 submit_single_xfer(ioat_chan_entry, ioat_task, dst, src); 336 } 337 } 338 339 static int 340 work_fn(void *arg) 341 { 342 uint64_t tsc_end; 343 struct worker_thread *worker = (struct worker_thread *)arg; 344 struct ioat_chan_entry *t = NULL; 345 346 printf("Starting thread on core %u\n", worker->core); 347 348 tsc_end = spdk_get_ticks() + g_user_config.time_in_sec * spdk_get_ticks_hz(); 349 350 t = worker->ctx; 351 while (t != NULL) { 352 // begin to submit transfers 353 submit_xfers(t, g_user_config.queue_depth); 354 t = t->next; 355 } 356 357 while (1) { 358 t = worker->ctx; 359 while (t != NULL) { 360 spdk_ioat_process_events(t->chan); 361 t = t->next; 362 } 363 364 if (spdk_get_ticks() > tsc_end) { 365 break; 366 } 367 } 368 369 t = worker->ctx; 370 while (t != NULL) { 371 // begin to drain io 372 t->is_draining = true; 373 drain_io(t); 374 t = t->next; 375 } 376 377 return 0; 378 } 379 380 static int 381 init(void) 382 { 383 struct spdk_env_opts opts; 384 385 spdk_env_opts_init(&opts); 386 opts.name = "perf"; 387 opts.core_mask = g_user_config.core_mask; 388 if (spdk_env_init(&opts) < 0) { 389 return -1; 390 } 391 392 return 0; 393 } 394 395 static int 396 dump_result(void) 397 { 398 uint64_t total_completed = 0; 399 uint64_t total_failed = 0; 400 uint64_t total_xfer_per_sec, total_bw_in_MiBps; 401 struct worker_thread *worker = g_workers; 402 403 printf("Channel_ID Core Transfers Bandwidth Failed\n"); 404 printf("-----------------------------------------------------------\n"); 405 while (worker != NULL) { 406 struct ioat_chan_entry *t = worker->ctx; 407 while (t) { 408 uint64_t xfer_per_sec = t->xfer_completed / g_user_config.time_in_sec; 409 uint64_t bw_in_MiBps = (t->xfer_completed * g_user_config.xfer_size_bytes) / 410 (g_user_config.time_in_sec * 1024 * 1024); 411 412 total_completed += t->xfer_completed; 413 total_failed += t->xfer_failed; 414 415 if (xfer_per_sec) { 416 printf("%10d%10d%12" PRIu64 "/s%8" PRIu64 " MiB/s%11" PRIu64 "\n", 417 t->ioat_chan_id, worker->core, xfer_per_sec, 418 bw_in_MiBps, t->xfer_failed); 419 } 420 t = t->next; 421 } 422 worker = worker->next; 423 } 424 425 total_xfer_per_sec = total_completed / g_user_config.time_in_sec; 426 total_bw_in_MiBps = (total_completed * g_user_config.xfer_size_bytes) / 427 (g_user_config.time_in_sec * 1024 * 1024); 428 429 printf("===========================================================\n"); 430 printf("Total:%26" PRIu64 "/s%8" PRIu64 " MiB/s%11" PRIu64 "\n", 431 total_xfer_per_sec, total_bw_in_MiBps, total_failed); 432 433 return total_failed ? 1 : 0; 434 } 435 436 static struct spdk_ioat_chan * 437 get_next_chan(void) 438 { 439 struct spdk_ioat_chan *chan; 440 441 if (g_next_device == NULL) { 442 return NULL; 443 } 444 445 chan = g_next_device->ioat; 446 447 g_next_device = TAILQ_NEXT(g_next_device, tailq); 448 449 return chan; 450 } 451 452 static int 453 associate_workers_with_chan(void) 454 { 455 struct spdk_ioat_chan *chan = get_next_chan(); 456 struct worker_thread *worker = g_workers; 457 struct ioat_chan_entry *t; 458 char buf_pool_name[30], task_pool_name[30]; 459 int i = 0; 460 461 while (chan != NULL) { 462 t = calloc(1, sizeof(struct ioat_chan_entry)); 463 if (!t) { 464 return -1; 465 } 466 467 t->ioat_chan_id = i; 468 snprintf(buf_pool_name, sizeof(buf_pool_name), "buf_pool_%d", i); 469 snprintf(task_pool_name, sizeof(task_pool_name), "task_pool_%d", i); 470 t->data_pool = spdk_mempool_create(buf_pool_name, 512, g_user_config.xfer_size_bytes, 471 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 472 SPDK_ENV_SOCKET_ID_ANY); 473 t->task_pool = spdk_mempool_create(task_pool_name, 512, sizeof(struct ioat_task), 474 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 475 SPDK_ENV_SOCKET_ID_ANY); 476 if (!t->data_pool || !t->task_pool) { 477 fprintf(stderr, "Could not allocate buffer pool.\n"); 478 spdk_mempool_free(t->data_pool); 479 spdk_mempool_free(t->task_pool); 480 free(t); 481 return 1; 482 } 483 printf("Associating ioat_channel %d with core %d\n", i, worker->core); 484 t->chan = chan; 485 t->next = worker->ctx; 486 worker->ctx = t; 487 488 worker = worker->next; 489 if (worker == NULL) { 490 worker = g_workers; 491 } 492 493 chan = get_next_chan(); 494 i++; 495 } 496 497 return 0; 498 } 499 500 int 501 main(int argc, char **argv) 502 { 503 int rc; 504 struct worker_thread *worker, *master_worker; 505 unsigned master_core; 506 507 if (parse_args(argc, argv) != 0) { 508 return 1; 509 } 510 511 if (init() != 0) { 512 return 1; 513 } 514 515 if (register_workers() != 0) { 516 rc = -1; 517 goto cleanup; 518 } 519 520 if (ioat_init() != 0) { 521 rc = -1; 522 goto cleanup; 523 } 524 525 if (g_ioat_chan_num == 0) { 526 printf("No channels found\n"); 527 rc = 0; 528 goto cleanup; 529 } 530 531 if (g_user_config.ioat_chan_num > g_ioat_chan_num) { 532 printf("%d channels are requested, but only %d are found," 533 "so only test %d channels\n", g_user_config.ioat_chan_num, 534 g_ioat_chan_num, g_ioat_chan_num); 535 g_user_config.ioat_chan_num = g_ioat_chan_num; 536 } 537 538 g_next_device = TAILQ_FIRST(&g_devices); 539 dump_user_config(&g_user_config); 540 541 if (associate_workers_with_chan() != 0) { 542 rc = -1; 543 goto cleanup; 544 } 545 546 /* Launch all of the slave workers */ 547 master_core = spdk_env_get_current_core(); 548 master_worker = NULL; 549 worker = g_workers; 550 while (worker != NULL) { 551 if (worker->core != master_core) { 552 spdk_env_thread_launch_pinned(worker->core, work_fn, worker); 553 } else { 554 assert(master_worker == NULL); 555 master_worker = worker; 556 } 557 worker = worker->next; 558 } 559 560 assert(master_worker != NULL); 561 rc = work_fn(master_worker); 562 if (rc < 0) { 563 goto cleanup; 564 } 565 566 spdk_env_thread_wait_all(); 567 568 rc = dump_result(); 569 570 cleanup: 571 unregister_workers(); 572 ioat_exit(); 573 574 return rc; 575 } 576