1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2015 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 8 #include "spdk/ioat.h" 9 #include "spdk/env.h" 10 #include "spdk/queue.h" 11 #include "spdk/string.h" 12 13 struct user_config { 14 int xfer_size_bytes; 15 int queue_depth; 16 int time_in_sec; 17 bool verify; 18 char *core_mask; 19 int ioat_chan_num; 20 }; 21 22 struct ioat_device { 23 struct spdk_ioat_chan *ioat; 24 TAILQ_ENTRY(ioat_device) tailq; 25 }; 26 27 static TAILQ_HEAD(, ioat_device) g_devices = TAILQ_HEAD_INITIALIZER(g_devices); 28 static struct ioat_device *g_next_device; 29 30 static struct user_config g_user_config; 31 32 struct ioat_chan_entry { 33 struct spdk_ioat_chan *chan; 34 int ioat_chan_id; 35 uint64_t xfer_completed; 36 uint64_t xfer_failed; 37 uint64_t current_queue_depth; 38 uint64_t waiting_for_flush; 39 uint64_t flush_threshold; 40 bool is_draining; 41 struct spdk_mempool *data_pool; 42 struct spdk_mempool *task_pool; 43 struct ioat_chan_entry *next; 44 }; 45 46 struct worker_thread { 47 struct ioat_chan_entry *ctx; 48 struct worker_thread *next; 49 unsigned core; 50 }; 51 52 struct ioat_task { 53 struct ioat_chan_entry *ioat_chan_entry; 54 void *src; 55 void *dst; 56 }; 57 58 static struct worker_thread *g_workers = NULL; 59 static int g_num_workers = 0; 60 static int g_ioat_chan_num = 0; 61 62 static void submit_single_xfer(struct ioat_chan_entry *ioat_chan_entry, struct ioat_task *ioat_task, 63 void *dst, void *src); 64 65 static void 66 construct_user_config(struct user_config *self) 67 { 68 self->xfer_size_bytes = 4096; 69 self->ioat_chan_num = 1; 70 self->queue_depth = 256; 71 self->time_in_sec = 10; 72 self->verify = false; 73 self->core_mask = "0x1"; 74 } 75 76 static void 77 dump_user_config(struct user_config *self) 78 { 79 printf("User configuration:\n"); 80 printf("Number of channels: %u\n", self->ioat_chan_num); 81 printf("Transfer size: %u bytes\n", self->xfer_size_bytes); 82 printf("Queue depth: %u\n", self->queue_depth); 83 printf("Run time: %u seconds\n", self->time_in_sec); 84 printf("Core mask: %s\n", self->core_mask); 85 printf("Verify: %s\n\n", self->verify ? "Yes" : "No"); 86 } 87 88 static void 89 ioat_exit(void) 90 { 91 struct ioat_device *dev; 92 93 while (!TAILQ_EMPTY(&g_devices)) { 94 dev = TAILQ_FIRST(&g_devices); 95 TAILQ_REMOVE(&g_devices, dev, tailq); 96 if (dev->ioat) { 97 spdk_ioat_detach(dev->ioat); 98 } 99 spdk_dma_free(dev); 100 } 101 } 102 103 static void 104 ioat_done(void *cb_arg) 105 { 106 struct ioat_task *ioat_task = (struct ioat_task *)cb_arg; 107 struct ioat_chan_entry *ioat_chan_entry = ioat_task->ioat_chan_entry; 108 109 if (g_user_config.verify && memcmp(ioat_task->src, ioat_task->dst, g_user_config.xfer_size_bytes)) { 110 ioat_chan_entry->xfer_failed++; 111 } else { 112 ioat_chan_entry->xfer_completed++; 113 } 114 115 ioat_chan_entry->current_queue_depth--; 116 117 if (ioat_chan_entry->is_draining) { 118 spdk_mempool_put(ioat_chan_entry->data_pool, ioat_task->src); 119 spdk_mempool_put(ioat_chan_entry->data_pool, ioat_task->dst); 120 spdk_mempool_put(ioat_chan_entry->task_pool, ioat_task); 121 } else { 122 submit_single_xfer(ioat_chan_entry, ioat_task, ioat_task->dst, ioat_task->src); 123 } 124 } 125 126 static int 127 register_workers(void) 128 { 129 uint32_t i; 130 struct worker_thread *worker; 131 132 g_workers = NULL; 133 g_num_workers = 0; 134 135 SPDK_ENV_FOREACH_CORE(i) { 136 worker = calloc(1, sizeof(*worker)); 137 if (worker == NULL) { 138 fprintf(stderr, "Unable to allocate worker\n"); 139 return 1; 140 } 141 142 worker->core = i; 143 worker->next = g_workers; 144 g_workers = worker; 145 g_num_workers++; 146 } 147 148 return 0; 149 } 150 151 static void 152 unregister_workers(void) 153 { 154 struct worker_thread *worker = g_workers; 155 struct ioat_chan_entry *entry, *entry1; 156 157 /* Free ioat_chan_entry and worker thread */ 158 while (worker) { 159 struct worker_thread *next_worker = worker->next; 160 entry = worker->ctx; 161 while (entry) { 162 entry1 = entry->next; 163 spdk_mempool_free(entry->data_pool); 164 spdk_mempool_free(entry->task_pool); 165 free(entry); 166 entry = entry1; 167 } 168 free(worker); 169 worker = next_worker; 170 } 171 } 172 173 static bool 174 probe_cb(void *cb_ctx, struct spdk_pci_device *pci_dev) 175 { 176 printf(" Found matching device at %04x:%02x:%02x.%x " 177 "vendor:0x%04x device:0x%04x\n", 178 spdk_pci_device_get_domain(pci_dev), 179 spdk_pci_device_get_bus(pci_dev), spdk_pci_device_get_dev(pci_dev), 180 spdk_pci_device_get_func(pci_dev), 181 spdk_pci_device_get_vendor_id(pci_dev), spdk_pci_device_get_device_id(pci_dev)); 182 183 return true; 184 } 185 186 static void 187 attach_cb(void *cb_ctx, struct spdk_pci_device *pci_dev, struct spdk_ioat_chan *ioat) 188 { 189 struct ioat_device *dev; 190 191 if (g_ioat_chan_num >= g_user_config.ioat_chan_num) { 192 return; 193 } 194 195 dev = spdk_dma_zmalloc(sizeof(*dev), 0, NULL); 196 if (dev == NULL) { 197 printf("Failed to allocate device struct\n"); 198 return; 199 } 200 201 dev->ioat = ioat; 202 g_ioat_chan_num++; 203 TAILQ_INSERT_TAIL(&g_devices, dev, tailq); 204 } 205 206 static int 207 ioat_init(void) 208 { 209 if (spdk_ioat_probe(NULL, probe_cb, attach_cb) != 0) { 210 fprintf(stderr, "ioat_probe() failed\n"); 211 return 1; 212 } 213 214 return 0; 215 } 216 217 static void 218 usage(char *program_name) 219 { 220 printf("%s options\n", program_name); 221 printf("\t[-h help message]\n"); 222 printf("\t[-c core mask for distributing I/O submission/completion work]\n"); 223 printf("\t[-q queue depth]\n"); 224 printf("\t[-n number of channels]\n"); 225 printf("\t[-o transfer size in bytes]\n"); 226 printf("\t[-t time in seconds]\n"); 227 printf("\t[-v verify copy result if this switch is on]\n"); 228 } 229 230 static int 231 parse_args(int argc, char **argv) 232 { 233 int op; 234 235 construct_user_config(&g_user_config); 236 while ((op = getopt(argc, argv, "c:hn:o:q:t:v")) != -1) { 237 switch (op) { 238 case 'o': 239 g_user_config.xfer_size_bytes = spdk_strtol(optarg, 10); 240 break; 241 case 'n': 242 g_user_config.ioat_chan_num = spdk_strtol(optarg, 10); 243 break; 244 case 'q': 245 g_user_config.queue_depth = spdk_strtol(optarg, 10); 246 break; 247 case 't': 248 g_user_config.time_in_sec = spdk_strtol(optarg, 10); 249 break; 250 case 'c': 251 g_user_config.core_mask = optarg; 252 break; 253 case 'v': 254 g_user_config.verify = true; 255 break; 256 case 'h': 257 usage(argv[0]); 258 exit(0); 259 default: 260 usage(argv[0]); 261 return 1; 262 } 263 } 264 if (g_user_config.xfer_size_bytes <= 0 || g_user_config.queue_depth <= 0 || 265 g_user_config.time_in_sec <= 0 || !g_user_config.core_mask || 266 g_user_config.ioat_chan_num <= 0) { 267 usage(argv[0]); 268 return 1; 269 } 270 271 return 0; 272 } 273 274 static void 275 drain_io(struct ioat_chan_entry *ioat_chan_entry) 276 { 277 spdk_ioat_flush(ioat_chan_entry->chan); 278 while (ioat_chan_entry->current_queue_depth > 0) { 279 spdk_ioat_process_events(ioat_chan_entry->chan); 280 } 281 } 282 283 static void 284 submit_single_xfer(struct ioat_chan_entry *ioat_chan_entry, struct ioat_task *ioat_task, void *dst, 285 void *src) 286 { 287 ioat_task->ioat_chan_entry = ioat_chan_entry; 288 ioat_task->src = src; 289 ioat_task->dst = dst; 290 291 spdk_ioat_build_copy(ioat_chan_entry->chan, ioat_task, ioat_done, dst, src, 292 g_user_config.xfer_size_bytes); 293 ioat_chan_entry->waiting_for_flush++; 294 if (ioat_chan_entry->waiting_for_flush >= ioat_chan_entry->flush_threshold) { 295 spdk_ioat_flush(ioat_chan_entry->chan); 296 ioat_chan_entry->waiting_for_flush = 0; 297 } 298 299 ioat_chan_entry->current_queue_depth++; 300 } 301 302 static int 303 submit_xfers(struct ioat_chan_entry *ioat_chan_entry, uint64_t queue_depth) 304 { 305 while (queue_depth-- > 0) { 306 void *src = NULL, *dst = NULL; 307 struct ioat_task *ioat_task = NULL; 308 309 src = spdk_mempool_get(ioat_chan_entry->data_pool); 310 dst = spdk_mempool_get(ioat_chan_entry->data_pool); 311 ioat_task = spdk_mempool_get(ioat_chan_entry->task_pool); 312 if (!ioat_task) { 313 fprintf(stderr, "Unable to get ioat_task\n"); 314 return 1; 315 } 316 317 submit_single_xfer(ioat_chan_entry, ioat_task, dst, src); 318 } 319 return 0; 320 } 321 322 static int 323 work_fn(void *arg) 324 { 325 uint64_t tsc_end; 326 struct worker_thread *worker = (struct worker_thread *)arg; 327 struct ioat_chan_entry *t = NULL; 328 329 printf("Starting thread on core %u\n", worker->core); 330 331 tsc_end = spdk_get_ticks() + g_user_config.time_in_sec * spdk_get_ticks_hz(); 332 333 t = worker->ctx; 334 while (t != NULL) { 335 /* begin to submit transfers */ 336 t->waiting_for_flush = 0; 337 t->flush_threshold = g_user_config.queue_depth / 2; 338 if (submit_xfers(t, g_user_config.queue_depth) != 0) { 339 return 1; 340 } 341 t = t->next; 342 } 343 344 while (1) { 345 t = worker->ctx; 346 while (t != NULL) { 347 spdk_ioat_process_events(t->chan); 348 t = t->next; 349 } 350 351 if (spdk_get_ticks() > tsc_end) { 352 break; 353 } 354 } 355 356 t = worker->ctx; 357 while (t != NULL) { 358 /* begin to drain io */ 359 t->is_draining = true; 360 drain_io(t); 361 t = t->next; 362 } 363 364 return 0; 365 } 366 367 static int 368 init(void) 369 { 370 struct spdk_env_opts opts; 371 372 opts.opts_size = sizeof(opts); 373 spdk_env_opts_init(&opts); 374 opts.name = "ioat_perf"; 375 opts.core_mask = g_user_config.core_mask; 376 if (spdk_env_init(&opts) < 0) { 377 return 1; 378 } 379 380 return 0; 381 } 382 383 static int 384 dump_result(void) 385 { 386 uint64_t total_completed = 0; 387 uint64_t total_failed = 0; 388 uint64_t total_xfer_per_sec, total_bw_in_MiBps; 389 struct worker_thread *worker = g_workers; 390 391 printf("Channel_ID Core Transfers Bandwidth Failed\n"); 392 printf("-----------------------------------------------------------\n"); 393 while (worker != NULL) { 394 struct ioat_chan_entry *t = worker->ctx; 395 while (t) { 396 uint64_t xfer_per_sec = t->xfer_completed / g_user_config.time_in_sec; 397 uint64_t bw_in_MiBps = (t->xfer_completed * g_user_config.xfer_size_bytes) / 398 (g_user_config.time_in_sec * 1024 * 1024); 399 400 total_completed += t->xfer_completed; 401 total_failed += t->xfer_failed; 402 403 if (xfer_per_sec) { 404 printf("%10d%10d%12" PRIu64 "/s%8" PRIu64 " MiB/s%11" PRIu64 "\n", 405 t->ioat_chan_id, worker->core, xfer_per_sec, 406 bw_in_MiBps, t->xfer_failed); 407 } 408 t = t->next; 409 } 410 worker = worker->next; 411 } 412 413 total_xfer_per_sec = total_completed / g_user_config.time_in_sec; 414 total_bw_in_MiBps = (total_completed * g_user_config.xfer_size_bytes) / 415 (g_user_config.time_in_sec * 1024 * 1024); 416 417 printf("===========================================================\n"); 418 printf("Total:%26" PRIu64 "/s%8" PRIu64 " MiB/s%11" PRIu64 "\n", 419 total_xfer_per_sec, total_bw_in_MiBps, total_failed); 420 421 return total_failed ? 1 : 0; 422 } 423 424 static struct spdk_ioat_chan * 425 get_next_chan(void) 426 { 427 struct spdk_ioat_chan *chan; 428 429 if (g_next_device == NULL) { 430 return NULL; 431 } 432 433 chan = g_next_device->ioat; 434 435 g_next_device = TAILQ_NEXT(g_next_device, tailq); 436 437 return chan; 438 } 439 440 static int 441 associate_workers_with_chan(void) 442 { 443 struct spdk_ioat_chan *chan = get_next_chan(); 444 struct worker_thread *worker = g_workers; 445 struct ioat_chan_entry *t; 446 char buf_pool_name[30], task_pool_name[30]; 447 int i = 0; 448 449 while (chan != NULL) { 450 t = calloc(1, sizeof(struct ioat_chan_entry)); 451 if (!t) { 452 return 1; 453 } 454 455 t->ioat_chan_id = i; 456 snprintf(buf_pool_name, sizeof(buf_pool_name), "buf_pool_%d", i); 457 snprintf(task_pool_name, sizeof(task_pool_name), "task_pool_%d", i); 458 t->data_pool = spdk_mempool_create(buf_pool_name, 459 g_user_config.queue_depth * 2, /* src + dst */ 460 g_user_config.xfer_size_bytes, 461 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 462 SPDK_ENV_SOCKET_ID_ANY); 463 t->task_pool = spdk_mempool_create(task_pool_name, 464 g_user_config.queue_depth, 465 sizeof(struct ioat_task), 466 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 467 SPDK_ENV_SOCKET_ID_ANY); 468 if (!t->data_pool || !t->task_pool) { 469 fprintf(stderr, "Could not allocate buffer pool.\n"); 470 spdk_mempool_free(t->data_pool); 471 spdk_mempool_free(t->task_pool); 472 free(t); 473 return 1; 474 } 475 printf("Associating ioat_channel %d with core %d\n", i, worker->core); 476 t->chan = chan; 477 t->next = worker->ctx; 478 worker->ctx = t; 479 480 worker = worker->next; 481 if (worker == NULL) { 482 worker = g_workers; 483 } 484 485 chan = get_next_chan(); 486 i++; 487 } 488 489 return 0; 490 } 491 492 int 493 main(int argc, char **argv) 494 { 495 int rc; 496 struct worker_thread *worker, *main_worker; 497 unsigned main_core; 498 499 if (parse_args(argc, argv) != 0) { 500 return 1; 501 } 502 503 if (init() != 0) { 504 return 1; 505 } 506 507 if (register_workers() != 0) { 508 rc = 1; 509 goto cleanup; 510 } 511 512 if (ioat_init() != 0) { 513 rc = 1; 514 goto cleanup; 515 } 516 517 if (g_ioat_chan_num == 0) { 518 printf("No channels found\n"); 519 rc = 1; 520 goto cleanup; 521 } 522 523 if (g_user_config.ioat_chan_num > g_ioat_chan_num) { 524 printf("%d channels are requested, but only %d are found," 525 "so only test %d channels\n", g_user_config.ioat_chan_num, 526 g_ioat_chan_num, g_ioat_chan_num); 527 g_user_config.ioat_chan_num = g_ioat_chan_num; 528 } 529 530 g_next_device = TAILQ_FIRST(&g_devices); 531 dump_user_config(&g_user_config); 532 533 if (associate_workers_with_chan() != 0) { 534 rc = 1; 535 goto cleanup; 536 } 537 538 /* Launch all of the secondary workers */ 539 main_core = spdk_env_get_current_core(); 540 main_worker = NULL; 541 worker = g_workers; 542 while (worker != NULL) { 543 if (worker->core != main_core) { 544 spdk_env_thread_launch_pinned(worker->core, work_fn, worker); 545 } else { 546 assert(main_worker == NULL); 547 main_worker = worker; 548 } 549 worker = worker->next; 550 } 551 552 assert(main_worker != NULL); 553 rc = work_fn(main_worker); 554 if (rc != 0) { 555 goto cleanup; 556 } 557 558 spdk_env_thread_wait_all(); 559 560 rc = dump_result(); 561 562 cleanup: 563 unregister_workers(); 564 ioat_exit(); 565 566 spdk_env_fini(); 567 return rc; 568 } 569