1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 8 #include "spdk/ioat.h" 9 #include "spdk/env.h" 10 #include "spdk/queue.h" 11 #include "spdk/string.h" 12 13 struct user_config { 14 int xfer_size_bytes; 15 int queue_depth; 16 int time_in_sec; 17 bool verify; 18 char *core_mask; 19 int ioat_chan_num; 20 }; 21 22 struct ioat_device { 23 struct spdk_ioat_chan *ioat; 24 TAILQ_ENTRY(ioat_device) tailq; 25 }; 26 27 static TAILQ_HEAD(, ioat_device) g_devices = TAILQ_HEAD_INITIALIZER(g_devices); 28 static struct ioat_device *g_next_device; 29 30 static struct user_config g_user_config; 31 32 struct ioat_chan_entry { 33 struct spdk_ioat_chan *chan; 34 int ioat_chan_id; 35 uint64_t xfer_completed; 36 uint64_t xfer_failed; 37 uint64_t current_queue_depth; 38 uint64_t waiting_for_flush; 39 uint64_t flush_threshold; 40 bool is_draining; 41 struct spdk_mempool *data_pool; 42 struct spdk_mempool *task_pool; 43 struct ioat_chan_entry *next; 44 }; 45 46 struct worker_thread { 47 struct ioat_chan_entry *ctx; 48 struct worker_thread *next; 49 unsigned core; 50 }; 51 52 struct ioat_task { 53 struct ioat_chan_entry *ioat_chan_entry; 54 void *src; 55 void *dst; 56 }; 57 58 static struct worker_thread *g_workers = NULL; 59 static int g_num_workers = 0; 60 static int g_ioat_chan_num = 0; 61 62 static void submit_single_xfer(struct ioat_chan_entry *ioat_chan_entry, struct ioat_task *ioat_task, 63 void *dst, void *src); 64 65 static void 66 construct_user_config(struct user_config *self) 67 { 68 self->xfer_size_bytes = 4096; 69 self->ioat_chan_num = 1; 70 self->queue_depth = 256; 71 self->time_in_sec = 10; 72 self->verify = false; 73 self->core_mask = "0x1"; 74 } 75 76 static void 77 dump_user_config(struct user_config *self) 78 { 79 printf("User configuration:\n"); 80 printf("Number of channels: %u\n", self->ioat_chan_num); 81 printf("Transfer size: %u bytes\n", self->xfer_size_bytes); 82 printf("Queue depth: %u\n", self->queue_depth); 83 printf("Run time: %u seconds\n", self->time_in_sec); 84 printf("Core mask: %s\n", self->core_mask); 85 printf("Verify: %s\n\n", self->verify ? "Yes" : "No"); 86 } 87 88 static void 89 ioat_exit(void) 90 { 91 struct ioat_device *dev; 92 93 while (!TAILQ_EMPTY(&g_devices)) { 94 dev = TAILQ_FIRST(&g_devices); 95 TAILQ_REMOVE(&g_devices, dev, tailq); 96 if (dev->ioat) { 97 spdk_ioat_detach(dev->ioat); 98 } 99 spdk_dma_free(dev); 100 } 101 } 102 103 static void 104 ioat_done(void *cb_arg) 105 { 106 struct ioat_task *ioat_task = (struct ioat_task *)cb_arg; 107 struct ioat_chan_entry *ioat_chan_entry = ioat_task->ioat_chan_entry; 108 109 if (g_user_config.verify && memcmp(ioat_task->src, ioat_task->dst, g_user_config.xfer_size_bytes)) { 110 ioat_chan_entry->xfer_failed++; 111 } else { 112 ioat_chan_entry->xfer_completed++; 113 } 114 115 ioat_chan_entry->current_queue_depth--; 116 117 if (ioat_chan_entry->is_draining) { 118 spdk_mempool_put(ioat_chan_entry->data_pool, ioat_task->src); 119 spdk_mempool_put(ioat_chan_entry->data_pool, ioat_task->dst); 120 spdk_mempool_put(ioat_chan_entry->task_pool, ioat_task); 121 } else { 122 submit_single_xfer(ioat_chan_entry, ioat_task, ioat_task->dst, ioat_task->src); 123 } 124 } 125 126 static int 127 register_workers(void) 128 { 129 uint32_t i; 130 struct worker_thread *worker; 131 132 g_workers = NULL; 133 g_num_workers = 0; 134 135 SPDK_ENV_FOREACH_CORE(i) { 136 worker = calloc(1, sizeof(*worker)); 137 if (worker == NULL) { 138 fprintf(stderr, "Unable to allocate worker\n"); 139 return 1; 140 } 141 142 worker->core = i; 143 worker->next = g_workers; 144 g_workers = worker; 145 g_num_workers++; 146 } 147 148 return 0; 149 } 150 151 static void 152 unregister_workers(void) 153 { 154 struct worker_thread *worker = g_workers; 155 struct ioat_chan_entry *entry, *entry1; 156 157 /* Free ioat_chan_entry and worker thread */ 158 while (worker) { 159 struct worker_thread *next_worker = worker->next; 160 entry = worker->ctx; 161 while (entry) { 162 entry1 = entry->next; 163 spdk_mempool_free(entry->data_pool); 164 spdk_mempool_free(entry->task_pool); 165 free(entry); 166 entry = entry1; 167 } 168 free(worker); 169 worker = next_worker; 170 } 171 } 172 173 static bool 174 probe_cb(void *cb_ctx, struct spdk_pci_device *pci_dev) 175 { 176 printf(" Found matching device at %04x:%02x:%02x.%x " 177 "vendor:0x%04x device:0x%04x\n", 178 spdk_pci_device_get_domain(pci_dev), 179 spdk_pci_device_get_bus(pci_dev), spdk_pci_device_get_dev(pci_dev), 180 spdk_pci_device_get_func(pci_dev), 181 spdk_pci_device_get_vendor_id(pci_dev), spdk_pci_device_get_device_id(pci_dev)); 182 183 return true; 184 } 185 186 static void 187 attach_cb(void *cb_ctx, struct spdk_pci_device *pci_dev, struct spdk_ioat_chan *ioat) 188 { 189 struct ioat_device *dev; 190 191 if (g_ioat_chan_num >= g_user_config.ioat_chan_num) { 192 return; 193 } 194 195 dev = spdk_dma_zmalloc(sizeof(*dev), 0, NULL); 196 if (dev == NULL) { 197 printf("Failed to allocate device struct\n"); 198 return; 199 } 200 201 dev->ioat = ioat; 202 g_ioat_chan_num++; 203 TAILQ_INSERT_TAIL(&g_devices, dev, tailq); 204 } 205 206 static int 207 ioat_init(void) 208 { 209 if (spdk_ioat_probe(NULL, probe_cb, attach_cb) != 0) { 210 fprintf(stderr, "ioat_probe() failed\n"); 211 return 1; 212 } 213 214 return 0; 215 } 216 217 static void 218 usage(char *program_name) 219 { 220 printf("%s options\n", program_name); 221 printf("\t[-h help message]\n"); 222 printf("\t[-c core mask for distributing I/O submission/completion work]\n"); 223 printf("\t[-q queue depth]\n"); 224 printf("\t[-n number of channels]\n"); 225 printf("\t[-o transfer size in bytes]\n"); 226 printf("\t[-t time in seconds]\n"); 227 printf("\t[-v verify copy result if this switch is on]\n"); 228 } 229 230 static int 231 parse_args(int argc, char **argv) 232 { 233 int op; 234 235 construct_user_config(&g_user_config); 236 while ((op = getopt(argc, argv, "c:hn:o:q:t:v")) != -1) { 237 switch (op) { 238 case 'o': 239 g_user_config.xfer_size_bytes = spdk_strtol(optarg, 10); 240 break; 241 case 'n': 242 g_user_config.ioat_chan_num = spdk_strtol(optarg, 10); 243 break; 244 case 'q': 245 g_user_config.queue_depth = spdk_strtol(optarg, 10); 246 break; 247 case 't': 248 g_user_config.time_in_sec = spdk_strtol(optarg, 10); 249 break; 250 case 'c': 251 g_user_config.core_mask = optarg; 252 break; 253 case 'v': 254 g_user_config.verify = true; 255 break; 256 case 'h': 257 usage(argv[0]); 258 exit(0); 259 default: 260 usage(argv[0]); 261 return 1; 262 } 263 } 264 if (g_user_config.xfer_size_bytes <= 0 || g_user_config.queue_depth <= 0 || 265 g_user_config.time_in_sec <= 0 || !g_user_config.core_mask || 266 g_user_config.ioat_chan_num <= 0) { 267 usage(argv[0]); 268 return 1; 269 } 270 271 return 0; 272 } 273 274 static void 275 drain_io(struct ioat_chan_entry *ioat_chan_entry) 276 { 277 spdk_ioat_flush(ioat_chan_entry->chan); 278 while (ioat_chan_entry->current_queue_depth > 0) { 279 spdk_ioat_process_events(ioat_chan_entry->chan); 280 } 281 } 282 283 static void 284 submit_single_xfer(struct ioat_chan_entry *ioat_chan_entry, struct ioat_task *ioat_task, void *dst, 285 void *src) 286 { 287 ioat_task->ioat_chan_entry = ioat_chan_entry; 288 ioat_task->src = src; 289 ioat_task->dst = dst; 290 291 spdk_ioat_build_copy(ioat_chan_entry->chan, ioat_task, ioat_done, dst, src, 292 g_user_config.xfer_size_bytes); 293 ioat_chan_entry->waiting_for_flush++; 294 if (ioat_chan_entry->waiting_for_flush >= ioat_chan_entry->flush_threshold) { 295 spdk_ioat_flush(ioat_chan_entry->chan); 296 ioat_chan_entry->waiting_for_flush = 0; 297 } 298 299 ioat_chan_entry->current_queue_depth++; 300 } 301 302 static int 303 submit_xfers(struct ioat_chan_entry *ioat_chan_entry, uint64_t queue_depth) 304 { 305 while (queue_depth-- > 0) { 306 void *src = NULL, *dst = NULL; 307 struct ioat_task *ioat_task = NULL; 308 309 src = spdk_mempool_get(ioat_chan_entry->data_pool); 310 dst = spdk_mempool_get(ioat_chan_entry->data_pool); 311 ioat_task = spdk_mempool_get(ioat_chan_entry->task_pool); 312 if (!ioat_task) { 313 fprintf(stderr, "Unable to get ioat_task\n"); 314 return 1; 315 } 316 317 submit_single_xfer(ioat_chan_entry, ioat_task, dst, src); 318 } 319 return 0; 320 } 321 322 static int 323 work_fn(void *arg) 324 { 325 uint64_t tsc_end; 326 struct worker_thread *worker = (struct worker_thread *)arg; 327 struct ioat_chan_entry *t = NULL; 328 329 printf("Starting thread on core %u\n", worker->core); 330 331 tsc_end = spdk_get_ticks() + g_user_config.time_in_sec * spdk_get_ticks_hz(); 332 333 t = worker->ctx; 334 while (t != NULL) { 335 /* begin to submit transfers */ 336 t->waiting_for_flush = 0; 337 t->flush_threshold = g_user_config.queue_depth / 2; 338 if (submit_xfers(t, g_user_config.queue_depth) != 0) { 339 return 1; 340 } 341 t = t->next; 342 } 343 344 while (1) { 345 t = worker->ctx; 346 while (t != NULL) { 347 spdk_ioat_process_events(t->chan); 348 t = t->next; 349 } 350 351 if (spdk_get_ticks() > tsc_end) { 352 break; 353 } 354 } 355 356 t = worker->ctx; 357 while (t != NULL) { 358 /* begin to drain io */ 359 t->is_draining = true; 360 drain_io(t); 361 t = t->next; 362 } 363 364 return 0; 365 } 366 367 static int 368 init(void) 369 { 370 struct spdk_env_opts opts; 371 372 spdk_env_opts_init(&opts); 373 opts.name = "ioat_perf"; 374 opts.core_mask = g_user_config.core_mask; 375 if (spdk_env_init(&opts) < 0) { 376 return 1; 377 } 378 379 return 0; 380 } 381 382 static int 383 dump_result(void) 384 { 385 uint64_t total_completed = 0; 386 uint64_t total_failed = 0; 387 uint64_t total_xfer_per_sec, total_bw_in_MiBps; 388 struct worker_thread *worker = g_workers; 389 390 printf("Channel_ID Core Transfers Bandwidth Failed\n"); 391 printf("-----------------------------------------------------------\n"); 392 while (worker != NULL) { 393 struct ioat_chan_entry *t = worker->ctx; 394 while (t) { 395 uint64_t xfer_per_sec = t->xfer_completed / g_user_config.time_in_sec; 396 uint64_t bw_in_MiBps = (t->xfer_completed * g_user_config.xfer_size_bytes) / 397 (g_user_config.time_in_sec * 1024 * 1024); 398 399 total_completed += t->xfer_completed; 400 total_failed += t->xfer_failed; 401 402 if (xfer_per_sec) { 403 printf("%10d%10d%12" PRIu64 "/s%8" PRIu64 " MiB/s%11" PRIu64 "\n", 404 t->ioat_chan_id, worker->core, xfer_per_sec, 405 bw_in_MiBps, t->xfer_failed); 406 } 407 t = t->next; 408 } 409 worker = worker->next; 410 } 411 412 total_xfer_per_sec = total_completed / g_user_config.time_in_sec; 413 total_bw_in_MiBps = (total_completed * g_user_config.xfer_size_bytes) / 414 (g_user_config.time_in_sec * 1024 * 1024); 415 416 printf("===========================================================\n"); 417 printf("Total:%26" PRIu64 "/s%8" PRIu64 " MiB/s%11" PRIu64 "\n", 418 total_xfer_per_sec, total_bw_in_MiBps, total_failed); 419 420 return total_failed ? 1 : 0; 421 } 422 423 static struct spdk_ioat_chan * 424 get_next_chan(void) 425 { 426 struct spdk_ioat_chan *chan; 427 428 if (g_next_device == NULL) { 429 return NULL; 430 } 431 432 chan = g_next_device->ioat; 433 434 g_next_device = TAILQ_NEXT(g_next_device, tailq); 435 436 return chan; 437 } 438 439 static int 440 associate_workers_with_chan(void) 441 { 442 struct spdk_ioat_chan *chan = get_next_chan(); 443 struct worker_thread *worker = g_workers; 444 struct ioat_chan_entry *t; 445 char buf_pool_name[30], task_pool_name[30]; 446 int i = 0; 447 448 while (chan != NULL) { 449 t = calloc(1, sizeof(struct ioat_chan_entry)); 450 if (!t) { 451 return 1; 452 } 453 454 t->ioat_chan_id = i; 455 snprintf(buf_pool_name, sizeof(buf_pool_name), "buf_pool_%d", i); 456 snprintf(task_pool_name, sizeof(task_pool_name), "task_pool_%d", i); 457 t->data_pool = spdk_mempool_create(buf_pool_name, 458 g_user_config.queue_depth * 2, /* src + dst */ 459 g_user_config.xfer_size_bytes, 460 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 461 SPDK_ENV_SOCKET_ID_ANY); 462 t->task_pool = spdk_mempool_create(task_pool_name, 463 g_user_config.queue_depth, 464 sizeof(struct ioat_task), 465 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 466 SPDK_ENV_SOCKET_ID_ANY); 467 if (!t->data_pool || !t->task_pool) { 468 fprintf(stderr, "Could not allocate buffer pool.\n"); 469 spdk_mempool_free(t->data_pool); 470 spdk_mempool_free(t->task_pool); 471 free(t); 472 return 1; 473 } 474 printf("Associating ioat_channel %d with core %d\n", i, worker->core); 475 t->chan = chan; 476 t->next = worker->ctx; 477 worker->ctx = t; 478 479 worker = worker->next; 480 if (worker == NULL) { 481 worker = g_workers; 482 } 483 484 chan = get_next_chan(); 485 i++; 486 } 487 488 return 0; 489 } 490 491 int 492 main(int argc, char **argv) 493 { 494 int rc; 495 struct worker_thread *worker, *main_worker; 496 unsigned main_core; 497 498 if (parse_args(argc, argv) != 0) { 499 return 1; 500 } 501 502 if (init() != 0) { 503 return 1; 504 } 505 506 if (register_workers() != 0) { 507 rc = 1; 508 goto cleanup; 509 } 510 511 if (ioat_init() != 0) { 512 rc = 1; 513 goto cleanup; 514 } 515 516 if (g_ioat_chan_num == 0) { 517 printf("No channels found\n"); 518 rc = 1; 519 goto cleanup; 520 } 521 522 if (g_user_config.ioat_chan_num > g_ioat_chan_num) { 523 printf("%d channels are requested, but only %d are found," 524 "so only test %d channels\n", g_user_config.ioat_chan_num, 525 g_ioat_chan_num, g_ioat_chan_num); 526 g_user_config.ioat_chan_num = g_ioat_chan_num; 527 } 528 529 g_next_device = TAILQ_FIRST(&g_devices); 530 dump_user_config(&g_user_config); 531 532 if (associate_workers_with_chan() != 0) { 533 rc = 1; 534 goto cleanup; 535 } 536 537 /* Launch all of the secondary workers */ 538 main_core = spdk_env_get_current_core(); 539 main_worker = NULL; 540 worker = g_workers; 541 while (worker != NULL) { 542 if (worker->core != main_core) { 543 spdk_env_thread_launch_pinned(worker->core, work_fn, worker); 544 } else { 545 assert(main_worker == NULL); 546 main_worker = worker; 547 } 548 worker = worker->next; 549 } 550 551 assert(main_worker != NULL); 552 rc = work_fn(main_worker); 553 if (rc != 0) { 554 goto cleanup; 555 } 556 557 spdk_env_thread_wait_all(); 558 559 rc = dump_result(); 560 561 cleanup: 562 unregister_workers(); 563 ioat_exit(); 564 565 spdk_env_fini(); 566 return rc; 567 } 568