1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include <rte_config.h> 37 #include <rte_launch.h> 38 39 #include "spdk/ioat.h" 40 #include "spdk/env.h" 41 #include "spdk/queue.h" 42 #include "spdk/string.h" 43 44 struct user_config { 45 int xfer_size_bytes; 46 int queue_depth; 47 int time_in_sec; 48 bool verify; 49 char *core_mask; 50 int ioat_chan_num; 51 }; 52 53 struct ioat_device { 54 struct spdk_ioat_chan *ioat; 55 TAILQ_ENTRY(ioat_device) tailq; 56 }; 57 58 static TAILQ_HEAD(, ioat_device) g_devices; 59 static struct ioat_device *g_next_device; 60 61 static struct user_config g_user_config; 62 63 struct ioat_chan_entry { 64 struct spdk_ioat_chan *chan; 65 int ioat_chan_id; 66 uint64_t xfer_completed; 67 uint64_t xfer_failed; 68 uint64_t current_queue_depth; 69 bool is_draining; 70 struct spdk_mempool *data_pool; 71 struct spdk_mempool *task_pool; 72 struct ioat_chan_entry *next; 73 }; 74 75 struct worker_thread { 76 struct ioat_chan_entry *ctx; 77 struct worker_thread *next; 78 unsigned core; 79 }; 80 81 struct ioat_task { 82 struct ioat_chan_entry *ioat_chan_entry; 83 void *src; 84 void *dst; 85 }; 86 87 static struct worker_thread *g_workers = NULL; 88 static int g_num_workers = 0; 89 static int g_ioat_chan_num = 0; 90 91 static void submit_single_xfer(struct ioat_chan_entry *ioat_chan_entry, struct ioat_task *ioat_task, 92 void *dst, void *src); 93 94 static void 95 construct_user_config(struct user_config *self) 96 { 97 self->xfer_size_bytes = 4096; 98 self->ioat_chan_num = 1; 99 self->queue_depth = 256; 100 self->time_in_sec = 10; 101 self->verify = false; 102 self->core_mask = "0x1"; 103 } 104 105 static void 106 dump_user_config(struct user_config *self) 107 { 108 printf("User configuration:\n"); 109 printf("Number of channels: %u\n", self->ioat_chan_num); 110 printf("Transfer size: %u bytes\n", self->xfer_size_bytes); 111 printf("Queue depth: %u\n", self->queue_depth); 112 printf("Run time: %u seconds\n", self->time_in_sec); 113 printf("Core mask: %s\n", self->core_mask); 114 printf("Verify: %s\n\n", self->verify ? "Yes" : "No"); 115 } 116 117 static void 118 ioat_exit(void) 119 { 120 struct ioat_device *dev; 121 122 while (!TAILQ_EMPTY(&g_devices)) { 123 dev = TAILQ_FIRST(&g_devices); 124 TAILQ_REMOVE(&g_devices, dev, tailq); 125 if (dev->ioat) { 126 spdk_ioat_detach(dev->ioat); 127 } 128 spdk_dma_free(dev); 129 } 130 } 131 132 static void 133 ioat_done(void *cb_arg) 134 { 135 struct ioat_task *ioat_task = (struct ioat_task *)cb_arg; 136 struct ioat_chan_entry *ioat_chan_entry = ioat_task->ioat_chan_entry; 137 138 if (g_user_config.verify && memcmp(ioat_task->src, ioat_task->dst, g_user_config.xfer_size_bytes)) { 139 ioat_chan_entry->xfer_failed++; 140 } else { 141 ioat_chan_entry->xfer_completed++; 142 } 143 144 ioat_chan_entry->current_queue_depth--; 145 146 if (ioat_chan_entry->is_draining) { 147 spdk_mempool_put(ioat_chan_entry->data_pool, ioat_task->src); 148 spdk_mempool_put(ioat_chan_entry->data_pool, ioat_task->dst); 149 spdk_mempool_put(ioat_chan_entry->task_pool, ioat_task); 150 } else { 151 submit_single_xfer(ioat_chan_entry, ioat_task, ioat_task->dst, ioat_task->src); 152 } 153 } 154 155 static int 156 register_workers(void) 157 { 158 uint32_t i; 159 struct worker_thread *worker; 160 161 g_workers = NULL; 162 g_num_workers = 0; 163 164 SPDK_ENV_FOREACH_CORE(i) { 165 worker = calloc(1, sizeof(*worker)); 166 if (worker == NULL) { 167 fprintf(stderr, "Unable to allocate worker\n"); 168 return -1; 169 } 170 171 worker->core = i; 172 worker->next = g_workers; 173 g_workers = worker; 174 g_num_workers++; 175 } 176 177 return 0; 178 } 179 180 static void 181 unregister_workers(void) 182 { 183 struct worker_thread *worker = g_workers; 184 struct ioat_chan_entry *entry, *entry1; 185 186 /* Free ioat_chan_entry and worker thread */ 187 while (worker) { 188 struct worker_thread *next_worker = worker->next; 189 entry = worker->ctx; 190 while (entry) { 191 entry1 = entry->next; 192 spdk_mempool_free(entry->data_pool); 193 spdk_mempool_free(entry->task_pool); 194 free(entry); 195 entry = entry1; 196 } 197 free(worker); 198 worker = next_worker; 199 } 200 } 201 202 static bool 203 probe_cb(void *cb_ctx, struct spdk_pci_device *pci_dev) 204 { 205 printf(" Found matching device at %04x:%02x:%02x.%x " 206 "vendor:0x%04x device:0x%04x\n", 207 spdk_pci_device_get_domain(pci_dev), 208 spdk_pci_device_get_bus(pci_dev), spdk_pci_device_get_dev(pci_dev), 209 spdk_pci_device_get_func(pci_dev), 210 spdk_pci_device_get_vendor_id(pci_dev), spdk_pci_device_get_device_id(pci_dev)); 211 212 return true; 213 } 214 215 static void 216 attach_cb(void *cb_ctx, struct spdk_pci_device *pci_dev, struct spdk_ioat_chan *ioat) 217 { 218 struct ioat_device *dev; 219 220 if (g_ioat_chan_num >= g_user_config.ioat_chan_num) { 221 return; 222 } 223 224 dev = spdk_dma_zmalloc(sizeof(*dev), 0, NULL); 225 if (dev == NULL) { 226 printf("Failed to allocate device struct\n"); 227 return; 228 } 229 230 dev->ioat = ioat; 231 g_ioat_chan_num++; 232 TAILQ_INSERT_TAIL(&g_devices, dev, tailq); 233 } 234 235 static int 236 ioat_init(void) 237 { 238 TAILQ_INIT(&g_devices); 239 240 if (spdk_ioat_probe(NULL, probe_cb, attach_cb) != 0) { 241 fprintf(stderr, "ioat_probe() failed\n"); 242 return 1; 243 } 244 245 return 0; 246 } 247 248 static void 249 usage(char *program_name) 250 { 251 printf("%s options\n", program_name); 252 printf("\t[-h help message]\n"); 253 printf("\t[-c core mask for distributing I/O submission/completion work]\n"); 254 printf("\t[-q queue depth]\n"); 255 printf("\t[-n number of channels]\n"); 256 printf("\t[-s transfer size in bytes]\n"); 257 printf("\t[-t time in seconds]\n"); 258 printf("\t[-v verify copy result if this switch is on]\n"); 259 } 260 261 static int 262 parse_args(int argc, char **argv) 263 { 264 int op; 265 266 construct_user_config(&g_user_config); 267 while ((op = getopt(argc, argv, "c:hn:q:s:t:v")) != -1) { 268 switch (op) { 269 case 's': 270 g_user_config.xfer_size_bytes = atoi(optarg); 271 break; 272 case 'n': 273 g_user_config.ioat_chan_num = atoi(optarg); 274 break; 275 case 'q': 276 g_user_config.queue_depth = atoi(optarg); 277 break; 278 case 't': 279 g_user_config.time_in_sec = atoi(optarg); 280 break; 281 case 'c': 282 g_user_config.core_mask = optarg; 283 break; 284 case 'v': 285 g_user_config.verify = true; 286 break; 287 case 'h': 288 usage(argv[0]); 289 exit(0); 290 default: 291 usage(argv[0]); 292 return 1; 293 } 294 } 295 if (!g_user_config.xfer_size_bytes || !g_user_config.queue_depth || 296 !g_user_config.time_in_sec || !g_user_config.core_mask || 297 !g_user_config.ioat_chan_num) { 298 usage(argv[0]); 299 return 1; 300 } 301 302 return 0; 303 } 304 305 static void 306 drain_io(struct ioat_chan_entry *ioat_chan_entry) 307 { 308 while (ioat_chan_entry->current_queue_depth > 0) { 309 spdk_ioat_process_events(ioat_chan_entry->chan); 310 } 311 } 312 313 static void 314 submit_single_xfer(struct ioat_chan_entry *ioat_chan_entry, struct ioat_task *ioat_task, void *dst, 315 void *src) 316 { 317 ioat_task->ioat_chan_entry = ioat_chan_entry; 318 ioat_task->src = src; 319 ioat_task->dst = dst; 320 321 spdk_ioat_submit_copy(ioat_chan_entry->chan, ioat_task, ioat_done, dst, src, 322 g_user_config.xfer_size_bytes); 323 324 ioat_chan_entry->current_queue_depth++; 325 } 326 327 static void 328 submit_xfers(struct ioat_chan_entry *ioat_chan_entry, uint64_t queue_depth) 329 { 330 while (queue_depth-- > 0) { 331 void *src = NULL, *dst = NULL; 332 struct ioat_task *ioat_task = NULL; 333 334 src = spdk_mempool_get(ioat_chan_entry->data_pool); 335 dst = spdk_mempool_get(ioat_chan_entry->data_pool); 336 ioat_task = spdk_mempool_get(ioat_chan_entry->task_pool); 337 338 submit_single_xfer(ioat_chan_entry, ioat_task, dst, src); 339 } 340 } 341 342 static int 343 work_fn(void *arg) 344 { 345 uint64_t tsc_end; 346 struct worker_thread *worker = (struct worker_thread *)arg; 347 struct ioat_chan_entry *t = NULL; 348 349 printf("Starting thread on core %u\n", worker->core); 350 351 tsc_end = spdk_get_ticks() + g_user_config.time_in_sec * spdk_get_ticks_hz(); 352 353 t = worker->ctx; 354 while (t != NULL) { 355 // begin to submit transfers 356 submit_xfers(t, g_user_config.queue_depth); 357 t = t->next; 358 } 359 360 while (1) { 361 t = worker->ctx; 362 while (t != NULL) { 363 spdk_ioat_process_events(t->chan); 364 t = t->next; 365 } 366 367 if (spdk_get_ticks() > tsc_end) { 368 break; 369 } 370 } 371 372 t = worker->ctx; 373 while (t != NULL) { 374 // begin to drain io 375 t->is_draining = true; 376 drain_io(t); 377 t = t->next; 378 } 379 380 return 0; 381 } 382 383 static int 384 init(void) 385 { 386 struct spdk_env_opts opts; 387 388 spdk_env_opts_init(&opts); 389 opts.name = "perf"; 390 opts.core_mask = g_user_config.core_mask; 391 spdk_env_init(&opts); 392 393 return 0; 394 } 395 396 static int 397 dump_result(void) 398 { 399 uint64_t total_completed = 0; 400 uint64_t total_failed = 0; 401 uint64_t total_xfer_per_sec, total_bw_in_MiBps; 402 struct worker_thread *worker = g_workers; 403 404 printf("Channel_ID Core Transfers Bandwidth Failed\n"); 405 printf("-----------------------------------------------------------\n"); 406 while (worker != NULL) { 407 struct ioat_chan_entry *t = worker->ctx; 408 while (t) { 409 uint64_t xfer_per_sec = t->xfer_completed / g_user_config.time_in_sec; 410 uint64_t bw_in_MiBps = (t->xfer_completed * g_user_config.xfer_size_bytes) / 411 (g_user_config.time_in_sec * 1024 * 1024); 412 413 total_completed += t->xfer_completed; 414 total_failed += t->xfer_failed; 415 416 if (xfer_per_sec) { 417 printf("%10d%10d%12" PRIu64 "/s%8" PRIu64 " MiB/s%11" PRIu64 "\n", 418 t->ioat_chan_id, worker->core, xfer_per_sec, 419 bw_in_MiBps, t->xfer_failed); 420 } 421 t = t->next; 422 } 423 worker = worker->next; 424 } 425 426 total_xfer_per_sec = total_completed / g_user_config.time_in_sec; 427 total_bw_in_MiBps = (total_completed * g_user_config.xfer_size_bytes) / 428 (g_user_config.time_in_sec * 1024 * 1024); 429 430 printf("===========================================================\n"); 431 printf("Total:%26" PRIu64 "/s%8" PRIu64 " MiB/s%11" PRIu64 "\n", 432 total_xfer_per_sec, total_bw_in_MiBps, total_failed); 433 434 return total_failed ? 1 : 0; 435 } 436 437 static struct spdk_ioat_chan * 438 get_next_chan(void) 439 { 440 struct spdk_ioat_chan *chan; 441 442 if (g_next_device == NULL) { 443 return NULL; 444 } 445 446 chan = g_next_device->ioat; 447 448 g_next_device = TAILQ_NEXT(g_next_device, tailq); 449 450 return chan; 451 } 452 453 static int 454 associate_workers_with_chan(void) 455 { 456 struct spdk_ioat_chan *chan = get_next_chan(); 457 struct worker_thread *worker = g_workers; 458 struct ioat_chan_entry *t; 459 char buf_pool_name[30], task_pool_name[30]; 460 int i = 0; 461 462 while (chan != NULL) { 463 t = calloc(1, sizeof(struct ioat_chan_entry)); 464 if (!t) { 465 return -1; 466 } 467 468 t->ioat_chan_id = i; 469 snprintf(buf_pool_name, sizeof(buf_pool_name), "buf_pool_%d", i); 470 snprintf(task_pool_name, sizeof(task_pool_name), "task_pool_%d", i); 471 t->data_pool = spdk_mempool_create(buf_pool_name, 512, g_user_config.xfer_size_bytes, 472 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 473 SPDK_ENV_SOCKET_ID_ANY); 474 t->task_pool = spdk_mempool_create(task_pool_name, 512, sizeof(struct ioat_task), 475 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 476 SPDK_ENV_SOCKET_ID_ANY); 477 if (!t->data_pool || !t->task_pool) { 478 fprintf(stderr, "Could not allocate buffer pool.\n"); 479 spdk_mempool_free(t->data_pool); 480 spdk_mempool_free(t->task_pool); 481 free(t); 482 return 1; 483 } 484 printf("Associating ioat_channel %d with core %d\n", i, worker->core); 485 t->chan = chan; 486 t->next = worker->ctx; 487 worker->ctx = t; 488 489 worker = worker->next; 490 if (worker == NULL) { 491 worker = g_workers; 492 } 493 494 chan = get_next_chan(); 495 i++; 496 } 497 498 return 0; 499 } 500 501 int 502 main(int argc, char **argv) 503 { 504 int rc; 505 struct worker_thread *worker, *master_worker; 506 unsigned master_core; 507 508 if (parse_args(argc, argv) != 0) { 509 return 1; 510 } 511 512 if (init() != 0) { 513 return 1; 514 } 515 516 if (register_workers() != 0) { 517 rc = -1; 518 goto cleanup; 519 } 520 521 if (ioat_init() != 0) { 522 rc = -1; 523 goto cleanup; 524 } 525 526 if (g_ioat_chan_num == 0) { 527 printf("No channels found\n"); 528 rc = 0; 529 goto cleanup; 530 } 531 532 if (g_user_config.ioat_chan_num > g_ioat_chan_num) { 533 printf("%d channels are requested, but only %d are found," 534 "so only test %d channels\n", g_user_config.ioat_chan_num, 535 g_ioat_chan_num, g_ioat_chan_num); 536 g_user_config.ioat_chan_num = g_ioat_chan_num; 537 } 538 539 g_next_device = TAILQ_FIRST(&g_devices); 540 dump_user_config(&g_user_config); 541 542 if (associate_workers_with_chan() != 0) { 543 rc = -1; 544 goto cleanup; 545 } 546 547 /* Launch all of the slave workers */ 548 master_core = spdk_env_get_current_core(); 549 master_worker = NULL; 550 worker = g_workers; 551 while (worker != NULL) { 552 if (worker->core != master_core) { 553 rte_eal_remote_launch(work_fn, worker, worker->core); 554 } else { 555 assert(master_worker == NULL); 556 master_worker = worker; 557 } 558 worker = worker->next; 559 } 560 561 assert(master_worker != NULL); 562 rc = work_fn(master_worker); 563 if (rc < 0) { 564 goto cleanup; 565 } 566 567 rte_eal_mp_wait_lcore(); 568 569 rc = dump_result(); 570 571 cleanup: 572 unregister_workers(); 573 ioat_exit(); 574 575 return rc; 576 } 577