1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2015 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 8 #include "spdk/ioat.h" 9 #include "spdk/env.h" 10 #include "spdk/queue.h" 11 #include "spdk/string.h" 12 #include "spdk/util.h" 13 14 #define SRC_BUFFER_SIZE (512*1024) 15 16 enum ioat_task_type { 17 IOAT_COPY_TYPE, 18 IOAT_FILL_TYPE, 19 }; 20 21 struct user_config { 22 int queue_depth; 23 int time_in_sec; 24 char *core_mask; 25 }; 26 27 struct ioat_device { 28 struct spdk_ioat_chan *ioat; 29 TAILQ_ENTRY(ioat_device) tailq; 30 }; 31 32 static TAILQ_HEAD(, ioat_device) g_devices = TAILQ_HEAD_INITIALIZER(g_devices); 33 static struct ioat_device *g_next_device; 34 35 static struct user_config g_user_config; 36 37 struct thread_entry { 38 struct spdk_ioat_chan *chan; 39 uint64_t xfer_completed; 40 uint64_t xfer_failed; 41 uint64_t fill_completed; 42 uint64_t fill_failed; 43 uint64_t current_queue_depth; 44 unsigned lcore_id; 45 bool is_draining; 46 bool init_failed; 47 struct spdk_mempool *data_pool; 48 struct spdk_mempool *task_pool; 49 }; 50 51 struct ioat_task { 52 enum ioat_task_type type; 53 struct thread_entry *thread_entry; 54 void *buffer; 55 int len; 56 uint64_t fill_pattern; 57 void *src; 58 void *dst; 59 }; 60 61 static __thread unsigned int seed = 0; 62 63 static unsigned char *g_src; 64 65 static void submit_single_xfer(struct ioat_task *ioat_task); 66 67 static void 68 construct_user_config(struct user_config *self) 69 { 70 self->queue_depth = 32; 71 self->time_in_sec = 10; 72 self->core_mask = "0x1"; 73 } 74 75 static void 76 dump_user_config(struct user_config *self) 77 { 78 printf("User configuration:\n"); 79 printf("Run time: %u seconds\n", self->time_in_sec); 80 printf("Core mask: %s\n", self->core_mask); 81 printf("Queue depth: %u\n", self->queue_depth); 82 } 83 84 static void 85 ioat_exit(void) 86 { 87 struct ioat_device *dev; 88 89 while (!TAILQ_EMPTY(&g_devices)) { 90 dev = TAILQ_FIRST(&g_devices); 91 TAILQ_REMOVE(&g_devices, dev, tailq); 92 if (dev->ioat) { 93 spdk_ioat_detach(dev->ioat); 94 } 95 free(dev); 96 } 97 } 98 static void 99 prepare_ioat_task(struct thread_entry *thread_entry, struct ioat_task *ioat_task) 100 { 101 int len; 102 uintptr_t src_offset; 103 uintptr_t dst_offset; 104 uint64_t fill_pattern; 105 106 if (ioat_task->type == IOAT_FILL_TYPE) { 107 fill_pattern = rand_r(&seed); 108 fill_pattern = fill_pattern << 32 | rand_r(&seed); 109 110 /* Ensure that the length of memset block is 8 Bytes aligned. 111 * In case the buffer crosses hugepage boundary and must be split, 112 * we also need to ensure 8 byte address alignment. We do it 113 * unconditionally to keep things simple. 114 */ 115 len = 8 + ((rand_r(&seed) % (SRC_BUFFER_SIZE - 16)) & ~0x7); 116 dst_offset = 8 + rand_r(&seed) % (SRC_BUFFER_SIZE - 8 - len); 117 ioat_task->fill_pattern = fill_pattern; 118 ioat_task->dst = (void *)(((uintptr_t)ioat_task->buffer + dst_offset) & ~0x7); 119 } else { 120 src_offset = rand_r(&seed) % SRC_BUFFER_SIZE; 121 len = rand_r(&seed) % (SRC_BUFFER_SIZE - src_offset); 122 dst_offset = rand_r(&seed) % (SRC_BUFFER_SIZE - len); 123 124 memset(ioat_task->buffer, 0, SRC_BUFFER_SIZE); 125 ioat_task->src = (void *)((uintptr_t)g_src + src_offset); 126 ioat_task->dst = (void *)((uintptr_t)ioat_task->buffer + dst_offset); 127 } 128 ioat_task->len = len; 129 ioat_task->thread_entry = thread_entry; 130 } 131 132 static void 133 ioat_done(void *cb_arg) 134 { 135 char *value; 136 int i, failed = 0; 137 struct ioat_task *ioat_task = (struct ioat_task *)cb_arg; 138 struct thread_entry *thread_entry = ioat_task->thread_entry; 139 140 if (ioat_task->type == IOAT_FILL_TYPE) { 141 value = ioat_task->dst; 142 for (i = 0; i < ioat_task->len / 8; i++) { 143 if (memcmp(value, &ioat_task->fill_pattern, 8) != 0) { 144 thread_entry->fill_failed++; 145 failed = 1; 146 break; 147 } 148 value += 8; 149 } 150 if (!failed) { 151 thread_entry->fill_completed++; 152 } 153 } else { 154 if (memcmp(ioat_task->src, ioat_task->dst, ioat_task->len)) { 155 thread_entry->xfer_failed++; 156 } else { 157 thread_entry->xfer_completed++; 158 } 159 } 160 161 thread_entry->current_queue_depth--; 162 if (thread_entry->is_draining) { 163 spdk_mempool_put(thread_entry->data_pool, ioat_task->buffer); 164 spdk_mempool_put(thread_entry->task_pool, ioat_task); 165 } else { 166 prepare_ioat_task(thread_entry, ioat_task); 167 submit_single_xfer(ioat_task); 168 } 169 } 170 171 static bool 172 probe_cb(void *cb_ctx, struct spdk_pci_device *pci_dev) 173 { 174 printf(" Found matching device at %04x:%02x:%02x.%x " 175 "vendor:0x%04x device:0x%04x\n", 176 spdk_pci_device_get_domain(pci_dev), 177 spdk_pci_device_get_bus(pci_dev), spdk_pci_device_get_dev(pci_dev), 178 spdk_pci_device_get_func(pci_dev), 179 spdk_pci_device_get_vendor_id(pci_dev), spdk_pci_device_get_device_id(pci_dev)); 180 181 return true; 182 } 183 184 static void 185 attach_cb(void *cb_ctx, struct spdk_pci_device *pci_dev, struct spdk_ioat_chan *ioat) 186 { 187 struct ioat_device *dev; 188 189 dev = malloc(sizeof(*dev)); 190 if (dev == NULL) { 191 printf("Failed to allocate device struct\n"); 192 return; 193 } 194 memset(dev, 0, sizeof(*dev)); 195 196 dev->ioat = ioat; 197 TAILQ_INSERT_TAIL(&g_devices, dev, tailq); 198 } 199 200 static int 201 ioat_init(void) 202 { 203 if (spdk_ioat_probe(NULL, probe_cb, attach_cb) != 0) { 204 fprintf(stderr, "ioat_probe() failed\n"); 205 return 1; 206 } 207 208 return 0; 209 } 210 211 static void 212 usage(char *program_name) 213 { 214 printf("%s options\n", program_name); 215 printf("\t[-h help message]\n"); 216 printf("\t[-c core mask for distributing I/O submission/completion work]\n"); 217 printf("\t[-t time in seconds]\n"); 218 printf("\t[-q queue depth]\n"); 219 } 220 221 static int 222 parse_args(int argc, char **argv) 223 { 224 int op; 225 226 construct_user_config(&g_user_config); 227 while ((op = getopt(argc, argv, "c:ht:q:")) != -1) { 228 switch (op) { 229 case 't': 230 g_user_config.time_in_sec = spdk_strtol(optarg, 10); 231 break; 232 case 'c': 233 g_user_config.core_mask = optarg; 234 break; 235 case 'q': 236 g_user_config.queue_depth = spdk_strtol(optarg, 10); 237 break; 238 case 'h': 239 usage(argv[0]); 240 exit(0); 241 default: 242 usage(argv[0]); 243 return 1; 244 } 245 } 246 if (g_user_config.time_in_sec <= 0 || !g_user_config.core_mask || 247 g_user_config.queue_depth <= 0) { 248 usage(argv[0]); 249 return 1; 250 } 251 252 return 0; 253 } 254 255 static void 256 drain_xfers(struct thread_entry *thread_entry) 257 { 258 while (thread_entry->current_queue_depth > 0) { 259 spdk_ioat_process_events(thread_entry->chan); 260 } 261 } 262 263 static void 264 submit_single_xfer(struct ioat_task *ioat_task) 265 { 266 if (ioat_task->type == IOAT_FILL_TYPE) 267 spdk_ioat_submit_fill(ioat_task->thread_entry->chan, ioat_task, ioat_done, 268 ioat_task->dst, ioat_task->fill_pattern, ioat_task->len); 269 else 270 spdk_ioat_submit_copy(ioat_task->thread_entry->chan, ioat_task, ioat_done, 271 ioat_task->dst, ioat_task->src, ioat_task->len); 272 ioat_task->thread_entry->current_queue_depth++; 273 } 274 275 static void 276 submit_xfers(struct thread_entry *thread_entry, uint64_t queue_depth) 277 { 278 while (queue_depth-- > 0) { 279 struct ioat_task *ioat_task = NULL; 280 ioat_task = spdk_mempool_get(thread_entry->task_pool); 281 assert(ioat_task != NULL); 282 ioat_task->buffer = spdk_mempool_get(thread_entry->data_pool); 283 assert(ioat_task->buffer != NULL); 284 285 ioat_task->type = IOAT_COPY_TYPE; 286 if (spdk_ioat_get_dma_capabilities(thread_entry->chan) & SPDK_IOAT_ENGINE_FILL_SUPPORTED) { 287 if (queue_depth % 2) { 288 ioat_task->type = IOAT_FILL_TYPE; 289 } 290 } 291 prepare_ioat_task(thread_entry, ioat_task); 292 submit_single_xfer(ioat_task); 293 } 294 } 295 296 static int 297 work_fn(void *arg) 298 { 299 uint64_t tsc_end; 300 char buf_pool_name[20], task_pool_name[20]; 301 struct thread_entry *t = (struct thread_entry *)arg; 302 303 if (!t->chan) { 304 return 1; 305 } 306 307 t->lcore_id = spdk_env_get_current_core(); 308 309 snprintf(buf_pool_name, sizeof(buf_pool_name), "buf_pool_%u", t->lcore_id); 310 snprintf(task_pool_name, sizeof(task_pool_name), "task_pool_%u", t->lcore_id); 311 t->data_pool = spdk_mempool_create(buf_pool_name, g_user_config.queue_depth, SRC_BUFFER_SIZE, 312 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 313 SPDK_ENV_NUMA_ID_ANY); 314 t->task_pool = spdk_mempool_create(task_pool_name, g_user_config.queue_depth, 315 sizeof(struct ioat_task), 316 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 317 SPDK_ENV_NUMA_ID_ANY); 318 if (!t->data_pool || !t->task_pool) { 319 fprintf(stderr, "Could not allocate buffer pool.\n"); 320 t->init_failed = true; 321 return 1; 322 } 323 324 tsc_end = spdk_get_ticks() + g_user_config.time_in_sec * spdk_get_ticks_hz(); 325 326 submit_xfers(t, g_user_config.queue_depth); 327 while (spdk_get_ticks() < tsc_end) { 328 spdk_ioat_process_events(t->chan); 329 } 330 331 t->is_draining = true; 332 drain_xfers(t); 333 334 return 0; 335 } 336 337 static int 338 init_src_buffer(void) 339 { 340 int i; 341 342 g_src = spdk_dma_zmalloc(SRC_BUFFER_SIZE, 512, NULL); 343 if (g_src == NULL) { 344 fprintf(stderr, "Allocate src buffer failed\n"); 345 return 1; 346 } 347 348 for (i = 0; i < SRC_BUFFER_SIZE / 4; i++) { 349 memset((g_src + (4 * i)), i, 4); 350 } 351 352 return 0; 353 } 354 355 static int 356 init(void) 357 { 358 struct spdk_env_opts opts; 359 360 opts.opts_size = sizeof(opts); 361 spdk_env_opts_init(&opts); 362 opts.name = "verify"; 363 opts.core_mask = g_user_config.core_mask; 364 if (spdk_env_init(&opts) < 0) { 365 fprintf(stderr, "Unable to initialize SPDK env\n"); 366 return 1; 367 } 368 369 if (init_src_buffer() != 0) { 370 fprintf(stderr, "Could not init src buffer\n"); 371 return 1; 372 } 373 if (ioat_init() != 0) { 374 fprintf(stderr, "Could not init ioat\n"); 375 return 1; 376 } 377 378 return 0; 379 } 380 381 static int 382 dump_result(struct thread_entry *threads, uint32_t num_threads) 383 { 384 uint32_t i; 385 uint64_t total_completed = 0; 386 uint64_t total_failed = 0; 387 388 for (i = 0; i < num_threads; i++) { 389 struct thread_entry *t = &threads[i]; 390 391 if (!t->chan) { 392 continue; 393 } 394 395 if (t->init_failed) { 396 total_failed++; 397 continue; 398 } 399 400 total_completed += t->xfer_completed; 401 total_completed += t->fill_completed; 402 total_failed += t->xfer_failed; 403 total_failed += t->fill_failed; 404 if (total_completed || total_failed) 405 printf("lcore = %d, copy success = %" PRIu64 ", copy failed = %" PRIu64 ", fill success = %" PRIu64 406 ", fill failed = %" PRIu64 "\n", 407 t->lcore_id, t->xfer_completed, t->xfer_failed, t->fill_completed, t->fill_failed); 408 } 409 return total_failed ? 1 : 0; 410 } 411 412 static struct spdk_ioat_chan * 413 get_next_chan(void) 414 { 415 struct spdk_ioat_chan *chan; 416 417 if (g_next_device == NULL) { 418 fprintf(stderr, "Not enough ioat channels found. Check that ioat channels are bound\n"); 419 fprintf(stderr, "to uio_pci_generic or vfio-pci. scripts/setup.sh can help with this.\n"); 420 return NULL; 421 } 422 423 chan = g_next_device->ioat; 424 425 g_next_device = TAILQ_NEXT(g_next_device, tailq); 426 427 return chan; 428 } 429 430 static uint32_t 431 get_max_core(void) 432 { 433 uint32_t i; 434 uint32_t max_core = 0; 435 436 SPDK_ENV_FOREACH_CORE(i) { 437 if (i > max_core) { 438 max_core = i; 439 } 440 } 441 442 return max_core; 443 } 444 445 int 446 main(int argc, char **argv) 447 { 448 uint32_t i, current_core; 449 struct thread_entry *threads; 450 uint32_t num_threads; 451 int rc; 452 453 if (parse_args(argc, argv) != 0) { 454 return 1; 455 } 456 457 if (init() != 0) { 458 return 1; 459 } 460 461 dump_user_config(&g_user_config); 462 463 g_next_device = TAILQ_FIRST(&g_devices); 464 465 num_threads = get_max_core() + 1; 466 threads = calloc(num_threads, sizeof(*threads)); 467 if (!threads) { 468 fprintf(stderr, "Thread memory allocation failed\n"); 469 rc = 1; 470 goto cleanup; 471 } 472 473 current_core = spdk_env_get_current_core(); 474 SPDK_ENV_FOREACH_CORE(i) { 475 if (i != current_core) { 476 threads[i].chan = get_next_chan(); 477 spdk_env_thread_launch_pinned(i, work_fn, &threads[i]); 478 } 479 } 480 481 threads[current_core].chan = get_next_chan(); 482 if (work_fn(&threads[current_core]) != 0) { 483 rc = 1; 484 goto cleanup; 485 } 486 487 spdk_env_thread_wait_all(); 488 rc = dump_result(threads, num_threads); 489 490 cleanup: 491 spdk_dma_free(g_src); 492 ioat_exit(); 493 free(threads); 494 495 spdk_env_fini(); 496 return rc; 497 } 498