1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 8 #include "spdk/env.h" 9 #include "spdk/log.h" 10 #include "spdk/nvme.h" 11 #include "spdk/queue.h" 12 #include "spdk/string.h" 13 #include "spdk/util.h" 14 #include "spdk/likely.h" 15 16 struct ctrlr_entry { 17 struct spdk_nvme_ctrlr *ctrlr; 18 enum spdk_nvme_transport_type trtype; 19 20 TAILQ_ENTRY(ctrlr_entry) link; 21 char name[1024]; 22 }; 23 24 struct ns_entry { 25 struct spdk_nvme_ctrlr *ctrlr; 26 struct spdk_nvme_ns *ns; 27 28 TAILQ_ENTRY(ns_entry) link; 29 uint32_t io_size_blocks; 30 uint32_t num_io_requests; 31 uint64_t size_in_ios; 32 uint32_t block_size; 33 char name[1024]; 34 }; 35 36 struct ctrlr_worker_ctx { 37 pthread_mutex_t mutex; 38 struct ctrlr_entry *entry; 39 uint64_t abort_submitted; 40 uint64_t abort_submit_failed; 41 uint64_t successful_abort; 42 uint64_t unsuccessful_abort; 43 uint64_t abort_failed; 44 uint64_t current_queue_depth; 45 struct spdk_nvme_ctrlr *ctrlr; 46 TAILQ_ENTRY(ctrlr_worker_ctx) link; 47 }; 48 49 struct ns_worker_ctx { 50 struct ns_entry *entry; 51 uint64_t io_submitted; 52 uint64_t io_completed; 53 uint64_t io_aborted; 54 uint64_t io_failed; 55 uint64_t current_queue_depth; 56 uint64_t offset_in_ios; 57 bool is_draining; 58 struct spdk_nvme_qpair *qpair; 59 struct ctrlr_worker_ctx *ctrlr_ctx; 60 TAILQ_ENTRY(ns_worker_ctx) link; 61 }; 62 63 struct perf_task { 64 struct ns_worker_ctx *ns_ctx; 65 void *buf; 66 }; 67 68 struct worker_thread { 69 TAILQ_HEAD(, ns_worker_ctx) ns_ctx; 70 TAILQ_HEAD(, ctrlr_worker_ctx) ctrlr_ctx; 71 TAILQ_ENTRY(worker_thread) link; 72 unsigned lcore; 73 }; 74 75 static const char *g_workload_type = "read"; 76 static TAILQ_HEAD(, ctrlr_entry) g_controllers = TAILQ_HEAD_INITIALIZER(g_controllers); 77 static TAILQ_HEAD(, ns_entry) g_namespaces = TAILQ_HEAD_INITIALIZER(g_namespaces); 78 static int g_num_namespaces; 79 static TAILQ_HEAD(, worker_thread) g_workers = TAILQ_HEAD_INITIALIZER(g_workers); 80 static int g_num_workers = 0; 81 static uint32_t g_main_core; 82 83 static int g_abort_interval = 1; 84 85 static uint64_t g_tsc_rate; 86 87 static uint32_t g_io_size_bytes = 131072; 88 static uint32_t g_max_io_size_blocks; 89 static int g_rw_percentage = -1; 90 static int g_is_random; 91 static int g_queue_depth = 128; 92 static int g_time_in_sec = 3; 93 static int g_dpdk_mem; 94 static int g_shm_id = -1; 95 static bool g_no_pci; 96 static bool g_warn; 97 static bool g_mix_specified; 98 99 static const char *g_core_mask; 100 101 struct trid_entry { 102 struct spdk_nvme_transport_id trid; 103 uint16_t nsid; 104 TAILQ_ENTRY(trid_entry) tailq; 105 }; 106 107 static TAILQ_HEAD(, trid_entry) g_trid_list = TAILQ_HEAD_INITIALIZER(g_trid_list); 108 109 static void io_complete(void *ctx, const struct spdk_nvme_cpl *cpl); 110 111 static int 112 build_nvme_name(char *name, size_t length, struct spdk_nvme_ctrlr *ctrlr) 113 { 114 const struct spdk_nvme_transport_id *trid; 115 int res = 0; 116 117 trid = spdk_nvme_ctrlr_get_transport_id(ctrlr); 118 119 switch (trid->trtype) { 120 case SPDK_NVME_TRANSPORT_PCIE: 121 res = snprintf(name, length, "PCIE (%s)", trid->traddr); 122 break; 123 case SPDK_NVME_TRANSPORT_RDMA: 124 res = snprintf(name, length, "RDMA (addr:%s subnqn:%s)", trid->traddr, trid->subnqn); 125 break; 126 case SPDK_NVME_TRANSPORT_TCP: 127 res = snprintf(name, length, "TCP (addr:%s subnqn:%s)", trid->traddr, trid->subnqn); 128 break; 129 case SPDK_NVME_TRANSPORT_CUSTOM: 130 res = snprintf(name, length, "CUSTOM (%s)", trid->traddr); 131 break; 132 133 default: 134 fprintf(stderr, "Unknown transport type %d\n", trid->trtype); 135 break; 136 } 137 return res; 138 } 139 140 static void 141 build_nvme_ns_name(char *name, size_t length, struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid) 142 { 143 int res = 0; 144 145 res = build_nvme_name(name, length, ctrlr); 146 if (res > 0) { 147 snprintf(name + res, length - res, " NSID %u", nsid); 148 } 149 150 } 151 152 static void 153 register_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns) 154 { 155 struct ns_entry *entry; 156 const struct spdk_nvme_ctrlr_data *cdata; 157 uint32_t max_xfer_size, entries, sector_size; 158 uint64_t ns_size; 159 struct spdk_nvme_io_qpair_opts opts; 160 161 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 162 163 if (!spdk_nvme_ns_is_active(ns)) { 164 printf("Controller %-20.20s (%-20.20s): Skipping inactive NS %u\n", 165 cdata->mn, cdata->sn, 166 spdk_nvme_ns_get_id(ns)); 167 g_warn = true; 168 return; 169 } 170 171 ns_size = spdk_nvme_ns_get_size(ns); 172 sector_size = spdk_nvme_ns_get_sector_size(ns); 173 174 if (ns_size < g_io_size_bytes || sector_size > g_io_size_bytes) { 175 printf("WARNING: controller %-20.20s (%-20.20s) ns %u has invalid " 176 "ns size %" PRIu64 " / block size %u for I/O size %u\n", 177 cdata->mn, cdata->sn, spdk_nvme_ns_get_id(ns), 178 ns_size, spdk_nvme_ns_get_sector_size(ns), g_io_size_bytes); 179 g_warn = true; 180 return; 181 } 182 183 max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns); 184 spdk_nvme_ctrlr_get_default_io_qpair_opts(ctrlr, &opts, sizeof(opts)); 185 /* NVMe driver may add additional entries based on 186 * stripe size and maximum transfer size, we assume 187 * 1 more entry be used for stripe. 188 */ 189 entries = (g_io_size_bytes - 1) / max_xfer_size + 2; 190 if ((g_queue_depth * entries) > opts.io_queue_size) { 191 printf("controller IO queue size %u less than required\n", 192 opts.io_queue_size); 193 printf("Consider using lower queue depth or small IO size because " 194 "IO requests may be queued at the NVMe driver.\n"); 195 } 196 /* For requests which have children requests, parent request itself 197 * will also occupy 1 entry. 198 */ 199 entries += 1; 200 201 entry = calloc(1, sizeof(struct ns_entry)); 202 if (entry == NULL) { 203 perror("ns_entry malloc"); 204 exit(1); 205 } 206 207 entry->ctrlr = ctrlr; 208 entry->ns = ns; 209 entry->num_io_requests = g_queue_depth * entries; 210 211 entry->size_in_ios = ns_size / g_io_size_bytes; 212 entry->io_size_blocks = g_io_size_bytes / sector_size; 213 214 entry->block_size = spdk_nvme_ns_get_sector_size(ns); 215 216 if (g_max_io_size_blocks < entry->io_size_blocks) { 217 g_max_io_size_blocks = entry->io_size_blocks; 218 } 219 220 build_nvme_ns_name(entry->name, sizeof(entry->name), ctrlr, spdk_nvme_ns_get_id(ns)); 221 222 g_num_namespaces++; 223 TAILQ_INSERT_TAIL(&g_namespaces, entry, link); 224 } 225 226 static void 227 unregister_namespaces(void) 228 { 229 struct ns_entry *entry, *tmp; 230 231 TAILQ_FOREACH_SAFE(entry, &g_namespaces, link, tmp) { 232 TAILQ_REMOVE(&g_namespaces, entry, link); 233 free(entry); 234 } 235 } 236 237 static void 238 register_ctrlr(struct spdk_nvme_ctrlr *ctrlr, struct trid_entry *trid_entry) 239 { 240 struct spdk_nvme_ns *ns; 241 struct ctrlr_entry *entry = malloc(sizeof(struct ctrlr_entry)); 242 uint32_t nsid; 243 244 if (entry == NULL) { 245 perror("ctrlr_entry malloc"); 246 exit(1); 247 } 248 249 build_nvme_name(entry->name, sizeof(entry->name), ctrlr); 250 251 entry->ctrlr = ctrlr; 252 entry->trtype = trid_entry->trid.trtype; 253 TAILQ_INSERT_TAIL(&g_controllers, entry, link); 254 255 if (trid_entry->nsid == 0) { 256 for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 257 nsid != 0; nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) { 258 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 259 if (ns == NULL) { 260 continue; 261 } 262 register_ns(ctrlr, ns); 263 } 264 } else { 265 ns = spdk_nvme_ctrlr_get_ns(ctrlr, trid_entry->nsid); 266 if (!ns) { 267 perror("Namespace does not exist."); 268 exit(1); 269 } 270 271 register_ns(ctrlr, ns); 272 } 273 } 274 275 static void 276 abort_complete(void *ctx, const struct spdk_nvme_cpl *cpl) 277 { 278 struct ctrlr_worker_ctx *ctrlr_ctx = ctx; 279 280 ctrlr_ctx->current_queue_depth--; 281 if (spdk_unlikely(spdk_nvme_cpl_is_error(cpl))) { 282 ctrlr_ctx->abort_failed++; 283 } else if ((cpl->cdw0 & 0x1) == 0) { 284 ctrlr_ctx->successful_abort++; 285 } else { 286 ctrlr_ctx->unsuccessful_abort++; 287 } 288 } 289 290 static void 291 abort_task(struct perf_task *task) 292 { 293 struct ns_worker_ctx *ns_ctx = task->ns_ctx; 294 struct ctrlr_worker_ctx *ctrlr_ctx = ns_ctx->ctrlr_ctx; 295 int rc; 296 297 /* Hold mutex to guard ctrlr_ctx->current_queue_depth. */ 298 pthread_mutex_lock(&ctrlr_ctx->mutex); 299 300 rc = spdk_nvme_ctrlr_cmd_abort_ext(ctrlr_ctx->ctrlr, ns_ctx->qpair, task, abort_complete, 301 ctrlr_ctx); 302 303 if (spdk_unlikely(rc != 0)) { 304 ctrlr_ctx->abort_submit_failed++; 305 } else { 306 ctrlr_ctx->current_queue_depth++; 307 ctrlr_ctx->abort_submitted++; 308 } 309 310 pthread_mutex_unlock(&ctrlr_ctx->mutex); 311 } 312 313 static __thread unsigned int seed = 0; 314 315 static inline void 316 submit_single_io(struct perf_task *task) 317 { 318 uint64_t offset_in_ios, lba; 319 int rc; 320 struct ns_worker_ctx *ns_ctx = task->ns_ctx; 321 struct ns_entry *entry = ns_ctx->entry; 322 323 if (g_is_random) { 324 offset_in_ios = rand_r(&seed) % entry->size_in_ios; 325 } else { 326 offset_in_ios = ns_ctx->offset_in_ios++; 327 if (ns_ctx->offset_in_ios == entry->size_in_ios) { 328 ns_ctx->offset_in_ios = 0; 329 } 330 } 331 332 lba = offset_in_ios * entry->io_size_blocks; 333 334 if ((g_rw_percentage == 100) || 335 (g_rw_percentage != 0 && (rand_r(&seed) % 100) < g_rw_percentage)) { 336 rc = spdk_nvme_ns_cmd_read(entry->ns, ns_ctx->qpair, task->buf, 337 lba, entry->io_size_blocks, io_complete, task, 0); 338 } else { 339 rc = spdk_nvme_ns_cmd_write(entry->ns, ns_ctx->qpair, task->buf, 340 lba, entry->io_size_blocks, io_complete, task, 0); 341 } 342 343 if (spdk_unlikely(rc != 0)) { 344 fprintf(stderr, "I/O submission failed\n"); 345 } else { 346 ns_ctx->current_queue_depth++; 347 ns_ctx->io_submitted++; 348 349 if ((ns_ctx->io_submitted % g_abort_interval) == 0) { 350 abort_task(task); 351 } 352 } 353 354 } 355 356 static void 357 io_complete(void *ctx, const struct spdk_nvme_cpl *cpl) 358 { 359 struct perf_task *task = ctx; 360 struct ns_worker_ctx *ns_ctx = task->ns_ctx; 361 362 ns_ctx->current_queue_depth--; 363 if (spdk_unlikely(spdk_nvme_cpl_is_error(cpl))) { 364 ns_ctx->io_failed++; 365 } else { 366 ns_ctx->io_completed++; 367 } 368 369 /* is_draining indicates when time has expired for the test run and we are 370 * just waiting for the previously submitted I/O to complete. In this case, 371 * do not submit a new I/O to replace the one just completed. 372 */ 373 if (spdk_unlikely(ns_ctx->is_draining)) { 374 spdk_dma_free(task->buf); 375 free(task); 376 } else { 377 submit_single_io(task); 378 } 379 } 380 381 static struct perf_task * 382 allocate_task(struct ns_worker_ctx *ns_ctx) 383 { 384 struct perf_task *task; 385 386 task = calloc(1, sizeof(*task)); 387 if (task == NULL) { 388 fprintf(stderr, "Failed to allocate task\n"); 389 exit(1); 390 } 391 392 task->buf = spdk_dma_zmalloc(g_io_size_bytes, 0x200, NULL); 393 if (task->buf == NULL) { 394 free(task); 395 fprintf(stderr, "Failed to allocate task->buf\n"); 396 exit(1); 397 } 398 399 task->ns_ctx = ns_ctx; 400 401 return task; 402 } 403 404 static void 405 submit_io(struct ns_worker_ctx *ns_ctx, int queue_depth) 406 { 407 struct perf_task *task; 408 409 while (queue_depth-- > 0) { 410 task = allocate_task(ns_ctx); 411 submit_single_io(task); 412 } 413 } 414 415 static int 416 work_fn(void *arg) 417 { 418 struct worker_thread *worker = (struct worker_thread *)arg; 419 struct ns_worker_ctx *ns_ctx; 420 struct ctrlr_worker_ctx *ctrlr_ctx; 421 struct ns_entry *ns_entry; 422 struct spdk_nvme_io_qpair_opts opts; 423 uint64_t tsc_end; 424 uint32_t unfinished_ctx; 425 426 /* Allocate queue pair for each namespace. */ 427 TAILQ_FOREACH(ns_ctx, &worker->ns_ctx, link) { 428 ns_entry = ns_ctx->entry; 429 430 spdk_nvme_ctrlr_get_default_io_qpair_opts(ns_entry->ctrlr, &opts, sizeof(opts)); 431 if (opts.io_queue_requests < ns_entry->num_io_requests) { 432 opts.io_queue_requests = ns_entry->num_io_requests; 433 } 434 435 ns_ctx->qpair = spdk_nvme_ctrlr_alloc_io_qpair(ns_entry->ctrlr, &opts, sizeof(opts)); 436 if (ns_ctx->qpair == NULL) { 437 fprintf(stderr, "spdk_nvme_ctrlr_alloc_io_qpair failed\n"); 438 return 1; 439 } 440 } 441 442 tsc_end = spdk_get_ticks() + g_time_in_sec * g_tsc_rate; 443 444 /* Submit initial I/O for each namespace. */ 445 TAILQ_FOREACH(ns_ctx, &worker->ns_ctx, link) { 446 submit_io(ns_ctx, g_queue_depth); 447 } 448 449 while (1) { 450 TAILQ_FOREACH(ns_ctx, &worker->ns_ctx, link) { 451 spdk_nvme_qpair_process_completions(ns_ctx->qpair, 0); 452 } 453 454 if (worker->lcore == g_main_core) { 455 TAILQ_FOREACH(ctrlr_ctx, &worker->ctrlr_ctx, link) { 456 /* Hold mutex to guard ctrlr_ctx->current_queue_depth. */ 457 pthread_mutex_lock(&ctrlr_ctx->mutex); 458 spdk_nvme_ctrlr_process_admin_completions(ctrlr_ctx->ctrlr); 459 pthread_mutex_unlock(&ctrlr_ctx->mutex); 460 } 461 } 462 463 if (spdk_get_ticks() > tsc_end) { 464 break; 465 } 466 } 467 468 do { 469 unfinished_ctx = 0; 470 471 TAILQ_FOREACH(ns_ctx, &worker->ns_ctx, link) { 472 if (!ns_ctx->is_draining) { 473 ns_ctx->is_draining = true; 474 } 475 if (ns_ctx->current_queue_depth > 0) { 476 spdk_nvme_qpair_process_completions(ns_ctx->qpair, 0); 477 if (ns_ctx->current_queue_depth == 0) { 478 spdk_nvme_ctrlr_free_io_qpair(ns_ctx->qpair); 479 } else { 480 unfinished_ctx++; 481 } 482 } 483 } 484 } while (unfinished_ctx > 0); 485 486 if (worker->lcore == g_main_core) { 487 do { 488 unfinished_ctx = 0; 489 490 TAILQ_FOREACH(ctrlr_ctx, &worker->ctrlr_ctx, link) { 491 pthread_mutex_lock(&ctrlr_ctx->mutex); 492 if (ctrlr_ctx->current_queue_depth > 0) { 493 spdk_nvme_ctrlr_process_admin_completions(ctrlr_ctx->ctrlr); 494 if (ctrlr_ctx->current_queue_depth > 0) { 495 unfinished_ctx++; 496 } 497 } 498 pthread_mutex_unlock(&ctrlr_ctx->mutex); 499 } 500 } while (unfinished_ctx > 0); 501 } 502 503 return 0; 504 } 505 506 static void 507 usage(char *program_name) 508 { 509 printf("%s options", program_name); 510 511 printf("\n"); 512 printf("\t[-q io depth]\n"); 513 printf("\t[-o io size in bytes]\n"); 514 printf("\t[-w io pattern type, must be one of\n"); 515 printf("\t\t(read, write, randread, randwrite, rw, randrw)]\n"); 516 printf("\t[-M rwmixread (100 for reads, 0 for writes)]\n"); 517 printf("\t[-t time in seconds]\n"); 518 printf("\t[-c core mask for I/O submission/completion.]\n"); 519 printf("\t\t(default: 1)\n"); 520 printf("\t[-r Transport ID for local PCIe NVMe or NVMeoF]\n"); 521 printf("\t Format: 'key:value [key:value] ...'\n"); 522 printf("\t Keys:\n"); 523 printf("\t trtype Transport type (e.g. PCIe, RDMA)\n"); 524 printf("\t adrfam Address family (e.g. IPv4, IPv6)\n"); 525 printf("\t traddr Transport address (e.g. 0000:04:00.0 for PCIe or 192.168.100.8 for RDMA)\n"); 526 printf("\t trsvcid Transport service identifier (e.g. 4420)\n"); 527 printf("\t subnqn Subsystem NQN (default: %s)\n", SPDK_NVMF_DISCOVERY_NQN); 528 printf("\t Example: -r 'trtype:PCIe traddr:0000:04:00.0' for PCIe or\n"); 529 printf("\t -r 'trtype:RDMA adrfam:IPv4 traddr:192.168.100.8 trsvcid:4420' for NVMeoF\n"); 530 printf("\t[-s DPDK huge memory size in MB.]\n"); 531 printf("\t[-i shared memory group ID]\n"); 532 printf("\t[-a abort interval.]\n"); 533 printf("\t"); 534 spdk_log_usage(stdout, "-T"); 535 #ifdef DEBUG 536 printf("\t[-G enable debug logging]\n"); 537 #else 538 printf("\t[-G enable debug logging (flag disabled, must reconfigure with --enable-debug)]\n"); 539 #endif 540 printf("\t[-l log level]\n"); 541 printf("\t Available log levels:\n"); 542 printf("\t disabled, error, warning, notice, info, debug\n"); 543 } 544 545 static void 546 unregister_trids(void) 547 { 548 struct trid_entry *trid_entry, *tmp; 549 550 TAILQ_FOREACH_SAFE(trid_entry, &g_trid_list, tailq, tmp) { 551 TAILQ_REMOVE(&g_trid_list, trid_entry, tailq); 552 free(trid_entry); 553 } 554 } 555 556 static int 557 add_trid(const char *trid_str) 558 { 559 struct trid_entry *trid_entry; 560 struct spdk_nvme_transport_id *trid; 561 char *ns; 562 563 trid_entry = calloc(1, sizeof(*trid_entry)); 564 if (trid_entry == NULL) { 565 return -1; 566 } 567 568 trid = &trid_entry->trid; 569 trid->trtype = SPDK_NVME_TRANSPORT_PCIE; 570 snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN); 571 572 if (spdk_nvme_transport_id_parse(trid, trid_str) != 0) { 573 fprintf(stderr, "Invalid transport ID format '%s'\n", trid_str); 574 free(trid_entry); 575 return 1; 576 } 577 578 spdk_nvme_transport_id_populate_trstring(trid, 579 spdk_nvme_transport_id_trtype_str(trid->trtype)); 580 581 ns = strcasestr(trid_str, "ns:"); 582 if (ns) { 583 char nsid_str[6]; /* 5 digits maximum in an nsid */ 584 int len; 585 int nsid; 586 587 ns += 3; 588 589 len = strcspn(ns, " \t\n"); 590 if (len > 5) { 591 fprintf(stderr, "NVMe namespace IDs must be 5 digits or less\n"); 592 free(trid_entry); 593 return 1; 594 } 595 596 memcpy(nsid_str, ns, len); 597 nsid_str[len] = '\0'; 598 599 nsid = spdk_strtol(nsid_str, 10); 600 if (nsid <= 0 || nsid > 65535) { 601 fprintf(stderr, "NVMe namespace IDs must be less than 65536 and greater than 0\n"); 602 free(trid_entry); 603 return 1; 604 } 605 606 trid_entry->nsid = (uint16_t)nsid; 607 } 608 609 TAILQ_INSERT_TAIL(&g_trid_list, trid_entry, tailq); 610 return 0; 611 } 612 613 static int 614 parse_args(int argc, char **argv) 615 { 616 int op; 617 long int val; 618 int rc; 619 620 while ((op = getopt(argc, argv, "a:c:i:l:o:q:r:s:t:w:GM:T:")) != -1) { 621 switch (op) { 622 case 'a': 623 case 'i': 624 case 'o': 625 case 'q': 626 case 's': 627 case 't': 628 case 'M': 629 val = spdk_strtol(optarg, 10); 630 if (val < 0) { 631 fprintf(stderr, "Converting a string to integer failed\n"); 632 return val; 633 } 634 switch (op) { 635 case 'a': 636 g_abort_interval = val; 637 break; 638 case 'i': 639 g_shm_id = val; 640 break; 641 case 'o': 642 g_io_size_bytes = val; 643 break; 644 case 'q': 645 g_queue_depth = val; 646 break; 647 case 's': 648 g_dpdk_mem = val; 649 break; 650 case 't': 651 g_time_in_sec = val; 652 break; 653 case 'M': 654 g_rw_percentage = val; 655 g_mix_specified = true; 656 break; 657 } 658 break; 659 case 'c': 660 g_core_mask = optarg; 661 break; 662 case 'r': 663 if (add_trid(optarg)) { 664 usage(argv[0]); 665 return 1; 666 } 667 break; 668 case 'w': 669 g_workload_type = optarg; 670 break; 671 case 'G': 672 #ifndef DEBUG 673 fprintf(stderr, "%s must be configured with --enable-debug for -G flag\n", 674 argv[0]); 675 usage(argv[0]); 676 return 1; 677 #else 678 spdk_log_set_flag("nvme"); 679 spdk_log_set_print_level(SPDK_LOG_DEBUG); 680 break; 681 #endif 682 case 'T': 683 rc = spdk_log_set_flag(optarg); 684 if (rc < 0) { 685 fprintf(stderr, "unknown flag\n"); 686 usage(argv[0]); 687 exit(EXIT_FAILURE); 688 } 689 #ifdef DEBUG 690 spdk_log_set_print_level(SPDK_LOG_DEBUG); 691 #endif 692 break; 693 case 'l': 694 if (!strcmp(optarg, "disabled")) { 695 spdk_log_set_print_level(SPDK_LOG_DISABLED); 696 } else if (!strcmp(optarg, "error")) { 697 spdk_log_set_print_level(SPDK_LOG_ERROR); 698 } else if (!strcmp(optarg, "warning")) { 699 spdk_log_set_print_level(SPDK_LOG_WARN); 700 } else if (!strcmp(optarg, "notice")) { 701 spdk_log_set_print_level(SPDK_LOG_NOTICE); 702 } else if (!strcmp(optarg, "info")) { 703 spdk_log_set_print_level(SPDK_LOG_INFO); 704 } else if (!strcmp(optarg, "debug")) { 705 spdk_log_set_print_level(SPDK_LOG_DEBUG); 706 } else { 707 fprintf(stderr, "Unrecognized log level: %s\n", optarg); 708 return 1; 709 } 710 break; 711 default: 712 usage(argv[0]); 713 return 1; 714 } 715 } 716 717 if (!g_queue_depth) { 718 fprintf(stderr, "missing -q (queue size) operand\n"); 719 usage(argv[0]); 720 return 1; 721 } 722 if (!g_io_size_bytes) { 723 fprintf(stderr, "missing -o (block size) operand\n"); 724 usage(argv[0]); 725 return 1; 726 } 727 if (!g_workload_type) { 728 fprintf(stderr, "missing -t (test time in seconds) operand\n"); 729 usage(argv[0]); 730 return 1; 731 } 732 733 if (!g_time_in_sec) { 734 usage(argv[0]); 735 return 1; 736 } 737 738 if (strncmp(g_workload_type, "rand", 4) == 0) { 739 g_is_random = 1; 740 g_workload_type = &g_workload_type[4]; 741 } 742 743 if (strcmp(g_workload_type, "read") == 0 || strcmp(g_workload_type, "write") == 0) { 744 g_rw_percentage = strcmp(g_workload_type, "read") == 0 ? 100 : 0; 745 if (g_mix_specified) { 746 fprintf(stderr, "Ignoring -M option... Please use -M option" 747 " only when using rw or randrw.\n"); 748 } 749 } else if (strcmp(g_workload_type, "rw") == 0) { 750 if (g_rw_percentage < 0 || g_rw_percentage > 100) { 751 fprintf(stderr, 752 "-M must be specified to value from 0 to 100 " 753 "for rw or randrw.\n"); 754 return 1; 755 } 756 } else { 757 fprintf(stderr, 758 "io pattern type must be one of\n" 759 "(read, write, randread, randwrite, rw, randrw)\n"); 760 return 1; 761 } 762 763 if (TAILQ_EMPTY(&g_trid_list)) { 764 /* If no transport IDs specified, default to enumerating all local PCIe devices */ 765 add_trid("trtype:PCIe"); 766 } else { 767 struct trid_entry *trid_entry, *trid_entry_tmp; 768 769 g_no_pci = true; 770 /* check whether there is local PCIe type */ 771 TAILQ_FOREACH_SAFE(trid_entry, &g_trid_list, tailq, trid_entry_tmp) { 772 if (trid_entry->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) { 773 g_no_pci = false; 774 break; 775 } 776 } 777 } 778 779 return 0; 780 } 781 782 static int 783 register_workers(void) 784 { 785 uint32_t i; 786 struct worker_thread *worker; 787 788 SPDK_ENV_FOREACH_CORE(i) { 789 worker = calloc(1, sizeof(*worker)); 790 if (worker == NULL) { 791 fprintf(stderr, "Unable to allocate worker\n"); 792 return -1; 793 } 794 795 TAILQ_INIT(&worker->ns_ctx); 796 TAILQ_INIT(&worker->ctrlr_ctx); 797 worker->lcore = i; 798 TAILQ_INSERT_TAIL(&g_workers, worker, link); 799 g_num_workers++; 800 } 801 802 return 0; 803 } 804 805 static void 806 unregister_workers(void) 807 { 808 struct worker_thread *worker, *tmp_worker; 809 struct ns_worker_ctx *ns_ctx, *tmp_ns_ctx; 810 struct ctrlr_worker_ctx *ctrlr_ctx, *tmp_ctrlr_ctx; 811 812 /* Free namespace context and worker thread */ 813 TAILQ_FOREACH_SAFE(worker, &g_workers, link, tmp_worker) { 814 TAILQ_REMOVE(&g_workers, worker, link); 815 816 TAILQ_FOREACH_SAFE(ns_ctx, &worker->ns_ctx, link, tmp_ns_ctx) { 817 TAILQ_REMOVE(&worker->ns_ctx, ns_ctx, link); 818 printf("NS: %s I/O completed: %" PRIu64 ", failed: %" PRIu64 "\n", 819 ns_ctx->entry->name, ns_ctx->io_completed, ns_ctx->io_failed); 820 free(ns_ctx); 821 } 822 823 TAILQ_FOREACH_SAFE(ctrlr_ctx, &worker->ctrlr_ctx, link, tmp_ctrlr_ctx) { 824 TAILQ_REMOVE(&worker->ctrlr_ctx, ctrlr_ctx, link); 825 printf("CTRLR: %s abort submitted %" PRIu64 ", failed to submit %" PRIu64 "\n", 826 ctrlr_ctx->entry->name, ctrlr_ctx->abort_submitted, 827 ctrlr_ctx->abort_submit_failed); 828 printf("\t success %" PRIu64 ", unsuccess %" PRIu64 ", failed %" PRIu64 "\n", 829 ctrlr_ctx->successful_abort, ctrlr_ctx->unsuccessful_abort, 830 ctrlr_ctx->abort_failed); 831 free(ctrlr_ctx); 832 } 833 834 free(worker); 835 } 836 } 837 838 static bool 839 probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 840 struct spdk_nvme_ctrlr_opts *opts) 841 { 842 uint16_t min_aq_size; 843 844 /* We need to make sure the admin queue is big enough to handle all of the aborts that 845 * will be sent by this test app. We add a few extra entries to account for any admin 846 * commands other than the aborts. */ 847 min_aq_size = spdk_divide_round_up(g_queue_depth, g_abort_interval) + 8; 848 opts->admin_queue_size = spdk_max(opts->admin_queue_size, min_aq_size); 849 850 return true; 851 } 852 853 static void 854 attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 855 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts) 856 { 857 struct trid_entry *trid_entry = cb_ctx; 858 struct spdk_pci_addr pci_addr; 859 struct spdk_pci_device *pci_dev; 860 struct spdk_pci_id pci_id; 861 862 if (trid->trtype != SPDK_NVME_TRANSPORT_PCIE) { 863 printf("Attached to NVMe over Fabrics controller at %s:%s: %s\n", 864 trid->traddr, trid->trsvcid, 865 trid->subnqn); 866 } else { 867 if (spdk_pci_addr_parse(&pci_addr, trid->traddr)) { 868 return; 869 } 870 871 pci_dev = spdk_nvme_ctrlr_get_pci_device(ctrlr); 872 if (!pci_dev) { 873 return; 874 } 875 876 pci_id = spdk_pci_device_get_id(pci_dev); 877 878 printf("Attached to NVMe Controller at %s [%04x:%04x]\n", 879 trid->traddr, 880 pci_id.vendor_id, pci_id.device_id); 881 } 882 883 register_ctrlr(ctrlr, trid_entry); 884 } 885 886 static int 887 register_controllers(void) 888 { 889 struct trid_entry *trid_entry; 890 891 printf("Initializing NVMe Controllers\n"); 892 893 TAILQ_FOREACH(trid_entry, &g_trid_list, tailq) { 894 if (spdk_nvme_probe(&trid_entry->trid, trid_entry, probe_cb, attach_cb, NULL) != 0) { 895 fprintf(stderr, "spdk_nvme_probe() failed for transport address '%s'\n", 896 trid_entry->trid.traddr); 897 return -1; 898 } 899 } 900 901 return 0; 902 } 903 904 static void 905 unregister_controllers(void) 906 { 907 struct ctrlr_entry *entry, *tmp; 908 struct spdk_nvme_detach_ctx *detach_ctx = NULL; 909 910 TAILQ_FOREACH_SAFE(entry, &g_controllers, link, tmp) { 911 TAILQ_REMOVE(&g_controllers, entry, link); 912 spdk_nvme_detach_async(entry->ctrlr, &detach_ctx); 913 free(entry); 914 } 915 916 if (detach_ctx) { 917 spdk_nvme_detach_poll(detach_ctx); 918 } 919 } 920 921 static int 922 associate_main_worker_with_ctrlr(void) 923 { 924 struct ctrlr_entry *entry; 925 struct worker_thread *worker; 926 struct ctrlr_worker_ctx *ctrlr_ctx; 927 928 TAILQ_FOREACH(worker, &g_workers, link) { 929 if (worker->lcore == g_main_core) { 930 break; 931 } 932 } 933 934 if (!worker) { 935 return -1; 936 } 937 938 TAILQ_FOREACH(entry, &g_controllers, link) { 939 ctrlr_ctx = calloc(1, sizeof(struct ctrlr_worker_ctx)); 940 if (!ctrlr_ctx) { 941 return -1; 942 } 943 944 pthread_mutex_init(&ctrlr_ctx->mutex, NULL); 945 ctrlr_ctx->entry = entry; 946 ctrlr_ctx->ctrlr = entry->ctrlr; 947 948 TAILQ_INSERT_TAIL(&worker->ctrlr_ctx, ctrlr_ctx, link); 949 } 950 951 return 0; 952 } 953 954 static struct ctrlr_worker_ctx * 955 get_ctrlr_worker_ctx(struct spdk_nvme_ctrlr *ctrlr) 956 { 957 struct worker_thread *worker; 958 struct ctrlr_worker_ctx *ctrlr_ctx; 959 960 TAILQ_FOREACH(worker, &g_workers, link) { 961 if (worker->lcore == g_main_core) { 962 break; 963 } 964 } 965 966 if (!worker) { 967 return NULL; 968 } 969 970 TAILQ_FOREACH(ctrlr_ctx, &worker->ctrlr_ctx, link) { 971 if (ctrlr_ctx->ctrlr == ctrlr) { 972 return ctrlr_ctx; 973 } 974 } 975 976 return NULL; 977 } 978 979 static int 980 associate_workers_with_ns(void) 981 { 982 struct ns_entry *entry = TAILQ_FIRST(&g_namespaces); 983 struct worker_thread *worker = TAILQ_FIRST(&g_workers); 984 struct ns_worker_ctx *ns_ctx; 985 int i, count; 986 987 count = g_num_namespaces > g_num_workers ? g_num_namespaces : g_num_workers; 988 989 for (i = 0; i < count; i++) { 990 if (entry == NULL) { 991 break; 992 } 993 994 ns_ctx = calloc(1, sizeof(struct ns_worker_ctx)); 995 if (!ns_ctx) { 996 return -1; 997 } 998 999 printf("Associating %s with lcore %d\n", entry->name, worker->lcore); 1000 ns_ctx->entry = entry; 1001 ns_ctx->ctrlr_ctx = get_ctrlr_worker_ctx(entry->ctrlr); 1002 if (!ns_ctx->ctrlr_ctx) { 1003 free(ns_ctx); 1004 return -1; 1005 } 1006 1007 TAILQ_INSERT_TAIL(&worker->ns_ctx, ns_ctx, link); 1008 1009 worker = TAILQ_NEXT(worker, link); 1010 if (worker == NULL) { 1011 worker = TAILQ_FIRST(&g_workers); 1012 } 1013 1014 entry = TAILQ_NEXT(entry, link); 1015 if (entry == NULL) { 1016 entry = TAILQ_FIRST(&g_namespaces); 1017 } 1018 } 1019 1020 return 0; 1021 } 1022 1023 int 1024 main(int argc, char **argv) 1025 { 1026 int rc; 1027 struct worker_thread *worker, *main_worker; 1028 struct spdk_env_opts opts; 1029 1030 rc = parse_args(argc, argv); 1031 if (rc != 0) { 1032 return rc; 1033 } 1034 1035 spdk_env_opts_init(&opts); 1036 opts.name = "abort"; 1037 opts.shm_id = g_shm_id; 1038 if (g_core_mask) { 1039 opts.core_mask = g_core_mask; 1040 } 1041 1042 if (g_dpdk_mem) { 1043 opts.mem_size = g_dpdk_mem; 1044 } 1045 if (g_no_pci) { 1046 opts.no_pci = g_no_pci; 1047 } 1048 if (spdk_env_init(&opts) < 0) { 1049 fprintf(stderr, "Unable to initialize SPDK env\n"); 1050 unregister_trids(); 1051 return -1; 1052 } 1053 1054 g_tsc_rate = spdk_get_ticks_hz(); 1055 1056 if (register_workers() != 0) { 1057 rc = -1; 1058 goto cleanup; 1059 } 1060 1061 if (register_controllers() != 0) { 1062 rc = -1; 1063 goto cleanup; 1064 } 1065 1066 if (g_warn) { 1067 printf("WARNING: Some requested NVMe devices were skipped\n"); 1068 } 1069 1070 if (g_num_namespaces == 0) { 1071 fprintf(stderr, "No valid NVMe controllers found\n"); 1072 rc = -1; 1073 goto cleanup; 1074 } 1075 1076 if (associate_main_worker_with_ctrlr() != 0) { 1077 rc = -1; 1078 goto cleanup; 1079 } 1080 1081 if (associate_workers_with_ns() != 0) { 1082 rc = -1; 1083 goto cleanup; 1084 } 1085 1086 printf("Initialization complete. Launching workers.\n"); 1087 1088 /* Launch all of the secondary workers */ 1089 g_main_core = spdk_env_get_current_core(); 1090 main_worker = NULL; 1091 TAILQ_FOREACH(worker, &g_workers, link) { 1092 if (worker->lcore != g_main_core) { 1093 spdk_env_thread_launch_pinned(worker->lcore, work_fn, worker); 1094 } else { 1095 assert(main_worker == NULL); 1096 main_worker = worker; 1097 } 1098 } 1099 1100 assert(main_worker != NULL); 1101 rc = work_fn(main_worker); 1102 1103 spdk_env_thread_wait_all(); 1104 1105 cleanup: 1106 unregister_trids(); 1107 unregister_workers(); 1108 unregister_namespaces(); 1109 unregister_controllers(); 1110 1111 spdk_env_fini(); 1112 1113 if (rc != 0) { 1114 fprintf(stderr, "%s: errors occurred\n", argv[0]); 1115 } 1116 1117 return rc; 1118 } 1119