1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2020 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 8 #include "spdk/env.h" 9 #include "spdk/log.h" 10 #include "spdk/nvme.h" 11 #include "spdk/queue.h" 12 #include "spdk/string.h" 13 #include "spdk/util.h" 14 #include "spdk/likely.h" 15 16 struct ctrlr_entry { 17 struct spdk_nvme_ctrlr *ctrlr; 18 enum spdk_nvme_transport_type trtype; 19 20 TAILQ_ENTRY(ctrlr_entry) link; 21 char name[1024]; 22 }; 23 24 struct ns_entry { 25 struct spdk_nvme_ctrlr *ctrlr; 26 struct spdk_nvme_ns *ns; 27 28 TAILQ_ENTRY(ns_entry) link; 29 uint32_t io_size_blocks; 30 uint32_t num_io_requests; 31 uint64_t size_in_ios; 32 uint32_t block_size; 33 char name[1024]; 34 }; 35 36 struct ctrlr_worker_ctx { 37 pthread_mutex_t mutex; 38 struct ctrlr_entry *entry; 39 uint64_t abort_submitted; 40 uint64_t abort_submit_failed; 41 uint64_t successful_abort; 42 uint64_t unsuccessful_abort; 43 uint64_t abort_failed; 44 uint64_t current_queue_depth; 45 struct spdk_nvme_ctrlr *ctrlr; 46 TAILQ_ENTRY(ctrlr_worker_ctx) link; 47 }; 48 49 struct ns_worker_ctx { 50 struct ns_entry *entry; 51 uint64_t io_submitted; 52 uint64_t io_completed; 53 uint64_t io_aborted; 54 uint64_t io_failed; 55 uint64_t current_queue_depth; 56 uint64_t offset_in_ios; 57 bool is_draining; 58 struct spdk_nvme_qpair *qpair; 59 struct ctrlr_worker_ctx *ctrlr_ctx; 60 TAILQ_ENTRY(ns_worker_ctx) link; 61 }; 62 63 struct perf_task { 64 struct ns_worker_ctx *ns_ctx; 65 void *buf; 66 }; 67 68 struct worker_thread { 69 TAILQ_HEAD(, ns_worker_ctx) ns_ctx; 70 TAILQ_HEAD(, ctrlr_worker_ctx) ctrlr_ctx; 71 TAILQ_ENTRY(worker_thread) link; 72 unsigned lcore; 73 }; 74 75 static const char *g_workload_type = "read"; 76 static TAILQ_HEAD(, ctrlr_entry) g_controllers = TAILQ_HEAD_INITIALIZER(g_controllers); 77 static TAILQ_HEAD(, ns_entry) g_namespaces = TAILQ_HEAD_INITIALIZER(g_namespaces); 78 static int g_num_namespaces; 79 static TAILQ_HEAD(, worker_thread) g_workers = TAILQ_HEAD_INITIALIZER(g_workers); 80 static int g_num_workers = 0; 81 static uint32_t g_main_core; 82 83 static int g_abort_interval = 1; 84 85 static uint64_t g_tsc_rate; 86 87 static uint32_t g_io_size_bytes = 131072; 88 static uint32_t g_max_io_size_blocks; 89 static int g_rw_percentage = -1; 90 static int g_is_random; 91 static int g_queue_depth = 128; 92 static int g_time_in_sec = 3; 93 static int g_dpdk_mem; 94 static int g_shm_id = -1; 95 static bool g_no_pci; 96 static bool g_warn; 97 static bool g_mix_specified; 98 99 static const char *g_core_mask; 100 101 struct trid_entry { 102 struct spdk_nvme_transport_id trid; 103 uint16_t nsid; 104 TAILQ_ENTRY(trid_entry) tailq; 105 }; 106 107 static TAILQ_HEAD(, trid_entry) g_trid_list = TAILQ_HEAD_INITIALIZER(g_trid_list); 108 109 static void io_complete(void *ctx, const struct spdk_nvme_cpl *cpl); 110 111 static int 112 build_nvme_name(char *name, size_t length, struct spdk_nvme_ctrlr *ctrlr) 113 { 114 const struct spdk_nvme_transport_id *trid; 115 int res = 0; 116 117 trid = spdk_nvme_ctrlr_get_transport_id(ctrlr); 118 119 switch (trid->trtype) { 120 case SPDK_NVME_TRANSPORT_PCIE: 121 res = snprintf(name, length, "PCIE (%s)", trid->traddr); 122 break; 123 case SPDK_NVME_TRANSPORT_RDMA: 124 res = snprintf(name, length, "RDMA (addr:%s subnqn:%s)", trid->traddr, trid->subnqn); 125 break; 126 case SPDK_NVME_TRANSPORT_TCP: 127 res = snprintf(name, length, "TCP (addr:%s subnqn:%s)", trid->traddr, trid->subnqn); 128 break; 129 case SPDK_NVME_TRANSPORT_CUSTOM: 130 res = snprintf(name, length, "CUSTOM (%s)", trid->traddr); 131 break; 132 133 default: 134 fprintf(stderr, "Unknown transport type %d\n", trid->trtype); 135 break; 136 } 137 return res; 138 } 139 140 static void 141 build_nvme_ns_name(char *name, size_t length, struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid) 142 { 143 int res = 0; 144 145 res = build_nvme_name(name, length, ctrlr); 146 if (res > 0) { 147 snprintf(name + res, length - res, " NSID %u", nsid); 148 } 149 150 } 151 152 static void 153 register_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns) 154 { 155 struct ns_entry *entry; 156 const struct spdk_nvme_ctrlr_data *cdata; 157 uint32_t max_xfer_size, entries, sector_size; 158 uint64_t ns_size; 159 struct spdk_nvme_io_qpair_opts opts; 160 161 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 162 163 if (!spdk_nvme_ns_is_active(ns)) { 164 printf("Controller %-20.20s (%-20.20s): Skipping inactive NS %u\n", 165 cdata->mn, cdata->sn, 166 spdk_nvme_ns_get_id(ns)); 167 g_warn = true; 168 return; 169 } 170 171 ns_size = spdk_nvme_ns_get_size(ns); 172 sector_size = spdk_nvme_ns_get_sector_size(ns); 173 174 if (ns_size < g_io_size_bytes || sector_size > g_io_size_bytes) { 175 printf("WARNING: controller %-20.20s (%-20.20s) ns %u has invalid " 176 "ns size %" PRIu64 " / block size %u for I/O size %u\n", 177 cdata->mn, cdata->sn, spdk_nvme_ns_get_id(ns), 178 ns_size, spdk_nvme_ns_get_sector_size(ns), g_io_size_bytes); 179 g_warn = true; 180 return; 181 } 182 183 max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns); 184 spdk_nvme_ctrlr_get_default_io_qpair_opts(ctrlr, &opts, sizeof(opts)); 185 /* NVMe driver may add additional entries based on 186 * stripe size and maximum transfer size, we assume 187 * 1 more entry be used for stripe. 188 */ 189 entries = (g_io_size_bytes - 1) / max_xfer_size + 2; 190 if ((g_queue_depth * entries) > opts.io_queue_size) { 191 printf("controller IO queue size %u less than required\n", 192 opts.io_queue_size); 193 printf("Consider using lower queue depth or small IO size because " 194 "IO requests may be queued at the NVMe driver.\n"); 195 } 196 /* For requests which have children requests, parent request itself 197 * will also occupy 1 entry. 198 */ 199 entries += 1; 200 201 entry = calloc(1, sizeof(struct ns_entry)); 202 if (entry == NULL) { 203 perror("ns_entry malloc"); 204 exit(1); 205 } 206 207 entry->ctrlr = ctrlr; 208 entry->ns = ns; 209 entry->num_io_requests = g_queue_depth * entries; 210 211 entry->size_in_ios = ns_size / g_io_size_bytes; 212 entry->io_size_blocks = g_io_size_bytes / sector_size; 213 214 entry->block_size = spdk_nvme_ns_get_sector_size(ns); 215 216 if (g_max_io_size_blocks < entry->io_size_blocks) { 217 g_max_io_size_blocks = entry->io_size_blocks; 218 } 219 220 build_nvme_ns_name(entry->name, sizeof(entry->name), ctrlr, spdk_nvme_ns_get_id(ns)); 221 222 g_num_namespaces++; 223 TAILQ_INSERT_TAIL(&g_namespaces, entry, link); 224 } 225 226 static void 227 unregister_namespaces(void) 228 { 229 struct ns_entry *entry, *tmp; 230 231 TAILQ_FOREACH_SAFE(entry, &g_namespaces, link, tmp) { 232 TAILQ_REMOVE(&g_namespaces, entry, link); 233 free(entry); 234 } 235 } 236 237 static void 238 register_ctrlr(struct spdk_nvme_ctrlr *ctrlr, struct trid_entry *trid_entry) 239 { 240 struct spdk_nvme_ns *ns; 241 struct ctrlr_entry *entry = malloc(sizeof(struct ctrlr_entry)); 242 uint32_t nsid; 243 244 if (entry == NULL) { 245 perror("ctrlr_entry malloc"); 246 exit(1); 247 } 248 249 build_nvme_name(entry->name, sizeof(entry->name), ctrlr); 250 251 entry->ctrlr = ctrlr; 252 entry->trtype = trid_entry->trid.trtype; 253 TAILQ_INSERT_TAIL(&g_controllers, entry, link); 254 255 if (trid_entry->nsid == 0) { 256 for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 257 nsid != 0; nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) { 258 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 259 if (ns == NULL) { 260 continue; 261 } 262 register_ns(ctrlr, ns); 263 } 264 } else { 265 ns = spdk_nvme_ctrlr_get_ns(ctrlr, trid_entry->nsid); 266 if (!ns) { 267 perror("Namespace does not exist."); 268 exit(1); 269 } 270 271 register_ns(ctrlr, ns); 272 } 273 } 274 275 static void 276 abort_complete(void *ctx, const struct spdk_nvme_cpl *cpl) 277 { 278 struct ctrlr_worker_ctx *ctrlr_ctx = ctx; 279 280 ctrlr_ctx->current_queue_depth--; 281 if (spdk_unlikely(spdk_nvme_cpl_is_error(cpl))) { 282 ctrlr_ctx->abort_failed++; 283 } else if ((cpl->cdw0 & 0x1) == 0) { 284 ctrlr_ctx->successful_abort++; 285 } else { 286 ctrlr_ctx->unsuccessful_abort++; 287 } 288 } 289 290 static void 291 abort_task(struct perf_task *task) 292 { 293 struct ns_worker_ctx *ns_ctx = task->ns_ctx; 294 struct ctrlr_worker_ctx *ctrlr_ctx = ns_ctx->ctrlr_ctx; 295 int rc; 296 297 /* Hold mutex to guard ctrlr_ctx->current_queue_depth. */ 298 pthread_mutex_lock(&ctrlr_ctx->mutex); 299 300 rc = spdk_nvme_ctrlr_cmd_abort_ext(ctrlr_ctx->ctrlr, ns_ctx->qpair, task, abort_complete, 301 ctrlr_ctx); 302 303 if (spdk_unlikely(rc != 0)) { 304 ctrlr_ctx->abort_submit_failed++; 305 } else { 306 ctrlr_ctx->current_queue_depth++; 307 ctrlr_ctx->abort_submitted++; 308 } 309 310 pthread_mutex_unlock(&ctrlr_ctx->mutex); 311 } 312 313 static __thread unsigned int seed = 0; 314 315 static inline void 316 submit_single_io(struct perf_task *task) 317 { 318 uint64_t offset_in_ios, lba; 319 int rc; 320 struct ns_worker_ctx *ns_ctx = task->ns_ctx; 321 struct ns_entry *entry = ns_ctx->entry; 322 323 if (g_is_random) { 324 offset_in_ios = rand_r(&seed) % entry->size_in_ios; 325 } else { 326 offset_in_ios = ns_ctx->offset_in_ios++; 327 if (ns_ctx->offset_in_ios == entry->size_in_ios) { 328 ns_ctx->offset_in_ios = 0; 329 } 330 } 331 332 lba = offset_in_ios * entry->io_size_blocks; 333 334 if ((g_rw_percentage == 100) || 335 (g_rw_percentage != 0 && (rand_r(&seed) % 100) < g_rw_percentage)) { 336 rc = spdk_nvme_ns_cmd_read(entry->ns, ns_ctx->qpair, task->buf, 337 lba, entry->io_size_blocks, io_complete, task, 0); 338 } else { 339 rc = spdk_nvme_ns_cmd_write(entry->ns, ns_ctx->qpair, task->buf, 340 lba, entry->io_size_blocks, io_complete, task, 0); 341 } 342 343 if (spdk_unlikely(rc != 0)) { 344 fprintf(stderr, "I/O submission failed\n"); 345 } else { 346 ns_ctx->current_queue_depth++; 347 ns_ctx->io_submitted++; 348 349 if ((ns_ctx->io_submitted % g_abort_interval) == 0) { 350 abort_task(task); 351 } 352 } 353 354 } 355 356 static void 357 io_complete(void *ctx, const struct spdk_nvme_cpl *cpl) 358 { 359 struct perf_task *task = ctx; 360 struct ns_worker_ctx *ns_ctx = task->ns_ctx; 361 362 ns_ctx->current_queue_depth--; 363 if (spdk_unlikely(spdk_nvme_cpl_is_error(cpl))) { 364 ns_ctx->io_failed++; 365 } else { 366 ns_ctx->io_completed++; 367 } 368 369 /* is_draining indicates when time has expired for the test run and we are 370 * just waiting for the previously submitted I/O to complete. In this case, 371 * do not submit a new I/O to replace the one just completed. 372 */ 373 if (spdk_unlikely(ns_ctx->is_draining)) { 374 spdk_dma_free(task->buf); 375 free(task); 376 } else { 377 submit_single_io(task); 378 } 379 } 380 381 static struct perf_task * 382 allocate_task(struct ns_worker_ctx *ns_ctx) 383 { 384 struct perf_task *task; 385 386 task = calloc(1, sizeof(*task)); 387 if (task == NULL) { 388 fprintf(stderr, "Failed to allocate task\n"); 389 exit(1); 390 } 391 392 task->buf = spdk_dma_zmalloc(g_io_size_bytes, 0x200, NULL); 393 if (task->buf == NULL) { 394 free(task); 395 fprintf(stderr, "Failed to allocate task->buf\n"); 396 exit(1); 397 } 398 399 task->ns_ctx = ns_ctx; 400 401 return task; 402 } 403 404 static void 405 submit_io(struct ns_worker_ctx *ns_ctx, int queue_depth) 406 { 407 struct perf_task *task; 408 409 while (queue_depth-- > 0) { 410 task = allocate_task(ns_ctx); 411 submit_single_io(task); 412 } 413 } 414 415 static int 416 work_fn(void *arg) 417 { 418 struct worker_thread *worker = (struct worker_thread *)arg; 419 struct ns_worker_ctx *ns_ctx; 420 struct ctrlr_worker_ctx *ctrlr_ctx; 421 struct ns_entry *ns_entry; 422 struct spdk_nvme_io_qpair_opts opts; 423 uint64_t tsc_end; 424 uint32_t unfinished_ctx; 425 426 /* Allocate queue pair for each namespace. */ 427 TAILQ_FOREACH(ns_ctx, &worker->ns_ctx, link) { 428 ns_entry = ns_ctx->entry; 429 430 spdk_nvme_ctrlr_get_default_io_qpair_opts(ns_entry->ctrlr, &opts, sizeof(opts)); 431 if (opts.io_queue_requests < ns_entry->num_io_requests) { 432 opts.io_queue_requests = ns_entry->num_io_requests; 433 } 434 435 ns_ctx->qpair = spdk_nvme_ctrlr_alloc_io_qpair(ns_entry->ctrlr, &opts, sizeof(opts)); 436 if (ns_ctx->qpair == NULL) { 437 fprintf(stderr, "spdk_nvme_ctrlr_alloc_io_qpair failed\n"); 438 return 1; 439 } 440 } 441 442 tsc_end = spdk_get_ticks() + g_time_in_sec * g_tsc_rate; 443 444 /* Submit initial I/O for each namespace. */ 445 TAILQ_FOREACH(ns_ctx, &worker->ns_ctx, link) { 446 submit_io(ns_ctx, g_queue_depth); 447 } 448 449 while (1) { 450 TAILQ_FOREACH(ns_ctx, &worker->ns_ctx, link) { 451 spdk_nvme_qpair_process_completions(ns_ctx->qpair, 0); 452 } 453 454 if (worker->lcore == g_main_core) { 455 TAILQ_FOREACH(ctrlr_ctx, &worker->ctrlr_ctx, link) { 456 /* Hold mutex to guard ctrlr_ctx->current_queue_depth. */ 457 pthread_mutex_lock(&ctrlr_ctx->mutex); 458 spdk_nvme_ctrlr_process_admin_completions(ctrlr_ctx->ctrlr); 459 pthread_mutex_unlock(&ctrlr_ctx->mutex); 460 } 461 } 462 463 if (spdk_get_ticks() > tsc_end) { 464 break; 465 } 466 } 467 468 do { 469 unfinished_ctx = 0; 470 471 TAILQ_FOREACH(ns_ctx, &worker->ns_ctx, link) { 472 if (!ns_ctx->is_draining) { 473 ns_ctx->is_draining = true; 474 } 475 if (ns_ctx->current_queue_depth > 0) { 476 spdk_nvme_qpair_process_completions(ns_ctx->qpair, 0); 477 unfinished_ctx++; 478 } else { 479 spdk_nvme_ctrlr_free_io_qpair(ns_ctx->qpair); 480 } 481 } 482 } while (unfinished_ctx > 0); 483 484 if (worker->lcore == g_main_core) { 485 do { 486 unfinished_ctx = 0; 487 488 TAILQ_FOREACH(ctrlr_ctx, &worker->ctrlr_ctx, link) { 489 pthread_mutex_lock(&ctrlr_ctx->mutex); 490 if (ctrlr_ctx->current_queue_depth > 0) { 491 spdk_nvme_ctrlr_process_admin_completions(ctrlr_ctx->ctrlr); 492 unfinished_ctx++; 493 } 494 pthread_mutex_unlock(&ctrlr_ctx->mutex); 495 } 496 } while (unfinished_ctx > 0); 497 } 498 499 return 0; 500 } 501 502 static void 503 usage(char *program_name) 504 { 505 printf("%s options", program_name); 506 507 printf("\n"); 508 printf("\t[-q io depth]\n"); 509 printf("\t[-o io size in bytes]\n"); 510 printf("\t[-w io pattern type, must be one of\n"); 511 printf("\t\t(read, write, randread, randwrite, rw, randrw)]\n"); 512 printf("\t[-M rwmixread (100 for reads, 0 for writes)]\n"); 513 printf("\t[-t time in seconds]\n"); 514 printf("\t[-c core mask for I/O submission/completion.]\n"); 515 printf("\t\t(default: 1)\n"); 516 printf("\t[-r Transport ID for local PCIe NVMe or NVMeoF]\n"); 517 printf("\t Format: 'key:value [key:value] ...'\n"); 518 printf("\t Keys:\n"); 519 printf("\t trtype Transport type (e.g. PCIe, RDMA)\n"); 520 printf("\t adrfam Address family (e.g. IPv4, IPv6)\n"); 521 printf("\t traddr Transport address (e.g. 0000:04:00.0 for PCIe or 192.168.100.8 for RDMA)\n"); 522 printf("\t trsvcid Transport service identifier (e.g. 4420)\n"); 523 printf("\t subnqn Subsystem NQN (default: %s)\n", SPDK_NVMF_DISCOVERY_NQN); 524 printf("\t Example: -r 'trtype:PCIe traddr:0000:04:00.0' for PCIe or\n"); 525 printf("\t -r 'trtype:RDMA adrfam:IPv4 traddr:192.168.100.8 trsvcid:4420' for NVMeoF\n"); 526 printf("\t[-s DPDK huge memory size in MB.]\n"); 527 printf("\t[-i shared memory group ID]\n"); 528 printf("\t[-a abort interval.]\n"); 529 printf("\t"); 530 spdk_log_usage(stdout, "-T"); 531 #ifdef DEBUG 532 printf("\t[-G enable debug logging]\n"); 533 #else 534 printf("\t[-G enable debug logging (flag disabled, must reconfigure with --enable-debug)]\n"); 535 #endif 536 printf("\t[-l log level]\n"); 537 printf("\t Available log levels:\n"); 538 printf("\t disabled, error, warning, notice, info, debug\n"); 539 } 540 541 static void 542 unregister_trids(void) 543 { 544 struct trid_entry *trid_entry, *tmp; 545 546 TAILQ_FOREACH_SAFE(trid_entry, &g_trid_list, tailq, tmp) { 547 TAILQ_REMOVE(&g_trid_list, trid_entry, tailq); 548 free(trid_entry); 549 } 550 } 551 552 static int 553 add_trid(const char *trid_str) 554 { 555 struct trid_entry *trid_entry; 556 struct spdk_nvme_transport_id *trid; 557 char *ns; 558 559 trid_entry = calloc(1, sizeof(*trid_entry)); 560 if (trid_entry == NULL) { 561 return -1; 562 } 563 564 trid = &trid_entry->trid; 565 trid->trtype = SPDK_NVME_TRANSPORT_PCIE; 566 snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN); 567 568 if (spdk_nvme_transport_id_parse(trid, trid_str) != 0) { 569 fprintf(stderr, "Invalid transport ID format '%s'\n", trid_str); 570 free(trid_entry); 571 return 1; 572 } 573 574 spdk_nvme_transport_id_populate_trstring(trid, 575 spdk_nvme_transport_id_trtype_str(trid->trtype)); 576 577 ns = strcasestr(trid_str, "ns:"); 578 if (ns) { 579 char nsid_str[6]; /* 5 digits maximum in an nsid */ 580 int len; 581 int nsid; 582 583 ns += 3; 584 585 len = strcspn(ns, " \t\n"); 586 if (len > 5) { 587 fprintf(stderr, "NVMe namespace IDs must be 5 digits or less\n"); 588 free(trid_entry); 589 return 1; 590 } 591 592 memcpy(nsid_str, ns, len); 593 nsid_str[len] = '\0'; 594 595 nsid = spdk_strtol(nsid_str, 10); 596 if (nsid <= 0 || nsid > 65535) { 597 fprintf(stderr, "NVMe namespace IDs must be less than 65536 and greater than 0\n"); 598 free(trid_entry); 599 return 1; 600 } 601 602 trid_entry->nsid = (uint16_t)nsid; 603 } 604 605 TAILQ_INSERT_TAIL(&g_trid_list, trid_entry, tailq); 606 return 0; 607 } 608 609 static int 610 parse_args(int argc, char **argv) 611 { 612 int op; 613 long int val; 614 int rc; 615 616 while ((op = getopt(argc, argv, "a:c:i:l:o:q:r:s:t:w:GM:T:")) != -1) { 617 switch (op) { 618 case 'a': 619 case 'i': 620 case 'o': 621 case 'q': 622 case 's': 623 case 't': 624 case 'M': 625 val = spdk_strtol(optarg, 10); 626 if (val < 0) { 627 fprintf(stderr, "Converting a string to integer failed\n"); 628 return val; 629 } 630 switch (op) { 631 case 'a': 632 g_abort_interval = val; 633 break; 634 case 'i': 635 g_shm_id = val; 636 break; 637 case 'o': 638 g_io_size_bytes = val; 639 break; 640 case 'q': 641 g_queue_depth = val; 642 break; 643 case 's': 644 g_dpdk_mem = val; 645 break; 646 case 't': 647 g_time_in_sec = val; 648 break; 649 case 'M': 650 g_rw_percentage = val; 651 g_mix_specified = true; 652 break; 653 } 654 break; 655 case 'c': 656 g_core_mask = optarg; 657 break; 658 case 'r': 659 if (add_trid(optarg)) { 660 usage(argv[0]); 661 return 1; 662 } 663 break; 664 case 'w': 665 g_workload_type = optarg; 666 break; 667 case 'G': 668 #ifndef DEBUG 669 fprintf(stderr, "%s must be configured with --enable-debug for -G flag\n", 670 argv[0]); 671 usage(argv[0]); 672 return 1; 673 #else 674 spdk_log_set_flag("nvme"); 675 spdk_log_set_print_level(SPDK_LOG_DEBUG); 676 break; 677 #endif 678 case 'T': 679 rc = spdk_log_set_flag(optarg); 680 if (rc < 0) { 681 fprintf(stderr, "unknown flag\n"); 682 usage(argv[0]); 683 exit(EXIT_FAILURE); 684 } 685 #ifdef DEBUG 686 spdk_log_set_print_level(SPDK_LOG_DEBUG); 687 #endif 688 break; 689 case 'l': 690 if (!strcmp(optarg, "disabled")) { 691 spdk_log_set_print_level(SPDK_LOG_DISABLED); 692 } else if (!strcmp(optarg, "error")) { 693 spdk_log_set_print_level(SPDK_LOG_ERROR); 694 } else if (!strcmp(optarg, "warning")) { 695 spdk_log_set_print_level(SPDK_LOG_WARN); 696 } else if (!strcmp(optarg, "notice")) { 697 spdk_log_set_print_level(SPDK_LOG_NOTICE); 698 } else if (!strcmp(optarg, "info")) { 699 spdk_log_set_print_level(SPDK_LOG_INFO); 700 } else if (!strcmp(optarg, "debug")) { 701 spdk_log_set_print_level(SPDK_LOG_DEBUG); 702 } else { 703 fprintf(stderr, "Unrecognized log level: %s\n", optarg); 704 return 1; 705 } 706 break; 707 default: 708 usage(argv[0]); 709 return 1; 710 } 711 } 712 713 if (!g_queue_depth) { 714 fprintf(stderr, "missing -q (queue size) operand\n"); 715 usage(argv[0]); 716 return 1; 717 } 718 if (!g_io_size_bytes) { 719 fprintf(stderr, "missing -o (block size) operand\n"); 720 usage(argv[0]); 721 return 1; 722 } 723 if (!g_workload_type) { 724 fprintf(stderr, "missing -t (test time in seconds) operand\n"); 725 usage(argv[0]); 726 return 1; 727 } 728 729 if (!g_time_in_sec) { 730 usage(argv[0]); 731 return 1; 732 } 733 734 if (strncmp(g_workload_type, "rand", 4) == 0) { 735 g_is_random = 1; 736 g_workload_type = &g_workload_type[4]; 737 } 738 739 if (strcmp(g_workload_type, "read") == 0 || strcmp(g_workload_type, "write") == 0) { 740 g_rw_percentage = strcmp(g_workload_type, "read") == 0 ? 100 : 0; 741 if (g_mix_specified) { 742 fprintf(stderr, "Ignoring -M option... Please use -M option" 743 " only when using rw or randrw.\n"); 744 } 745 } else if (strcmp(g_workload_type, "rw") == 0) { 746 if (g_rw_percentage < 0 || g_rw_percentage > 100) { 747 fprintf(stderr, 748 "-M must be specified to value from 0 to 100 " 749 "for rw or randrw.\n"); 750 return 1; 751 } 752 } else { 753 fprintf(stderr, 754 "io pattern type must be one of\n" 755 "(read, write, randread, randwrite, rw, randrw)\n"); 756 return 1; 757 } 758 759 if (TAILQ_EMPTY(&g_trid_list)) { 760 /* If no transport IDs specified, default to enumerating all local PCIe devices */ 761 add_trid("trtype:PCIe"); 762 } else { 763 struct trid_entry *trid_entry, *trid_entry_tmp; 764 765 g_no_pci = true; 766 /* check whether there is local PCIe type */ 767 TAILQ_FOREACH_SAFE(trid_entry, &g_trid_list, tailq, trid_entry_tmp) { 768 if (trid_entry->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) { 769 g_no_pci = false; 770 break; 771 } 772 } 773 } 774 775 return 0; 776 } 777 778 static int 779 register_workers(void) 780 { 781 uint32_t i; 782 struct worker_thread *worker; 783 784 SPDK_ENV_FOREACH_CORE(i) { 785 worker = calloc(1, sizeof(*worker)); 786 if (worker == NULL) { 787 fprintf(stderr, "Unable to allocate worker\n"); 788 return -1; 789 } 790 791 TAILQ_INIT(&worker->ns_ctx); 792 TAILQ_INIT(&worker->ctrlr_ctx); 793 worker->lcore = i; 794 TAILQ_INSERT_TAIL(&g_workers, worker, link); 795 g_num_workers++; 796 } 797 798 return 0; 799 } 800 801 static void 802 unregister_workers(void) 803 { 804 struct worker_thread *worker, *tmp_worker; 805 struct ns_worker_ctx *ns_ctx, *tmp_ns_ctx; 806 struct ctrlr_worker_ctx *ctrlr_ctx, *tmp_ctrlr_ctx; 807 808 /* Free namespace context and worker thread */ 809 TAILQ_FOREACH_SAFE(worker, &g_workers, link, tmp_worker) { 810 TAILQ_REMOVE(&g_workers, worker, link); 811 812 TAILQ_FOREACH_SAFE(ns_ctx, &worker->ns_ctx, link, tmp_ns_ctx) { 813 TAILQ_REMOVE(&worker->ns_ctx, ns_ctx, link); 814 printf("NS: %s I/O completed: %" PRIu64 ", failed: %" PRIu64 "\n", 815 ns_ctx->entry->name, ns_ctx->io_completed, ns_ctx->io_failed); 816 free(ns_ctx); 817 } 818 819 TAILQ_FOREACH_SAFE(ctrlr_ctx, &worker->ctrlr_ctx, link, tmp_ctrlr_ctx) { 820 TAILQ_REMOVE(&worker->ctrlr_ctx, ctrlr_ctx, link); 821 printf("CTRLR: %s abort submitted %" PRIu64 ", failed to submit %" PRIu64 "\n", 822 ctrlr_ctx->entry->name, ctrlr_ctx->abort_submitted, 823 ctrlr_ctx->abort_submit_failed); 824 printf("\t success %" PRIu64 ", unsuccess %" PRIu64 ", failed %" PRIu64 "\n", 825 ctrlr_ctx->successful_abort, ctrlr_ctx->unsuccessful_abort, 826 ctrlr_ctx->abort_failed); 827 free(ctrlr_ctx); 828 } 829 830 free(worker); 831 } 832 } 833 834 static bool 835 probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 836 struct spdk_nvme_ctrlr_opts *opts) 837 { 838 uint16_t min_aq_size; 839 840 /* We need to make sure the admin queue is big enough to handle all of the aborts that 841 * will be sent by this test app. We add a few extra entries to account for any admin 842 * commands other than the aborts. */ 843 min_aq_size = spdk_divide_round_up(g_queue_depth, g_abort_interval) + 8; 844 opts->admin_queue_size = spdk_max(opts->admin_queue_size, min_aq_size); 845 846 /* Avoid possible nvme_qpair_abort_queued_reqs_with_cbarg ERROR when IO queue size is 128. */ 847 opts->disable_error_logging = true; 848 849 return true; 850 } 851 852 static void 853 attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 854 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts) 855 { 856 struct trid_entry *trid_entry = cb_ctx; 857 struct spdk_pci_addr pci_addr; 858 struct spdk_pci_device *pci_dev; 859 struct spdk_pci_id pci_id; 860 861 if (trid->trtype != SPDK_NVME_TRANSPORT_PCIE) { 862 printf("Attached to NVMe over Fabrics controller at %s:%s: %s\n", 863 trid->traddr, trid->trsvcid, 864 trid->subnqn); 865 } else { 866 if (spdk_pci_addr_parse(&pci_addr, trid->traddr)) { 867 return; 868 } 869 870 pci_dev = spdk_nvme_ctrlr_get_pci_device(ctrlr); 871 if (!pci_dev) { 872 return; 873 } 874 875 pci_id = spdk_pci_device_get_id(pci_dev); 876 877 printf("Attached to NVMe Controller at %s [%04x:%04x]\n", 878 trid->traddr, 879 pci_id.vendor_id, pci_id.device_id); 880 } 881 882 register_ctrlr(ctrlr, trid_entry); 883 } 884 885 static int 886 register_controllers(void) 887 { 888 struct trid_entry *trid_entry; 889 890 printf("Initializing NVMe Controllers\n"); 891 892 TAILQ_FOREACH(trid_entry, &g_trid_list, tailq) { 893 if (spdk_nvme_probe(&trid_entry->trid, trid_entry, probe_cb, attach_cb, NULL) != 0) { 894 fprintf(stderr, "spdk_nvme_probe() failed for transport address '%s'\n", 895 trid_entry->trid.traddr); 896 return -1; 897 } 898 } 899 900 return 0; 901 } 902 903 static void 904 unregister_controllers(void) 905 { 906 struct ctrlr_entry *entry, *tmp; 907 struct spdk_nvme_detach_ctx *detach_ctx = NULL; 908 909 TAILQ_FOREACH_SAFE(entry, &g_controllers, link, tmp) { 910 TAILQ_REMOVE(&g_controllers, entry, link); 911 spdk_nvme_detach_async(entry->ctrlr, &detach_ctx); 912 free(entry); 913 } 914 915 if (detach_ctx) { 916 spdk_nvme_detach_poll(detach_ctx); 917 } 918 } 919 920 static int 921 associate_main_worker_with_ctrlr(void) 922 { 923 struct ctrlr_entry *entry; 924 struct worker_thread *worker; 925 struct ctrlr_worker_ctx *ctrlr_ctx; 926 927 TAILQ_FOREACH(worker, &g_workers, link) { 928 if (worker->lcore == g_main_core) { 929 break; 930 } 931 } 932 933 if (!worker) { 934 return -1; 935 } 936 937 TAILQ_FOREACH(entry, &g_controllers, link) { 938 ctrlr_ctx = calloc(1, sizeof(struct ctrlr_worker_ctx)); 939 if (!ctrlr_ctx) { 940 return -1; 941 } 942 943 pthread_mutex_init(&ctrlr_ctx->mutex, NULL); 944 ctrlr_ctx->entry = entry; 945 ctrlr_ctx->ctrlr = entry->ctrlr; 946 947 TAILQ_INSERT_TAIL(&worker->ctrlr_ctx, ctrlr_ctx, link); 948 } 949 950 return 0; 951 } 952 953 static struct ctrlr_worker_ctx * 954 get_ctrlr_worker_ctx(struct spdk_nvme_ctrlr *ctrlr) 955 { 956 struct worker_thread *worker; 957 struct ctrlr_worker_ctx *ctrlr_ctx; 958 959 TAILQ_FOREACH(worker, &g_workers, link) { 960 if (worker->lcore == g_main_core) { 961 break; 962 } 963 } 964 965 if (!worker) { 966 return NULL; 967 } 968 969 TAILQ_FOREACH(ctrlr_ctx, &worker->ctrlr_ctx, link) { 970 if (ctrlr_ctx->ctrlr == ctrlr) { 971 return ctrlr_ctx; 972 } 973 } 974 975 return NULL; 976 } 977 978 static int 979 associate_workers_with_ns(void) 980 { 981 struct ns_entry *entry = TAILQ_FIRST(&g_namespaces); 982 struct worker_thread *worker = TAILQ_FIRST(&g_workers); 983 struct ns_worker_ctx *ns_ctx; 984 int i, count; 985 986 count = g_num_namespaces > g_num_workers ? g_num_namespaces : g_num_workers; 987 988 for (i = 0; i < count; i++) { 989 if (entry == NULL) { 990 break; 991 } 992 993 ns_ctx = calloc(1, sizeof(struct ns_worker_ctx)); 994 if (!ns_ctx) { 995 return -1; 996 } 997 998 printf("Associating %s with lcore %d\n", entry->name, worker->lcore); 999 ns_ctx->entry = entry; 1000 ns_ctx->ctrlr_ctx = get_ctrlr_worker_ctx(entry->ctrlr); 1001 if (!ns_ctx->ctrlr_ctx) { 1002 free(ns_ctx); 1003 return -1; 1004 } 1005 1006 TAILQ_INSERT_TAIL(&worker->ns_ctx, ns_ctx, link); 1007 1008 worker = TAILQ_NEXT(worker, link); 1009 if (worker == NULL) { 1010 worker = TAILQ_FIRST(&g_workers); 1011 } 1012 1013 entry = TAILQ_NEXT(entry, link); 1014 if (entry == NULL) { 1015 entry = TAILQ_FIRST(&g_namespaces); 1016 } 1017 } 1018 1019 return 0; 1020 } 1021 1022 int 1023 main(int argc, char **argv) 1024 { 1025 int rc; 1026 struct worker_thread *worker, *main_worker; 1027 struct spdk_env_opts opts; 1028 1029 rc = parse_args(argc, argv); 1030 if (rc != 0) { 1031 return rc; 1032 } 1033 1034 spdk_env_opts_init(&opts); 1035 opts.name = "abort"; 1036 opts.shm_id = g_shm_id; 1037 if (g_core_mask) { 1038 opts.core_mask = g_core_mask; 1039 } 1040 1041 if (g_dpdk_mem) { 1042 opts.mem_size = g_dpdk_mem; 1043 } 1044 if (g_no_pci) { 1045 opts.no_pci = g_no_pci; 1046 } 1047 if (spdk_env_init(&opts) < 0) { 1048 fprintf(stderr, "Unable to initialize SPDK env\n"); 1049 unregister_trids(); 1050 return -1; 1051 } 1052 1053 g_tsc_rate = spdk_get_ticks_hz(); 1054 1055 if (register_workers() != 0) { 1056 rc = -1; 1057 goto cleanup; 1058 } 1059 1060 if (register_controllers() != 0) { 1061 rc = -1; 1062 goto cleanup; 1063 } 1064 1065 if (g_warn) { 1066 printf("WARNING: Some requested NVMe devices were skipped\n"); 1067 } 1068 1069 if (g_num_namespaces == 0) { 1070 fprintf(stderr, "No valid NVMe controllers found\n"); 1071 rc = -1; 1072 goto cleanup; 1073 } 1074 1075 if (associate_main_worker_with_ctrlr() != 0) { 1076 rc = -1; 1077 goto cleanup; 1078 } 1079 1080 if (associate_workers_with_ns() != 0) { 1081 rc = -1; 1082 goto cleanup; 1083 } 1084 1085 printf("Initialization complete. Launching workers.\n"); 1086 1087 /* Launch all of the secondary workers */ 1088 g_main_core = spdk_env_get_current_core(); 1089 main_worker = NULL; 1090 TAILQ_FOREACH(worker, &g_workers, link) { 1091 if (worker->lcore != g_main_core) { 1092 spdk_env_thread_launch_pinned(worker->lcore, work_fn, worker); 1093 } else { 1094 assert(main_worker == NULL); 1095 main_worker = worker; 1096 } 1097 } 1098 1099 assert(main_worker != NULL); 1100 rc = work_fn(main_worker); 1101 1102 spdk_env_thread_wait_all(); 1103 1104 cleanup: 1105 unregister_trids(); 1106 unregister_workers(); 1107 unregister_namespaces(); 1108 unregister_controllers(); 1109 1110 spdk_env_fini(); 1111 1112 if (rc != 0) { 1113 fprintf(stderr, "%s: errors occurred\n", argv[0]); 1114 } 1115 1116 return rc; 1117 } 1118