1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2020 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 8 #include "spdk/env.h" 9 #include "spdk/log.h" 10 #include "spdk/nvme.h" 11 #include "spdk/queue.h" 12 #include "spdk/string.h" 13 #include "spdk/util.h" 14 #include "spdk/likely.h" 15 16 struct ctrlr_entry { 17 struct spdk_nvme_ctrlr *ctrlr; 18 enum spdk_nvme_transport_type trtype; 19 20 TAILQ_ENTRY(ctrlr_entry) link; 21 char name[1024]; 22 }; 23 24 struct ns_entry { 25 struct spdk_nvme_ctrlr *ctrlr; 26 struct spdk_nvme_ns *ns; 27 28 TAILQ_ENTRY(ns_entry) link; 29 uint32_t io_size_blocks; 30 uint32_t num_io_requests; 31 uint64_t size_in_ios; 32 uint32_t block_size; 33 char name[1024]; 34 }; 35 36 struct ctrlr_worker_ctx { 37 pthread_mutex_t mutex; 38 struct ctrlr_entry *entry; 39 uint64_t abort_submitted; 40 uint64_t abort_submit_failed; 41 uint64_t successful_abort; 42 uint64_t unsuccessful_abort; 43 uint64_t abort_failed; 44 uint64_t current_queue_depth; 45 struct spdk_nvme_ctrlr *ctrlr; 46 TAILQ_ENTRY(ctrlr_worker_ctx) link; 47 }; 48 49 struct ns_worker_ctx { 50 struct ns_entry *entry; 51 uint64_t io_submitted; 52 uint64_t io_completed; 53 uint64_t io_aborted; 54 uint64_t io_failed; 55 uint64_t current_queue_depth; 56 uint64_t offset_in_ios; 57 bool is_draining; 58 struct spdk_nvme_qpair *qpair; 59 struct ctrlr_worker_ctx *ctrlr_ctx; 60 TAILQ_ENTRY(ns_worker_ctx) link; 61 }; 62 63 struct perf_task { 64 struct ns_worker_ctx *ns_ctx; 65 void *buf; 66 }; 67 68 struct worker_thread { 69 TAILQ_HEAD(, ns_worker_ctx) ns_ctx; 70 TAILQ_HEAD(, ctrlr_worker_ctx) ctrlr_ctx; 71 TAILQ_ENTRY(worker_thread) link; 72 unsigned lcore; 73 int status; 74 }; 75 76 static const char *g_workload_type = "read"; 77 static TAILQ_HEAD(, ctrlr_entry) g_controllers = TAILQ_HEAD_INITIALIZER(g_controllers); 78 static TAILQ_HEAD(, ns_entry) g_namespaces = TAILQ_HEAD_INITIALIZER(g_namespaces); 79 static int g_num_namespaces; 80 static TAILQ_HEAD(, worker_thread) g_workers = TAILQ_HEAD_INITIALIZER(g_workers); 81 static int g_num_workers = 0; 82 static uint32_t g_main_core; 83 84 static int g_abort_interval = 1; 85 86 static uint64_t g_tsc_rate; 87 88 static uint32_t g_io_size_bytes = 131072; 89 static uint32_t g_max_io_size_blocks; 90 static int g_rw_percentage = -1; 91 static int g_is_random; 92 static int g_queue_depth = 128; 93 static int g_time_in_sec = 3; 94 static int g_dpdk_mem; 95 static int g_shm_id = -1; 96 static bool g_no_pci; 97 static bool g_warn; 98 static bool g_mix_specified; 99 100 static const char *g_core_mask; 101 102 struct trid_entry { 103 struct spdk_nvme_transport_id trid; 104 uint16_t nsid; 105 TAILQ_ENTRY(trid_entry) tailq; 106 }; 107 108 static TAILQ_HEAD(, trid_entry) g_trid_list = TAILQ_HEAD_INITIALIZER(g_trid_list); 109 110 static void io_complete(void *ctx, const struct spdk_nvme_cpl *cpl); 111 112 static int 113 build_nvme_name(char *name, size_t length, struct spdk_nvme_ctrlr *ctrlr) 114 { 115 const struct spdk_nvme_transport_id *trid; 116 int res = 0; 117 118 trid = spdk_nvme_ctrlr_get_transport_id(ctrlr); 119 120 switch (trid->trtype) { 121 case SPDK_NVME_TRANSPORT_PCIE: 122 res = snprintf(name, length, "PCIE (%s)", trid->traddr); 123 break; 124 case SPDK_NVME_TRANSPORT_RDMA: 125 res = snprintf(name, length, "RDMA (addr:%s subnqn:%s)", trid->traddr, trid->subnqn); 126 break; 127 case SPDK_NVME_TRANSPORT_TCP: 128 res = snprintf(name, length, "TCP (addr:%s subnqn:%s)", trid->traddr, trid->subnqn); 129 break; 130 case SPDK_NVME_TRANSPORT_CUSTOM: 131 res = snprintf(name, length, "CUSTOM (%s)", trid->traddr); 132 break; 133 134 default: 135 fprintf(stderr, "Unknown transport type %d\n", trid->trtype); 136 break; 137 } 138 return res; 139 } 140 141 static void 142 build_nvme_ns_name(char *name, size_t length, struct spdk_nvme_ctrlr *ctrlr, uint32_t nsid) 143 { 144 int res = 0; 145 146 res = build_nvme_name(name, length, ctrlr); 147 if (res > 0) { 148 snprintf(name + res, length - res, " NSID %u", nsid); 149 } 150 151 } 152 153 static void 154 register_ns(struct spdk_nvme_ctrlr *ctrlr, struct spdk_nvme_ns *ns) 155 { 156 struct ns_entry *entry; 157 const struct spdk_nvme_ctrlr_data *cdata; 158 uint32_t max_xfer_size, entries, sector_size; 159 uint64_t ns_size; 160 struct spdk_nvme_io_qpair_opts opts; 161 162 cdata = spdk_nvme_ctrlr_get_data(ctrlr); 163 164 if (!spdk_nvme_ns_is_active(ns)) { 165 printf("Controller %-20.20s (%-20.20s): Skipping inactive NS %u\n", 166 cdata->mn, cdata->sn, 167 spdk_nvme_ns_get_id(ns)); 168 g_warn = true; 169 return; 170 } 171 172 ns_size = spdk_nvme_ns_get_size(ns); 173 sector_size = spdk_nvme_ns_get_sector_size(ns); 174 175 if (ns_size < g_io_size_bytes || sector_size > g_io_size_bytes) { 176 printf("WARNING: controller %-20.20s (%-20.20s) ns %u has invalid " 177 "ns size %" PRIu64 " / block size %u for I/O size %u\n", 178 cdata->mn, cdata->sn, spdk_nvme_ns_get_id(ns), 179 ns_size, spdk_nvme_ns_get_sector_size(ns), g_io_size_bytes); 180 g_warn = true; 181 return; 182 } 183 184 max_xfer_size = spdk_nvme_ns_get_max_io_xfer_size(ns); 185 spdk_nvme_ctrlr_get_default_io_qpair_opts(ctrlr, &opts, sizeof(opts)); 186 /* NVMe driver may add additional entries based on 187 * stripe size and maximum transfer size, we assume 188 * 1 more entry be used for stripe. 189 */ 190 entries = (g_io_size_bytes - 1) / max_xfer_size + 2; 191 if ((g_queue_depth * entries) > opts.io_queue_size) { 192 printf("controller IO queue size %u less than required\n", 193 opts.io_queue_size); 194 printf("Consider using lower queue depth or small IO size because " 195 "IO requests may be queued at the NVMe driver.\n"); 196 } 197 /* For requests which have children requests, parent request itself 198 * will also occupy 1 entry. 199 */ 200 entries += 1; 201 202 entry = calloc(1, sizeof(struct ns_entry)); 203 if (entry == NULL) { 204 perror("ns_entry malloc"); 205 exit(1); 206 } 207 208 entry->ctrlr = ctrlr; 209 entry->ns = ns; 210 entry->num_io_requests = g_queue_depth * entries; 211 212 entry->size_in_ios = ns_size / g_io_size_bytes; 213 entry->io_size_blocks = g_io_size_bytes / sector_size; 214 215 entry->block_size = spdk_nvme_ns_get_sector_size(ns); 216 217 if (g_max_io_size_blocks < entry->io_size_blocks) { 218 g_max_io_size_blocks = entry->io_size_blocks; 219 } 220 221 build_nvme_ns_name(entry->name, sizeof(entry->name), ctrlr, spdk_nvme_ns_get_id(ns)); 222 223 g_num_namespaces++; 224 TAILQ_INSERT_TAIL(&g_namespaces, entry, link); 225 } 226 227 static void 228 unregister_namespaces(void) 229 { 230 struct ns_entry *entry, *tmp; 231 232 TAILQ_FOREACH_SAFE(entry, &g_namespaces, link, tmp) { 233 TAILQ_REMOVE(&g_namespaces, entry, link); 234 free(entry); 235 } 236 } 237 238 static void 239 register_ctrlr(struct spdk_nvme_ctrlr *ctrlr, struct trid_entry *trid_entry) 240 { 241 struct spdk_nvme_ns *ns; 242 struct ctrlr_entry *entry = malloc(sizeof(struct ctrlr_entry)); 243 uint32_t nsid; 244 245 if (entry == NULL) { 246 perror("ctrlr_entry malloc"); 247 exit(1); 248 } 249 250 build_nvme_name(entry->name, sizeof(entry->name), ctrlr); 251 252 entry->ctrlr = ctrlr; 253 entry->trtype = trid_entry->trid.trtype; 254 TAILQ_INSERT_TAIL(&g_controllers, entry, link); 255 256 if (trid_entry->nsid == 0) { 257 for (nsid = spdk_nvme_ctrlr_get_first_active_ns(ctrlr); 258 nsid != 0; nsid = spdk_nvme_ctrlr_get_next_active_ns(ctrlr, nsid)) { 259 ns = spdk_nvme_ctrlr_get_ns(ctrlr, nsid); 260 if (ns == NULL) { 261 continue; 262 } 263 register_ns(ctrlr, ns); 264 } 265 } else { 266 ns = spdk_nvme_ctrlr_get_ns(ctrlr, trid_entry->nsid); 267 if (!ns) { 268 perror("Namespace does not exist."); 269 exit(1); 270 } 271 272 register_ns(ctrlr, ns); 273 } 274 } 275 276 static void 277 abort_complete(void *ctx, const struct spdk_nvme_cpl *cpl) 278 { 279 struct ctrlr_worker_ctx *ctrlr_ctx = ctx; 280 281 ctrlr_ctx->current_queue_depth--; 282 if (spdk_unlikely(spdk_nvme_cpl_is_error(cpl))) { 283 ctrlr_ctx->abort_failed++; 284 } else if ((cpl->cdw0 & 0x1) == 0) { 285 ctrlr_ctx->successful_abort++; 286 } else { 287 ctrlr_ctx->unsuccessful_abort++; 288 } 289 } 290 291 static void 292 abort_task(struct perf_task *task) 293 { 294 struct ns_worker_ctx *ns_ctx = task->ns_ctx; 295 struct ctrlr_worker_ctx *ctrlr_ctx = ns_ctx->ctrlr_ctx; 296 int rc; 297 298 /* Hold mutex to guard ctrlr_ctx->current_queue_depth. */ 299 pthread_mutex_lock(&ctrlr_ctx->mutex); 300 301 rc = spdk_nvme_ctrlr_cmd_abort_ext(ctrlr_ctx->ctrlr, ns_ctx->qpair, task, abort_complete, 302 ctrlr_ctx); 303 304 if (spdk_unlikely(rc != 0)) { 305 ctrlr_ctx->abort_submit_failed++; 306 } else { 307 ctrlr_ctx->current_queue_depth++; 308 ctrlr_ctx->abort_submitted++; 309 } 310 311 pthread_mutex_unlock(&ctrlr_ctx->mutex); 312 } 313 314 static __thread unsigned int seed = 0; 315 316 static inline void 317 submit_single_io(struct perf_task *task) 318 { 319 uint64_t offset_in_ios, lba; 320 int rc; 321 struct ns_worker_ctx *ns_ctx = task->ns_ctx; 322 struct ns_entry *entry = ns_ctx->entry; 323 324 if (g_is_random) { 325 offset_in_ios = rand_r(&seed) % entry->size_in_ios; 326 } else { 327 offset_in_ios = ns_ctx->offset_in_ios++; 328 if (ns_ctx->offset_in_ios == entry->size_in_ios) { 329 ns_ctx->offset_in_ios = 0; 330 } 331 } 332 333 lba = offset_in_ios * entry->io_size_blocks; 334 335 if ((g_rw_percentage == 100) || 336 (g_rw_percentage != 0 && (rand_r(&seed) % 100) < g_rw_percentage)) { 337 rc = spdk_nvme_ns_cmd_read(entry->ns, ns_ctx->qpair, task->buf, 338 lba, entry->io_size_blocks, io_complete, task, 0); 339 } else { 340 rc = spdk_nvme_ns_cmd_write(entry->ns, ns_ctx->qpair, task->buf, 341 lba, entry->io_size_blocks, io_complete, task, 0); 342 } 343 344 if (spdk_unlikely(rc != 0)) { 345 fprintf(stderr, "I/O submission failed\n"); 346 } else { 347 ns_ctx->current_queue_depth++; 348 ns_ctx->io_submitted++; 349 350 if ((ns_ctx->io_submitted % g_abort_interval) == 0) { 351 abort_task(task); 352 } 353 } 354 355 } 356 357 static void 358 io_complete(void *ctx, const struct spdk_nvme_cpl *cpl) 359 { 360 struct perf_task *task = ctx; 361 struct ns_worker_ctx *ns_ctx = task->ns_ctx; 362 363 ns_ctx->current_queue_depth--; 364 if (spdk_unlikely(spdk_nvme_cpl_is_error(cpl))) { 365 ns_ctx->io_failed++; 366 } else { 367 ns_ctx->io_completed++; 368 } 369 370 /* is_draining indicates when time has expired for the test run and we are 371 * just waiting for the previously submitted I/O to complete. In this case, 372 * do not submit a new I/O to replace the one just completed. 373 */ 374 if (spdk_unlikely(ns_ctx->is_draining)) { 375 spdk_dma_free(task->buf); 376 free(task); 377 } else { 378 submit_single_io(task); 379 } 380 } 381 382 static struct perf_task * 383 allocate_task(struct ns_worker_ctx *ns_ctx) 384 { 385 struct perf_task *task; 386 387 task = calloc(1, sizeof(*task)); 388 if (task == NULL) { 389 fprintf(stderr, "Failed to allocate task\n"); 390 exit(1); 391 } 392 393 task->buf = spdk_dma_zmalloc(g_io_size_bytes, 0x200, NULL); 394 if (task->buf == NULL) { 395 free(task); 396 fprintf(stderr, "Failed to allocate task->buf\n"); 397 exit(1); 398 } 399 400 task->ns_ctx = ns_ctx; 401 402 return task; 403 } 404 405 static void 406 submit_io(struct ns_worker_ctx *ns_ctx, int queue_depth) 407 { 408 struct perf_task *task; 409 410 while (queue_depth-- > 0) { 411 task = allocate_task(ns_ctx); 412 submit_single_io(task); 413 } 414 } 415 416 static int 417 work_fn(void *arg) 418 { 419 struct worker_thread *worker = (struct worker_thread *)arg; 420 struct ns_worker_ctx *ns_ctx; 421 struct ctrlr_worker_ctx *ctrlr_ctx; 422 struct ns_entry *ns_entry; 423 struct spdk_nvme_io_qpair_opts opts; 424 uint64_t tsc_end; 425 uint32_t unfinished_ctx; 426 int rc = 0; 427 428 /* Allocate queue pair for each namespace. */ 429 TAILQ_FOREACH(ns_ctx, &worker->ns_ctx, link) { 430 ns_entry = ns_ctx->entry; 431 432 spdk_nvme_ctrlr_get_default_io_qpair_opts(ns_entry->ctrlr, &opts, sizeof(opts)); 433 if (opts.io_queue_requests < ns_entry->num_io_requests) { 434 opts.io_queue_requests = ns_entry->num_io_requests; 435 } 436 437 ns_ctx->qpair = spdk_nvme_ctrlr_alloc_io_qpair(ns_entry->ctrlr, &opts, sizeof(opts)); 438 if (ns_ctx->qpair == NULL) { 439 fprintf(stderr, "spdk_nvme_ctrlr_alloc_io_qpair failed\n"); 440 worker->status = -ENOMEM; 441 goto out; 442 } 443 } 444 445 tsc_end = spdk_get_ticks() + g_time_in_sec * g_tsc_rate; 446 447 /* Submit initial I/O for each namespace. */ 448 TAILQ_FOREACH(ns_ctx, &worker->ns_ctx, link) { 449 submit_io(ns_ctx, g_queue_depth); 450 } 451 452 while (1) { 453 TAILQ_FOREACH(ns_ctx, &worker->ns_ctx, link) { 454 rc = spdk_nvme_qpair_process_completions(ns_ctx->qpair, 0); 455 if (rc < 0) { 456 fprintf(stderr, "spdk_nvme_qpair_process_completions returned " 457 "%d\n", rc); 458 worker->status = rc; 459 goto out; 460 } 461 } 462 463 if (worker->lcore == g_main_core) { 464 TAILQ_FOREACH(ctrlr_ctx, &worker->ctrlr_ctx, link) { 465 /* Hold mutex to guard ctrlr_ctx->current_queue_depth. */ 466 pthread_mutex_lock(&ctrlr_ctx->mutex); 467 rc = spdk_nvme_ctrlr_process_admin_completions(ctrlr_ctx->ctrlr); 468 pthread_mutex_unlock(&ctrlr_ctx->mutex); 469 if (rc < 0) { 470 fprintf(stderr, "spdk_nvme_ctrlr_process_admin_completions " 471 "returned %d\n", rc); 472 worker->status = rc; 473 goto out; 474 } 475 } 476 } 477 478 if (spdk_get_ticks() > tsc_end) { 479 break; 480 } 481 } 482 483 do { 484 unfinished_ctx = 0; 485 486 TAILQ_FOREACH(ns_ctx, &worker->ns_ctx, link) { 487 if (!ns_ctx->is_draining) { 488 ns_ctx->is_draining = true; 489 } 490 if (ns_ctx->current_queue_depth > 0) { 491 rc = spdk_nvme_qpair_process_completions(ns_ctx->qpair, 0); 492 if (rc < 0) { 493 fprintf(stderr, "spdk_nvme_qpair_process_completions " 494 "returned %d\n", rc); 495 worker->status = rc; 496 goto out; 497 } 498 unfinished_ctx++; 499 } 500 } 501 } while (unfinished_ctx > 0); 502 503 if (worker->lcore == g_main_core) { 504 do { 505 unfinished_ctx = 0; 506 507 TAILQ_FOREACH(ctrlr_ctx, &worker->ctrlr_ctx, link) { 508 pthread_mutex_lock(&ctrlr_ctx->mutex); 509 if (ctrlr_ctx->current_queue_depth > 0) { 510 rc = spdk_nvme_ctrlr_process_admin_completions(ctrlr_ctx->ctrlr); 511 unfinished_ctx++; 512 } 513 pthread_mutex_unlock(&ctrlr_ctx->mutex); 514 if (rc < 0) { 515 fprintf(stderr, "spdk_nvme_ctrlr_process_admin_completions " 516 "returned %d\n", rc); 517 worker->status = rc; 518 goto out; 519 } 520 } 521 } while (unfinished_ctx > 0); 522 } 523 out: 524 TAILQ_FOREACH(ns_ctx, &worker->ns_ctx, link) { 525 /* Make sure we don't submit any IOs at this point */ 526 ns_ctx->is_draining = true; 527 spdk_nvme_ctrlr_free_io_qpair(ns_ctx->qpair); 528 } 529 530 return worker->status != 0; 531 } 532 533 static void 534 usage(char *program_name) 535 { 536 printf("%s options", program_name); 537 538 printf("\n"); 539 printf("\t[-q io depth]\n"); 540 printf("\t[-o io size in bytes]\n"); 541 printf("\t[-w io pattern type, must be one of\n"); 542 printf("\t\t(read, write, randread, randwrite, rw, randrw)]\n"); 543 printf("\t[-M rwmixread (100 for reads, 0 for writes)]\n"); 544 printf("\t[-t time in seconds]\n"); 545 printf("\t[-c core mask for I/O submission/completion.]\n"); 546 printf("\t\t(default: 1)\n"); 547 printf("\t[-r Transport ID for local PCIe NVMe or NVMeoF]\n"); 548 printf("\t Format: 'key:value [key:value] ...'\n"); 549 printf("\t Keys:\n"); 550 printf("\t trtype Transport type (e.g. PCIe, RDMA)\n"); 551 printf("\t adrfam Address family (e.g. IPv4, IPv6)\n"); 552 printf("\t traddr Transport address (e.g. 0000:04:00.0 for PCIe or 192.168.100.8 for RDMA)\n"); 553 printf("\t trsvcid Transport service identifier (e.g. 4420)\n"); 554 printf("\t subnqn Subsystem NQN (default: %s)\n", SPDK_NVMF_DISCOVERY_NQN); 555 printf("\t Example: -r 'trtype:PCIe traddr:0000:04:00.0' for PCIe or\n"); 556 printf("\t -r 'trtype:RDMA adrfam:IPv4 traddr:192.168.100.8 trsvcid:4420' for NVMeoF\n"); 557 printf("\t[-s DPDK huge memory size in MB.]\n"); 558 printf("\t[-i shared memory group ID]\n"); 559 printf("\t[-a abort interval.]\n"); 560 printf("\t"); 561 spdk_log_usage(stdout, "-T"); 562 #ifdef DEBUG 563 printf("\t[-G enable debug logging]\n"); 564 #else 565 printf("\t[-G enable debug logging (flag disabled, must reconfigure with --enable-debug)]\n"); 566 #endif 567 printf("\t[-l log level]\n"); 568 printf("\t Available log levels:\n"); 569 printf("\t disabled, error, warning, notice, info, debug\n"); 570 } 571 572 static void 573 unregister_trids(void) 574 { 575 struct trid_entry *trid_entry, *tmp; 576 577 TAILQ_FOREACH_SAFE(trid_entry, &g_trid_list, tailq, tmp) { 578 TAILQ_REMOVE(&g_trid_list, trid_entry, tailq); 579 free(trid_entry); 580 } 581 } 582 583 static int 584 add_trid(const char *trid_str) 585 { 586 struct trid_entry *trid_entry; 587 struct spdk_nvme_transport_id *trid; 588 char *ns; 589 590 trid_entry = calloc(1, sizeof(*trid_entry)); 591 if (trid_entry == NULL) { 592 return -1; 593 } 594 595 trid = &trid_entry->trid; 596 trid->trtype = SPDK_NVME_TRANSPORT_PCIE; 597 snprintf(trid->subnqn, sizeof(trid->subnqn), "%s", SPDK_NVMF_DISCOVERY_NQN); 598 599 if (spdk_nvme_transport_id_parse(trid, trid_str) != 0) { 600 fprintf(stderr, "Invalid transport ID format '%s'\n", trid_str); 601 free(trid_entry); 602 return 1; 603 } 604 605 spdk_nvme_transport_id_populate_trstring(trid, 606 spdk_nvme_transport_id_trtype_str(trid->trtype)); 607 608 ns = strcasestr(trid_str, "ns:"); 609 if (ns) { 610 char nsid_str[6]; /* 5 digits maximum in an nsid */ 611 int len; 612 int nsid; 613 614 ns += 3; 615 616 len = strcspn(ns, " \t\n"); 617 if (len > 5) { 618 fprintf(stderr, "NVMe namespace IDs must be 5 digits or less\n"); 619 free(trid_entry); 620 return 1; 621 } 622 623 memcpy(nsid_str, ns, len); 624 nsid_str[len] = '\0'; 625 626 nsid = spdk_strtol(nsid_str, 10); 627 if (nsid <= 0 || nsid > 65535) { 628 fprintf(stderr, "NVMe namespace IDs must be less than 65536 and greater than 0\n"); 629 free(trid_entry); 630 return 1; 631 } 632 633 trid_entry->nsid = (uint16_t)nsid; 634 } 635 636 TAILQ_INSERT_TAIL(&g_trid_list, trid_entry, tailq); 637 return 0; 638 } 639 640 static int 641 parse_args(int argc, char **argv) 642 { 643 int op; 644 long int val; 645 int rc; 646 647 while ((op = getopt(argc, argv, "a:c:i:l:o:q:r:s:t:w:GM:T:")) != -1) { 648 switch (op) { 649 case 'a': 650 case 'i': 651 case 'o': 652 case 'q': 653 case 's': 654 case 't': 655 case 'M': 656 val = spdk_strtol(optarg, 10); 657 if (val < 0) { 658 fprintf(stderr, "Converting a string to integer failed\n"); 659 return val; 660 } 661 switch (op) { 662 case 'a': 663 g_abort_interval = val; 664 break; 665 case 'i': 666 g_shm_id = val; 667 break; 668 case 'o': 669 g_io_size_bytes = val; 670 break; 671 case 'q': 672 g_queue_depth = val; 673 break; 674 case 's': 675 g_dpdk_mem = val; 676 break; 677 case 't': 678 g_time_in_sec = val; 679 break; 680 case 'M': 681 g_rw_percentage = val; 682 g_mix_specified = true; 683 break; 684 } 685 break; 686 case 'c': 687 g_core_mask = optarg; 688 break; 689 case 'r': 690 if (add_trid(optarg)) { 691 usage(argv[0]); 692 return 1; 693 } 694 break; 695 case 'w': 696 g_workload_type = optarg; 697 break; 698 case 'G': 699 #ifndef DEBUG 700 fprintf(stderr, "%s must be configured with --enable-debug for -G flag\n", 701 argv[0]); 702 usage(argv[0]); 703 return 1; 704 #else 705 spdk_log_set_flag("nvme"); 706 spdk_log_set_print_level(SPDK_LOG_DEBUG); 707 break; 708 #endif 709 case 'T': 710 rc = spdk_log_set_flag(optarg); 711 if (rc < 0) { 712 fprintf(stderr, "unknown flag\n"); 713 usage(argv[0]); 714 exit(EXIT_FAILURE); 715 } 716 #ifdef DEBUG 717 spdk_log_set_print_level(SPDK_LOG_DEBUG); 718 #endif 719 break; 720 case 'l': 721 if (!strcmp(optarg, "disabled")) { 722 spdk_log_set_print_level(SPDK_LOG_DISABLED); 723 } else if (!strcmp(optarg, "error")) { 724 spdk_log_set_print_level(SPDK_LOG_ERROR); 725 } else if (!strcmp(optarg, "warning")) { 726 spdk_log_set_print_level(SPDK_LOG_WARN); 727 } else if (!strcmp(optarg, "notice")) { 728 spdk_log_set_print_level(SPDK_LOG_NOTICE); 729 } else if (!strcmp(optarg, "info")) { 730 spdk_log_set_print_level(SPDK_LOG_INFO); 731 } else if (!strcmp(optarg, "debug")) { 732 spdk_log_set_print_level(SPDK_LOG_DEBUG); 733 } else { 734 fprintf(stderr, "Unrecognized log level: %s\n", optarg); 735 return 1; 736 } 737 break; 738 default: 739 usage(argv[0]); 740 return 1; 741 } 742 } 743 744 if (!g_queue_depth) { 745 fprintf(stderr, "missing -q (queue size) operand\n"); 746 usage(argv[0]); 747 return 1; 748 } 749 if (!g_io_size_bytes) { 750 fprintf(stderr, "missing -o (block size) operand\n"); 751 usage(argv[0]); 752 return 1; 753 } 754 if (!g_workload_type) { 755 fprintf(stderr, "missing -t (test time in seconds) operand\n"); 756 usage(argv[0]); 757 return 1; 758 } 759 760 if (!g_time_in_sec) { 761 usage(argv[0]); 762 return 1; 763 } 764 765 if (strncmp(g_workload_type, "rand", 4) == 0) { 766 g_is_random = 1; 767 g_workload_type = &g_workload_type[4]; 768 } 769 770 if (strcmp(g_workload_type, "read") == 0 || strcmp(g_workload_type, "write") == 0) { 771 g_rw_percentage = strcmp(g_workload_type, "read") == 0 ? 100 : 0; 772 if (g_mix_specified) { 773 fprintf(stderr, "Ignoring -M option... Please use -M option" 774 " only when using rw or randrw.\n"); 775 } 776 } else if (strcmp(g_workload_type, "rw") == 0) { 777 if (g_rw_percentage < 0 || g_rw_percentage > 100) { 778 fprintf(stderr, 779 "-M must be specified to value from 0 to 100 " 780 "for rw or randrw.\n"); 781 return 1; 782 } 783 } else { 784 fprintf(stderr, 785 "io pattern type must be one of\n" 786 "(read, write, randread, randwrite, rw, randrw)\n"); 787 return 1; 788 } 789 790 if (TAILQ_EMPTY(&g_trid_list)) { 791 /* If no transport IDs specified, default to enumerating all local PCIe devices */ 792 add_trid("trtype:PCIe"); 793 } else { 794 struct trid_entry *trid_entry, *trid_entry_tmp; 795 796 g_no_pci = true; 797 /* check whether there is local PCIe type */ 798 TAILQ_FOREACH_SAFE(trid_entry, &g_trid_list, tailq, trid_entry_tmp) { 799 if (trid_entry->trid.trtype == SPDK_NVME_TRANSPORT_PCIE) { 800 g_no_pci = false; 801 break; 802 } 803 } 804 } 805 806 return 0; 807 } 808 809 static int 810 register_workers(void) 811 { 812 uint32_t i; 813 struct worker_thread *worker; 814 815 SPDK_ENV_FOREACH_CORE(i) { 816 worker = calloc(1, sizeof(*worker)); 817 if (worker == NULL) { 818 fprintf(stderr, "Unable to allocate worker\n"); 819 return -1; 820 } 821 822 TAILQ_INIT(&worker->ns_ctx); 823 TAILQ_INIT(&worker->ctrlr_ctx); 824 worker->lcore = i; 825 TAILQ_INSERT_TAIL(&g_workers, worker, link); 826 g_num_workers++; 827 } 828 829 return 0; 830 } 831 832 static void 833 unregister_workers(void) 834 { 835 struct worker_thread *worker, *tmp_worker; 836 struct ns_worker_ctx *ns_ctx, *tmp_ns_ctx; 837 struct ctrlr_worker_ctx *ctrlr_ctx, *tmp_ctrlr_ctx; 838 839 /* Free namespace context and worker thread */ 840 TAILQ_FOREACH_SAFE(worker, &g_workers, link, tmp_worker) { 841 TAILQ_REMOVE(&g_workers, worker, link); 842 843 TAILQ_FOREACH_SAFE(ns_ctx, &worker->ns_ctx, link, tmp_ns_ctx) { 844 TAILQ_REMOVE(&worker->ns_ctx, ns_ctx, link); 845 printf("NS: %s I/O completed: %" PRIu64 ", failed: %" PRIu64 "\n", 846 ns_ctx->entry->name, ns_ctx->io_completed, ns_ctx->io_failed); 847 free(ns_ctx); 848 } 849 850 TAILQ_FOREACH_SAFE(ctrlr_ctx, &worker->ctrlr_ctx, link, tmp_ctrlr_ctx) { 851 TAILQ_REMOVE(&worker->ctrlr_ctx, ctrlr_ctx, link); 852 printf("CTRLR: %s abort submitted %" PRIu64 ", failed to submit %" PRIu64 "\n", 853 ctrlr_ctx->entry->name, ctrlr_ctx->abort_submitted, 854 ctrlr_ctx->abort_submit_failed); 855 printf("\t success %" PRIu64 ", unsuccess %" PRIu64 ", failed %" PRIu64 "\n", 856 ctrlr_ctx->successful_abort, ctrlr_ctx->unsuccessful_abort, 857 ctrlr_ctx->abort_failed); 858 free(ctrlr_ctx); 859 } 860 861 free(worker); 862 } 863 } 864 865 static bool 866 probe_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 867 struct spdk_nvme_ctrlr_opts *opts) 868 { 869 uint16_t min_aq_size; 870 871 /* We need to make sure the admin queue is big enough to handle all of the aborts that 872 * will be sent by this test app. We add a few extra entries to account for any admin 873 * commands other than the aborts. */ 874 min_aq_size = spdk_divide_round_up(g_queue_depth, g_abort_interval) + 8; 875 opts->admin_queue_size = spdk_max(opts->admin_queue_size, min_aq_size); 876 877 /* Avoid possible nvme_qpair_abort_queued_reqs_with_cbarg ERROR when IO queue size is 128. */ 878 opts->disable_error_logging = true; 879 880 return true; 881 } 882 883 static void 884 attach_cb(void *cb_ctx, const struct spdk_nvme_transport_id *trid, 885 struct spdk_nvme_ctrlr *ctrlr, const struct spdk_nvme_ctrlr_opts *opts) 886 { 887 struct trid_entry *trid_entry = cb_ctx; 888 struct spdk_pci_addr pci_addr; 889 struct spdk_pci_device *pci_dev; 890 struct spdk_pci_id pci_id; 891 892 if (trid->trtype != SPDK_NVME_TRANSPORT_PCIE) { 893 printf("Attached to NVMe over Fabrics controller at %s:%s: %s\n", 894 trid->traddr, trid->trsvcid, 895 trid->subnqn); 896 } else { 897 if (spdk_pci_addr_parse(&pci_addr, trid->traddr)) { 898 return; 899 } 900 901 pci_dev = spdk_nvme_ctrlr_get_pci_device(ctrlr); 902 if (!pci_dev) { 903 return; 904 } 905 906 pci_id = spdk_pci_device_get_id(pci_dev); 907 908 printf("Attached to NVMe Controller at %s [%04x:%04x]\n", 909 trid->traddr, 910 pci_id.vendor_id, pci_id.device_id); 911 } 912 913 register_ctrlr(ctrlr, trid_entry); 914 } 915 916 static int 917 register_controllers(void) 918 { 919 struct trid_entry *trid_entry; 920 921 printf("Initializing NVMe Controllers\n"); 922 923 TAILQ_FOREACH(trid_entry, &g_trid_list, tailq) { 924 if (spdk_nvme_probe(&trid_entry->trid, trid_entry, probe_cb, attach_cb, NULL) != 0) { 925 fprintf(stderr, "spdk_nvme_probe() failed for transport address '%s'\n", 926 trid_entry->trid.traddr); 927 return -1; 928 } 929 } 930 931 return 0; 932 } 933 934 static void 935 unregister_controllers(void) 936 { 937 struct ctrlr_entry *entry, *tmp; 938 struct spdk_nvme_detach_ctx *detach_ctx = NULL; 939 940 TAILQ_FOREACH_SAFE(entry, &g_controllers, link, tmp) { 941 TAILQ_REMOVE(&g_controllers, entry, link); 942 spdk_nvme_detach_async(entry->ctrlr, &detach_ctx); 943 free(entry); 944 } 945 946 if (detach_ctx) { 947 spdk_nvme_detach_poll(detach_ctx); 948 } 949 } 950 951 static int 952 associate_main_worker_with_ctrlr(void) 953 { 954 struct ctrlr_entry *entry; 955 struct worker_thread *worker; 956 struct ctrlr_worker_ctx *ctrlr_ctx; 957 958 TAILQ_FOREACH(worker, &g_workers, link) { 959 if (worker->lcore == g_main_core) { 960 break; 961 } 962 } 963 964 if (!worker) { 965 return -1; 966 } 967 968 TAILQ_FOREACH(entry, &g_controllers, link) { 969 ctrlr_ctx = calloc(1, sizeof(struct ctrlr_worker_ctx)); 970 if (!ctrlr_ctx) { 971 return -1; 972 } 973 974 pthread_mutex_init(&ctrlr_ctx->mutex, NULL); 975 ctrlr_ctx->entry = entry; 976 ctrlr_ctx->ctrlr = entry->ctrlr; 977 978 TAILQ_INSERT_TAIL(&worker->ctrlr_ctx, ctrlr_ctx, link); 979 } 980 981 return 0; 982 } 983 984 static struct ctrlr_worker_ctx * 985 get_ctrlr_worker_ctx(struct spdk_nvme_ctrlr *ctrlr) 986 { 987 struct worker_thread *worker; 988 struct ctrlr_worker_ctx *ctrlr_ctx; 989 990 TAILQ_FOREACH(worker, &g_workers, link) { 991 if (worker->lcore == g_main_core) { 992 break; 993 } 994 } 995 996 if (!worker) { 997 return NULL; 998 } 999 1000 TAILQ_FOREACH(ctrlr_ctx, &worker->ctrlr_ctx, link) { 1001 if (ctrlr_ctx->ctrlr == ctrlr) { 1002 return ctrlr_ctx; 1003 } 1004 } 1005 1006 return NULL; 1007 } 1008 1009 static int 1010 associate_workers_with_ns(void) 1011 { 1012 struct ns_entry *entry = TAILQ_FIRST(&g_namespaces); 1013 struct worker_thread *worker = TAILQ_FIRST(&g_workers); 1014 struct ns_worker_ctx *ns_ctx; 1015 int i, count; 1016 1017 count = g_num_namespaces > g_num_workers ? g_num_namespaces : g_num_workers; 1018 1019 for (i = 0; i < count; i++) { 1020 if (entry == NULL) { 1021 break; 1022 } 1023 1024 ns_ctx = calloc(1, sizeof(struct ns_worker_ctx)); 1025 if (!ns_ctx) { 1026 return -1; 1027 } 1028 1029 printf("Associating %s with lcore %d\n", entry->name, worker->lcore); 1030 ns_ctx->entry = entry; 1031 ns_ctx->ctrlr_ctx = get_ctrlr_worker_ctx(entry->ctrlr); 1032 if (!ns_ctx->ctrlr_ctx) { 1033 free(ns_ctx); 1034 return -1; 1035 } 1036 1037 TAILQ_INSERT_TAIL(&worker->ns_ctx, ns_ctx, link); 1038 1039 worker = TAILQ_NEXT(worker, link); 1040 if (worker == NULL) { 1041 worker = TAILQ_FIRST(&g_workers); 1042 } 1043 1044 entry = TAILQ_NEXT(entry, link); 1045 if (entry == NULL) { 1046 entry = TAILQ_FIRST(&g_namespaces); 1047 } 1048 } 1049 1050 return 0; 1051 } 1052 1053 int 1054 main(int argc, char **argv) 1055 { 1056 int rc; 1057 struct worker_thread *worker, *main_worker; 1058 struct spdk_env_opts opts; 1059 1060 rc = parse_args(argc, argv); 1061 if (rc != 0) { 1062 return rc; 1063 } 1064 1065 spdk_env_opts_init(&opts); 1066 opts.name = "abort"; 1067 opts.shm_id = g_shm_id; 1068 if (g_core_mask) { 1069 opts.core_mask = g_core_mask; 1070 } 1071 1072 if (g_dpdk_mem) { 1073 opts.mem_size = g_dpdk_mem; 1074 } 1075 if (g_no_pci) { 1076 opts.no_pci = g_no_pci; 1077 } 1078 if (spdk_env_init(&opts) < 0) { 1079 fprintf(stderr, "Unable to initialize SPDK env\n"); 1080 unregister_trids(); 1081 return -1; 1082 } 1083 1084 g_tsc_rate = spdk_get_ticks_hz(); 1085 1086 if (register_workers() != 0) { 1087 rc = -1; 1088 goto cleanup; 1089 } 1090 1091 if (register_controllers() != 0) { 1092 rc = -1; 1093 goto cleanup; 1094 } 1095 1096 if (g_warn) { 1097 printf("WARNING: Some requested NVMe devices were skipped\n"); 1098 } 1099 1100 if (g_num_namespaces == 0) { 1101 fprintf(stderr, "No valid NVMe controllers found\n"); 1102 rc = -1; 1103 goto cleanup; 1104 } 1105 1106 if (associate_main_worker_with_ctrlr() != 0) { 1107 rc = -1; 1108 goto cleanup; 1109 } 1110 1111 if (associate_workers_with_ns() != 0) { 1112 rc = -1; 1113 goto cleanup; 1114 } 1115 1116 printf("Initialization complete. Launching workers.\n"); 1117 1118 /* Launch all of the secondary workers */ 1119 g_main_core = spdk_env_get_current_core(); 1120 main_worker = NULL; 1121 TAILQ_FOREACH(worker, &g_workers, link) { 1122 if (worker->lcore != g_main_core) { 1123 spdk_env_thread_launch_pinned(worker->lcore, work_fn, worker); 1124 } else { 1125 assert(main_worker == NULL); 1126 main_worker = worker; 1127 } 1128 } 1129 1130 assert(main_worker != NULL); 1131 rc = work_fn(main_worker); 1132 1133 spdk_env_thread_wait_all(); 1134 1135 TAILQ_FOREACH(worker, &g_workers, link) { 1136 if (worker->status != 0) { 1137 rc = 1; 1138 break; 1139 } 1140 } 1141 1142 cleanup: 1143 unregister_trids(); 1144 unregister_workers(); 1145 unregister_namespaces(); 1146 unregister_controllers(); 1147 1148 spdk_env_fini(); 1149 1150 if (rc != 0) { 1151 fprintf(stderr, "%s: errors occurred\n", argv[0]); 1152 } 1153 1154 return rc; 1155 } 1156