1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 #include "spdk/nvme.h" 36 #include "spdk/io_channel.h" 37 #include "spdk/bdev_module.h" 38 #include "spdk_internal/log.h" 39 #include "spdk/ftl.h" 40 #include "ftl_core.h" 41 #include "ftl_anm.h" 42 #include "ftl_io.h" 43 #include "ftl_reloc.h" 44 #include "ftl_rwb.h" 45 #include "ftl_band.h" 46 #include "ftl_debug.h" 47 48 #define FTL_CORE_RING_SIZE 4096 49 #define FTL_INIT_TIMEOUT 30 50 #define FTL_NSID 1 51 52 #define ftl_range_intersect(s1, e1, s2, e2) \ 53 ((s1) <= (e2) && (s2) <= (e1)) 54 55 struct ftl_admin_cmpl { 56 struct spdk_nvme_cpl status; 57 58 int complete; 59 }; 60 61 static STAILQ_HEAD(, spdk_ftl_dev) g_ftl_queue = STAILQ_HEAD_INITIALIZER(g_ftl_queue); 62 static pthread_mutex_t g_ftl_queue_lock = PTHREAD_MUTEX_INITIALIZER; 63 static const struct spdk_ftl_conf g_default_conf = { 64 .defrag = { 65 .limits = { 66 /* 5 free bands / 0 % host writes */ 67 [SPDK_FTL_LIMIT_CRIT] = { .thld = 5, .limit = 0 }, 68 /* 10 free bands / 5 % host writes */ 69 [SPDK_FTL_LIMIT_HIGH] = { .thld = 10, .limit = 5 }, 70 /* 20 free bands / 40 % host writes */ 71 [SPDK_FTL_LIMIT_LOW] = { .thld = 20, .limit = 40 }, 72 /* 40 free bands / 100 % host writes - defrag starts running */ 73 [SPDK_FTL_LIMIT_START] = { .thld = 40, .limit = 100 }, 74 }, 75 /* 10 percent valid lbks */ 76 .invalid_thld = 10, 77 }, 78 /* 20% spare lbks */ 79 .lba_rsvd = 20, 80 /* 6M write buffer */ 81 .rwb_size = 6 * 1024 * 1024, 82 /* 90% band fill threshold */ 83 .band_thld = 90, 84 /* Max 32 IO depth per band relocate */ 85 .max_reloc_qdepth = 32, 86 /* Max 3 active band relocates */ 87 .max_active_relocs = 3, 88 /* IO pool size per user thread (this should be adjusted to thread IO qdepth) */ 89 .user_io_pool_size = 2048, 90 /* Number of interleaving units per ws_opt */ 91 /* 1 for default and 3 for 3D TLC NAND */ 92 .num_interleave_units = 1, 93 }; 94 95 static void ftl_dev_free_sync(struct spdk_ftl_dev *dev); 96 97 static void 98 ftl_admin_cb(void *ctx, const struct spdk_nvme_cpl *cpl) 99 { 100 struct ftl_admin_cmpl *cmpl = ctx; 101 102 cmpl->complete = 1; 103 cmpl->status = *cpl; 104 } 105 106 static int 107 ftl_band_init_md(struct ftl_band *band) 108 { 109 struct ftl_md *md = &band->md; 110 111 md->vld_map = spdk_bit_array_create(ftl_num_band_lbks(band->dev)); 112 if (!md->vld_map) { 113 return -ENOMEM; 114 } 115 116 pthread_spin_init(&md->lock, PTHREAD_PROCESS_PRIVATE); 117 ftl_band_md_clear(&band->md); 118 return 0; 119 } 120 121 static int 122 ftl_check_conf(const struct spdk_ftl_conf *conf, 123 const struct spdk_ocssd_geometry_data *geo) 124 { 125 size_t i; 126 127 if (conf->defrag.invalid_thld >= 100) { 128 return -1; 129 } 130 if (conf->lba_rsvd >= 100) { 131 return -1; 132 } 133 if (conf->lba_rsvd == 0) { 134 return -1; 135 } 136 if (conf->rwb_size == 0) { 137 return -1; 138 } 139 if (conf->rwb_size % FTL_BLOCK_SIZE != 0) { 140 return -1; 141 } 142 if (geo->ws_opt % conf->num_interleave_units != 0) { 143 return -1; 144 } 145 146 for (i = 0; i < SPDK_FTL_LIMIT_MAX; ++i) { 147 if (conf->defrag.limits[i].limit > 100) { 148 return -1; 149 } 150 } 151 152 return 0; 153 } 154 155 static int 156 ftl_check_init_opts(const struct spdk_ftl_dev_init_opts *opts, 157 const struct spdk_ocssd_geometry_data *geo) 158 { 159 struct spdk_ftl_dev *dev; 160 size_t num_punits = geo->num_pu * geo->num_grp; 161 int rc = 0; 162 163 if (opts->range.begin > opts->range.end || opts->range.end >= num_punits) { 164 return -1; 165 } 166 167 if (ftl_check_conf(opts->conf, geo)) { 168 return -1; 169 } 170 171 pthread_mutex_lock(&g_ftl_queue_lock); 172 173 STAILQ_FOREACH(dev, &g_ftl_queue, stailq) { 174 if (spdk_nvme_transport_id_compare(&dev->trid, &opts->trid)) { 175 continue; 176 } 177 178 if (ftl_range_intersect(opts->range.begin, opts->range.end, 179 dev->range.begin, dev->range.end)) { 180 rc = -1; 181 goto out; 182 } 183 } 184 185 out: 186 pthread_mutex_unlock(&g_ftl_queue_lock); 187 return rc; 188 } 189 190 static int 191 ftl_retrieve_bbt_page(struct spdk_ftl_dev *dev, uint64_t offset, 192 struct spdk_ocssd_chunk_information_entry *info, 193 unsigned int num_entries) 194 { 195 volatile struct ftl_admin_cmpl cmpl = {}; 196 uint32_t nsid = spdk_nvme_ns_get_id(dev->ns); 197 198 if (spdk_nvme_ctrlr_cmd_get_log_page(dev->ctrlr, SPDK_OCSSD_LOG_CHUNK_INFO, nsid, 199 info, num_entries * sizeof(*info), 200 offset * sizeof(*info), 201 ftl_admin_cb, (void *)&cmpl)) { 202 return -1; 203 } 204 205 while (!cmpl.complete) { 206 spdk_nvme_ctrlr_process_admin_completions(dev->ctrlr); 207 } 208 209 if (spdk_nvme_cpl_is_error(&cmpl.status)) { 210 SPDK_ERRLOG("Unexpected status code: [%d], status code type: [%d]\n", 211 cmpl.status.status.sc, cmpl.status.status.sct); 212 return -1; 213 } 214 215 return 0; 216 } 217 218 static int 219 ftl_retrieve_bbt(struct spdk_ftl_dev *dev, const struct ftl_punit *punit, 220 struct spdk_ocssd_chunk_information_entry *info) 221 { 222 uint32_t i = 0; 223 unsigned int num_entries = PAGE_SIZE / sizeof(*info); 224 uint64_t off = (punit->start_ppa.grp * dev->geo.num_pu + punit->start_ppa.pu) * 225 dev->geo.num_chk; 226 227 for (i = 0; i < dev->geo.num_chk; i += num_entries) { 228 if (num_entries > dev->geo.num_chk - i) { 229 num_entries = dev->geo.num_chk - i; 230 } 231 232 if (ftl_retrieve_bbt_page(dev, off + i, &info[i], num_entries)) { 233 return -1; 234 } 235 } 236 237 return 0; 238 } 239 240 static unsigned char 241 ftl_get_chunk_state(const struct spdk_ocssd_chunk_information_entry *info) 242 { 243 if (info->cs.free) { 244 return FTL_CHUNK_STATE_FREE; 245 } 246 247 if (info->cs.open) { 248 return FTL_CHUNK_STATE_OPEN; 249 } 250 251 if (info->cs.closed) { 252 return FTL_CHUNK_STATE_CLOSED; 253 } 254 255 if (info->cs.offline) { 256 return FTL_CHUNK_STATE_BAD; 257 } 258 259 assert(0 && "Invalid block state"); 260 return FTL_CHUNK_STATE_BAD; 261 } 262 263 static void 264 ftl_remove_empty_bands(struct spdk_ftl_dev *dev) 265 { 266 struct ftl_band *band, *temp_band; 267 268 /* Remove band from shut_bands list to prevent further processing */ 269 /* if all blocks on this band are bad */ 270 LIST_FOREACH_SAFE(band, &dev->shut_bands, list_entry, temp_band) { 271 if (!band->num_chunks) { 272 dev->num_bands--; 273 LIST_REMOVE(band, list_entry); 274 } 275 } 276 } 277 278 static int 279 ftl_dev_init_bands(struct spdk_ftl_dev *dev) 280 { 281 struct spdk_ocssd_chunk_information_entry *info; 282 struct ftl_band *band, *pband; 283 struct ftl_punit *punit; 284 struct ftl_chunk *chunk; 285 unsigned int i, j; 286 char buf[128]; 287 int rc = 0; 288 289 LIST_INIT(&dev->free_bands); 290 LIST_INIT(&dev->shut_bands); 291 292 dev->num_free = 0; 293 dev->num_bands = ftl_dev_num_bands(dev); 294 dev->bands = calloc(ftl_dev_num_bands(dev), sizeof(*dev->bands)); 295 if (!dev->bands) { 296 return -1; 297 } 298 299 info = calloc(dev->geo.num_chk, sizeof(*info)); 300 if (!info) { 301 return -1; 302 } 303 304 for (i = 0; i < ftl_dev_num_bands(dev); ++i) { 305 band = &dev->bands[i]; 306 band->id = i; 307 band->dev = dev; 308 band->state = FTL_BAND_STATE_CLOSED; 309 310 if (LIST_EMPTY(&dev->shut_bands)) { 311 LIST_INSERT_HEAD(&dev->shut_bands, band, list_entry); 312 } else { 313 LIST_INSERT_AFTER(pband, band, list_entry); 314 } 315 pband = band; 316 317 CIRCLEQ_INIT(&band->chunks); 318 band->chunk_buf = calloc(ftl_dev_num_punits(dev), sizeof(*band->chunk_buf)); 319 if (!band->chunk_buf) { 320 SPDK_ERRLOG("Failed to allocate block state table for band: [%u]\n", i); 321 rc = -1; 322 goto out; 323 } 324 325 rc = ftl_band_init_md(band); 326 if (rc) { 327 SPDK_ERRLOG("Failed to initialize metadata structures for band [%u]\n", i); 328 goto out; 329 } 330 } 331 332 for (i = 0; i < ftl_dev_num_punits(dev); ++i) { 333 punit = &dev->punits[i]; 334 335 rc = ftl_retrieve_bbt(dev, punit, info); 336 if (rc) { 337 SPDK_ERRLOG("Failed to retrieve bbt for @ppa: %s [%lu]\n", 338 ftl_ppa2str(punit->start_ppa, buf, sizeof(buf)), 339 ftl_ppa_addr_pack(dev, punit->start_ppa)); 340 goto out; 341 } 342 343 for (j = 0; j < ftl_dev_num_bands(dev); ++j) { 344 band = &dev->bands[j]; 345 chunk = &band->chunk_buf[i]; 346 chunk->pos = i; 347 chunk->state = ftl_get_chunk_state(&info[j]); 348 chunk->punit = punit; 349 chunk->start_ppa = punit->start_ppa; 350 chunk->start_ppa.chk = band->id; 351 352 if (chunk->state != FTL_CHUNK_STATE_BAD) { 353 band->num_chunks++; 354 CIRCLEQ_INSERT_TAIL(&band->chunks, chunk, circleq); 355 } 356 } 357 } 358 359 ftl_remove_empty_bands(dev); 360 out: 361 free(info); 362 return rc; 363 } 364 365 static int 366 ftl_dev_init_punits(struct spdk_ftl_dev *dev) 367 { 368 unsigned int i, punit; 369 370 dev->punits = calloc(ftl_dev_num_punits(dev), sizeof(*dev->punits)); 371 if (!dev->punits) { 372 return -1; 373 } 374 375 for (i = 0; i < ftl_dev_num_punits(dev); ++i) { 376 dev->punits[i].dev = dev; 377 punit = dev->range.begin + i; 378 379 dev->punits[i].start_ppa.ppa = 0; 380 dev->punits[i].start_ppa.grp = punit % dev->geo.num_grp; 381 dev->punits[i].start_ppa.pu = punit / dev->geo.num_grp; 382 } 383 384 return 0; 385 } 386 387 static int 388 ftl_dev_retrieve_geo(struct spdk_ftl_dev *dev) 389 { 390 volatile struct ftl_admin_cmpl cmpl = {}; 391 uint32_t nsid = spdk_nvme_ns_get_id(dev->ns); 392 393 if (spdk_nvme_ocssd_ctrlr_cmd_geometry(dev->ctrlr, nsid, &dev->geo, sizeof(dev->geo), 394 ftl_admin_cb, (void *)&cmpl)) { 395 SPDK_ERRLOG("Unable to retrieve geometry\n"); 396 return -1; 397 } 398 399 /* TODO: add a timeout */ 400 while (!cmpl.complete) { 401 spdk_nvme_ctrlr_process_admin_completions(dev->ctrlr); 402 } 403 404 if (spdk_nvme_cpl_is_error(&cmpl.status)) { 405 SPDK_ERRLOG("Unexpected status code: [%d], status code type: [%d]\n", 406 cmpl.status.status.sc, cmpl.status.status.sct); 407 return -1; 408 } 409 410 /* TODO: add sanity checks for the geo */ 411 dev->ppa_len = dev->geo.lbaf.grp_len + 412 dev->geo.lbaf.pu_len + 413 dev->geo.lbaf.chk_len + 414 dev->geo.lbaf.lbk_len; 415 416 dev->ppaf.lbk_offset = 0; 417 dev->ppaf.lbk_mask = (1 << dev->geo.lbaf.lbk_len) - 1; 418 dev->ppaf.chk_offset = dev->ppaf.lbk_offset + dev->geo.lbaf.lbk_len; 419 dev->ppaf.chk_mask = (1 << dev->geo.lbaf.chk_len) - 1; 420 dev->ppaf.pu_offset = dev->ppaf.chk_offset + dev->geo.lbaf.chk_len; 421 dev->ppaf.pu_mask = (1 << dev->geo.lbaf.pu_len) - 1; 422 dev->ppaf.grp_offset = dev->ppaf.pu_offset + dev->geo.lbaf.pu_len; 423 dev->ppaf.grp_mask = (1 << dev->geo.lbaf.grp_len) - 1; 424 425 /* We're using optimal write size as our xfer size */ 426 dev->xfer_size = dev->geo.ws_opt; 427 428 return 0; 429 } 430 431 static int 432 ftl_dev_nvme_init(struct spdk_ftl_dev *dev, const struct spdk_ftl_dev_init_opts *opts) 433 { 434 uint32_t block_size; 435 436 dev->ctrlr = opts->ctrlr; 437 438 if (spdk_nvme_ctrlr_get_num_ns(dev->ctrlr) != 1) { 439 SPDK_ERRLOG("Unsupported number of namespaces\n"); 440 return -1; 441 } 442 443 dev->ns = spdk_nvme_ctrlr_get_ns(dev->ctrlr, FTL_NSID); 444 dev->trid = opts->trid; 445 dev->md_size = spdk_nvme_ns_get_md_size(dev->ns); 446 447 block_size = spdk_nvme_ns_get_extended_sector_size(dev->ns); 448 if (block_size != FTL_BLOCK_SIZE) { 449 SPDK_ERRLOG("Unsupported block size (%"PRIu32")\n", block_size); 450 return -1; 451 } 452 453 if (dev->md_size % sizeof(uint32_t) != 0) { 454 /* Metadata pointer must be dword aligned */ 455 SPDK_ERRLOG("Unsupported metadata size (%zu)\n", dev->md_size); 456 return -1; 457 } 458 459 return 0; 460 } 461 462 static int 463 ftl_dev_init_nv_cache(struct spdk_ftl_dev *dev, struct spdk_bdev_desc *bdev_desc) 464 { 465 struct spdk_bdev *bdev; 466 467 if (!bdev_desc) { 468 return 0; 469 } 470 471 bdev = spdk_bdev_desc_get_bdev(bdev_desc); 472 SPDK_INFOLOG(SPDK_LOG_FTL_INIT, "Using %s as write buffer cache\n", 473 spdk_bdev_get_name(bdev)); 474 475 if (spdk_bdev_get_block_size(bdev) != FTL_BLOCK_SIZE) { 476 SPDK_ERRLOG("Unsupported block size (%d)\n", spdk_bdev_get_block_size(bdev)); 477 return -1; 478 } 479 480 /* The cache needs to be capable of storing at least two full bands. This requirement comes 481 * from the fact that cache works as a protection against power loss, so before the data 482 * inside the cache can be overwritten, the band it's stored on has to be closed. 483 */ 484 if (spdk_bdev_get_num_blocks(bdev) < ftl_num_band_lbks(dev) * 2) { 485 SPDK_ERRLOG("Insufficient number of blocks for write buffer cache(%"PRIu64"\n", 486 spdk_bdev_get_num_blocks(bdev)); 487 return -1; 488 } 489 490 if (pthread_spin_init(&dev->nv_cache.lock, PTHREAD_PROCESS_PRIVATE)) { 491 SPDK_ERRLOG("Failed to initialize cache lock\n"); 492 return -1; 493 } 494 495 dev->nv_cache.bdev_desc = bdev_desc; 496 dev->nv_cache.current_addr = 0; 497 dev->nv_cache.num_available = spdk_bdev_get_num_blocks(bdev); 498 499 return 0; 500 } 501 502 void 503 spdk_ftl_conf_init_defaults(struct spdk_ftl_conf *conf) 504 { 505 *conf = g_default_conf; 506 } 507 508 static int 509 ftl_init_wptr_list(struct spdk_ftl_dev *dev) 510 { 511 #define POOL_NAME_LEN 128 512 char pool_name[POOL_NAME_LEN]; 513 int rc; 514 515 LIST_INIT(&dev->wptr_list); 516 LIST_INIT(&dev->flush_list); 517 518 rc = snprintf(pool_name, sizeof(pool_name), "%s-%s", dev->name, "ocssd-lba-pool"); 519 if (rc < 0 || rc >= POOL_NAME_LEN) { 520 return -ENAMETOOLONG; 521 } 522 523 /* We need to reserve at least 2 buffers for band close / open sequence 524 * alone, plus additional (8) buffers for handling write errors. 525 * TODO: This memory pool is utilized only by core thread - it introduce 526 * unnecessary overhead and should be replaced by different data structure. 527 */ 528 dev->lba_pool = spdk_mempool_create(pool_name, 2 + 8, 529 ftl_num_band_lbks(dev) * sizeof(uint64_t), 530 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 531 SPDK_ENV_SOCKET_ID_ANY); 532 if (!dev->lba_pool) { 533 return -ENOMEM; 534 } 535 536 return 0; 537 } 538 539 static size_t 540 ftl_dev_band_max_seq(struct spdk_ftl_dev *dev) 541 { 542 struct ftl_band *band; 543 size_t seq = 0; 544 545 LIST_FOREACH(band, &dev->shut_bands, list_entry) { 546 if (band->md.seq > seq) { 547 seq = band->md.seq; 548 } 549 } 550 551 return seq; 552 } 553 554 static void 555 _ftl_init_bands_state(void *ctx) 556 { 557 struct ftl_band *band, *temp_band; 558 struct spdk_ftl_dev *dev = ctx; 559 560 dev->seq = ftl_dev_band_max_seq(dev); 561 562 LIST_FOREACH_SAFE(band, &dev->shut_bands, list_entry, temp_band) { 563 if (!band->md.num_vld) { 564 ftl_band_set_state(band, FTL_BAND_STATE_FREE); 565 } 566 } 567 568 ftl_reloc_resume(dev->reloc); 569 /* Clear the limit applications as they're incremented incorrectly by */ 570 /* the initialization code */ 571 memset(dev->stats.limits, 0, sizeof(dev->stats.limits)); 572 } 573 574 static int 575 ftl_init_num_free_bands(struct spdk_ftl_dev *dev) 576 { 577 struct ftl_band *band; 578 int cnt = 0; 579 580 LIST_FOREACH(band, &dev->shut_bands, list_entry) { 581 if (band->num_chunks && !band->md.num_vld) { 582 cnt++; 583 } 584 } 585 return cnt; 586 } 587 588 static int 589 ftl_init_bands_state(struct spdk_ftl_dev *dev) 590 { 591 /* TODO: Should we abort initialization or expose read only device */ 592 /* if there is no free bands? */ 593 /* If we abort initialization should we depend on condition that */ 594 /* we have no free bands or should we have some minimal number of */ 595 /* free bands? */ 596 if (!ftl_init_num_free_bands(dev)) { 597 return -1; 598 } 599 600 spdk_thread_send_msg(ftl_get_core_thread(dev), _ftl_init_bands_state, dev); 601 return 0; 602 } 603 604 static void 605 _ftl_dev_init_thread(void *ctx) 606 { 607 struct ftl_thread *thread = ctx; 608 struct spdk_ftl_dev *dev = thread->dev; 609 610 thread->poller = spdk_poller_register(thread->poller_fn, thread, thread->period_us); 611 if (!thread->poller) { 612 SPDK_ERRLOG("Unable to register poller\n"); 613 assert(0); 614 } 615 616 if (spdk_get_thread() == ftl_get_core_thread(dev)) { 617 ftl_anm_register_device(dev, ftl_process_anm_event); 618 } 619 } 620 621 static int 622 ftl_dev_init_thread(struct spdk_ftl_dev *dev, struct ftl_thread *thread, 623 struct spdk_thread *spdk_thread, spdk_poller_fn fn, uint64_t period_us) 624 { 625 thread->dev = dev; 626 thread->poller_fn = fn; 627 thread->thread = spdk_thread; 628 thread->period_us = period_us; 629 630 thread->qpair = spdk_nvme_ctrlr_alloc_io_qpair(dev->ctrlr, NULL, 0); 631 if (!thread->qpair) { 632 SPDK_ERRLOG("Unable to initialize qpair\n"); 633 return -1; 634 } 635 636 spdk_thread_send_msg(spdk_thread, _ftl_dev_init_thread, thread); 637 return 0; 638 } 639 640 static int 641 ftl_dev_init_threads(struct spdk_ftl_dev *dev, const struct spdk_ftl_dev_init_opts *opts) 642 { 643 if (!opts->core_thread || !opts->read_thread) { 644 return -1; 645 } 646 647 if (ftl_dev_init_thread(dev, &dev->core_thread, opts->core_thread, ftl_task_core, 0)) { 648 SPDK_ERRLOG("Unable to initialize core thread\n"); 649 return -1; 650 } 651 652 if (ftl_dev_init_thread(dev, &dev->read_thread, opts->read_thread, ftl_task_read, 0)) { 653 SPDK_ERRLOG("Unable to initialize read thread\n"); 654 return -1; 655 } 656 657 return 0; 658 } 659 660 static void 661 ftl_dev_free_thread(struct spdk_ftl_dev *dev, struct ftl_thread *thread) 662 { 663 assert(thread->poller == NULL); 664 665 spdk_nvme_ctrlr_free_io_qpair(thread->qpair); 666 thread->thread = NULL; 667 thread->qpair = NULL; 668 } 669 670 static int 671 ftl_dev_l2p_alloc(struct spdk_ftl_dev *dev) 672 { 673 size_t addr_size; 674 uint64_t i; 675 676 if (dev->num_lbas == 0) { 677 SPDK_DEBUGLOG(SPDK_LOG_FTL_INIT, "Invalid l2p table size\n"); 678 return -1; 679 } 680 681 if (dev->l2p) { 682 SPDK_DEBUGLOG(SPDK_LOG_FTL_INIT, "L2p table already allocated\n"); 683 return -1; 684 } 685 686 addr_size = dev->ppa_len >= 32 ? 8 : 4; 687 dev->l2p = malloc(dev->num_lbas * addr_size); 688 if (!dev->l2p) { 689 SPDK_DEBUGLOG(SPDK_LOG_FTL_INIT, "Failed to allocate l2p table\n"); 690 return -1; 691 } 692 693 for (i = 0; i < dev->num_lbas; ++i) { 694 ftl_l2p_set(dev, i, ftl_to_ppa(FTL_PPA_INVALID)); 695 } 696 697 return 0; 698 } 699 700 static void 701 ftl_init_complete(struct spdk_ftl_dev *dev) 702 { 703 pthread_mutex_lock(&g_ftl_queue_lock); 704 STAILQ_INSERT_HEAD(&g_ftl_queue, dev, stailq); 705 pthread_mutex_unlock(&g_ftl_queue_lock); 706 707 dev->initialized = 1; 708 709 if (dev->init_cb) { 710 dev->init_cb(dev, dev->init_arg, 0); 711 } 712 713 dev->init_cb = NULL; 714 dev->init_arg = NULL; 715 } 716 717 static int 718 ftl_setup_initial_state(struct spdk_ftl_dev *dev) 719 { 720 struct spdk_ftl_conf *conf = &dev->conf; 721 size_t i; 722 723 spdk_uuid_generate(&dev->uuid); 724 725 dev->num_lbas = 0; 726 for (i = 0; i < ftl_dev_num_bands(dev); ++i) { 727 dev->num_lbas += ftl_band_num_usable_lbks(&dev->bands[i]); 728 } 729 730 dev->num_lbas = (dev->num_lbas * (100 - conf->lba_rsvd)) / 100; 731 732 if (ftl_dev_l2p_alloc(dev)) { 733 SPDK_ERRLOG("Unable to init l2p table\n"); 734 return -1; 735 } 736 737 if (ftl_init_bands_state(dev)) { 738 SPDK_ERRLOG("Unable to finish the initialization\n"); 739 return -1; 740 } 741 742 ftl_init_complete(dev); 743 return 0; 744 } 745 746 struct ftl_init_fail_ctx { 747 spdk_ftl_init_fn cb; 748 void *arg; 749 }; 750 751 static void 752 ftl_init_fail_cb(void *ctx, int status) 753 { 754 struct ftl_init_fail_ctx *fail_cb = ctx; 755 756 fail_cb->cb(NULL, fail_cb->arg, -ENODEV); 757 free(fail_cb); 758 } 759 760 static void 761 ftl_init_fail(struct spdk_ftl_dev *dev) 762 { 763 struct ftl_init_fail_ctx *fail_cb; 764 765 fail_cb = malloc(sizeof(*fail_cb)); 766 if (!fail_cb) { 767 SPDK_ERRLOG("Unable to allocate context to free the device\n"); 768 return; 769 } 770 771 fail_cb->cb = dev->init_cb; 772 fail_cb->arg = dev->init_arg; 773 dev->halt_cb = NULL; 774 775 if (spdk_ftl_dev_free(dev, ftl_init_fail_cb, fail_cb)) { 776 SPDK_ERRLOG("Unable to free the device\n"); 777 assert(0); 778 } 779 } 780 781 static void 782 ftl_restore_device_cb(struct spdk_ftl_dev *dev, struct ftl_restore *restore, int status) 783 { 784 if (status) { 785 SPDK_ERRLOG("Failed to restore the device from the SSD\n"); 786 goto error; 787 } 788 789 if (ftl_init_bands_state(dev)) { 790 SPDK_ERRLOG("Unable to finish the initialization\n"); 791 goto error; 792 } 793 794 ftl_init_complete(dev); 795 return; 796 error: 797 ftl_init_fail(dev); 798 } 799 800 static void 801 ftl_restore_md_cb(struct spdk_ftl_dev *dev, struct ftl_restore *restore, int status) 802 { 803 if (status) { 804 SPDK_ERRLOG("Failed to restore the metadata from the SSD\n"); 805 goto error; 806 } 807 808 /* After the metadata is read it should be possible to allocate the L2P */ 809 if (ftl_dev_l2p_alloc(dev)) { 810 SPDK_ERRLOG("Failed to allocate the L2P\n"); 811 goto error; 812 } 813 814 if (ftl_restore_device(restore, ftl_restore_device_cb)) { 815 SPDK_ERRLOG("Failed to start device restoration from the SSD\n"); 816 goto error; 817 } 818 819 return; 820 error: 821 ftl_init_fail(dev); 822 } 823 824 static int 825 ftl_restore_state(struct spdk_ftl_dev *dev, const struct spdk_ftl_dev_init_opts *opts) 826 { 827 dev->uuid = opts->uuid; 828 829 if (ftl_restore_md(dev, ftl_restore_md_cb)) { 830 SPDK_ERRLOG("Failed to start metadata restoration from the SSD\n"); 831 return -1; 832 } 833 834 return 0; 835 } 836 837 static int 838 ftl_io_channel_create_cb(void *io_device, void *ctx) 839 { 840 struct spdk_ftl_dev *dev = io_device; 841 struct ftl_io_channel *ioch = ctx; 842 char mempool_name[32]; 843 844 snprintf(mempool_name, sizeof(mempool_name), "ftl_io_%p", ioch); 845 ioch->cache_ioch = NULL; 846 ioch->elem_size = sizeof(struct ftl_md_io); 847 ioch->io_pool = spdk_mempool_create(mempool_name, 848 dev->conf.user_io_pool_size, 849 ioch->elem_size, 850 0, 851 SPDK_ENV_SOCKET_ID_ANY); 852 if (!ioch->io_pool) { 853 SPDK_ERRLOG("Failed to create IO channel's IO pool\n"); 854 return -1; 855 } 856 857 if (dev->nv_cache.bdev_desc) { 858 ioch->cache_ioch = spdk_bdev_get_io_channel(dev->nv_cache.bdev_desc); 859 if (!ioch->cache_ioch) { 860 SPDK_ERRLOG("Failed to create cache IO channel\n"); 861 spdk_mempool_free(ioch->io_pool); 862 return -1; 863 } 864 } 865 866 return 0; 867 } 868 869 static void 870 ftl_io_channel_destroy_cb(void *io_device, void *ctx) 871 { 872 struct ftl_io_channel *ioch = ctx; 873 874 spdk_mempool_free(ioch->io_pool); 875 876 if (ioch->cache_ioch) { 877 spdk_put_io_channel(ioch->cache_ioch); 878 } 879 } 880 881 static int 882 ftl_dev_init_io_channel(struct spdk_ftl_dev *dev) 883 { 884 spdk_io_device_register(dev, ftl_io_channel_create_cb, ftl_io_channel_destroy_cb, 885 sizeof(struct ftl_io_channel), 886 NULL); 887 888 dev->ioch = spdk_get_io_channel(dev); 889 if (!dev->ioch) { 890 spdk_io_device_unregister(dev, NULL); 891 return -1; 892 } 893 894 return 0; 895 } 896 897 int 898 spdk_ftl_dev_init(const struct spdk_ftl_dev_init_opts *_opts, spdk_ftl_init_fn cb, void *cb_arg) 899 { 900 struct spdk_ftl_dev *dev; 901 struct spdk_ftl_dev_init_opts opts = *_opts; 902 903 dev = calloc(1, sizeof(*dev)); 904 if (!dev) { 905 return -ENOMEM; 906 } 907 908 if (!opts.conf) { 909 opts.conf = &g_default_conf; 910 } 911 912 TAILQ_INIT(&dev->retry_queue); 913 dev->conf = *opts.conf; 914 dev->init_cb = cb; 915 dev->init_arg = cb_arg; 916 dev->range = opts.range; 917 dev->limit = SPDK_FTL_LIMIT_MAX; 918 919 dev->name = strdup(opts.name); 920 if (!dev->name) { 921 SPDK_ERRLOG("Unable to set device name\n"); 922 goto fail_sync; 923 } 924 925 if (ftl_dev_nvme_init(dev, &opts)) { 926 SPDK_ERRLOG("Unable to initialize NVMe structures\n"); 927 goto fail_sync; 928 } 929 930 /* In case of errors, we free all of the memory in ftl_dev_free_sync(), */ 931 /* so we don't have to clean up in each of the init functions. */ 932 if (ftl_dev_retrieve_geo(dev)) { 933 SPDK_ERRLOG("Unable to retrieve geometry\n"); 934 goto fail_sync; 935 } 936 937 if (ftl_check_init_opts(&opts, &dev->geo)) { 938 SPDK_ERRLOG("Invalid device configuration\n"); 939 goto fail_sync; 940 } 941 942 if (ftl_dev_init_punits(dev)) { 943 SPDK_ERRLOG("Unable to initialize LUNs\n"); 944 goto fail_sync; 945 } 946 947 if (ftl_init_wptr_list(dev)) { 948 SPDK_ERRLOG("Unable to init wptr\n"); 949 goto fail_sync; 950 } 951 952 if (ftl_dev_init_bands(dev)) { 953 SPDK_ERRLOG("Unable to initialize band array\n"); 954 goto fail_sync; 955 } 956 957 if (ftl_dev_init_nv_cache(dev, opts.cache_bdev_desc)) { 958 SPDK_ERRLOG("Unable to initialize persistent cache\n"); 959 goto fail_sync; 960 } 961 962 dev->rwb = ftl_rwb_init(&dev->conf, dev->geo.ws_opt, dev->md_size, ftl_dev_num_punits(dev)); 963 if (!dev->rwb) { 964 SPDK_ERRLOG("Unable to initialize rwb structures\n"); 965 goto fail_sync; 966 } 967 968 dev->reloc = ftl_reloc_init(dev); 969 if (!dev->reloc) { 970 SPDK_ERRLOG("Unable to initialize reloc structures\n"); 971 goto fail_sync; 972 } 973 974 if (ftl_dev_init_io_channel(dev)) { 975 SPDK_ERRLOG("Unable to initialize IO channels\n"); 976 goto fail_sync; 977 } 978 979 if (ftl_dev_init_threads(dev, &opts)) { 980 SPDK_ERRLOG("Unable to initialize device threads\n"); 981 goto fail_sync; 982 } 983 984 if (opts.mode & SPDK_FTL_MODE_CREATE) { 985 if (ftl_setup_initial_state(dev)) { 986 SPDK_ERRLOG("Failed to setup initial state of the device\n"); 987 goto fail_async; 988 } 989 } else { 990 if (ftl_restore_state(dev, &opts)) { 991 SPDK_ERRLOG("Unable to restore device's state from the SSD\n"); 992 goto fail_async; 993 } 994 } 995 996 return 0; 997 fail_sync: 998 ftl_dev_free_sync(dev); 999 return -ENOMEM; 1000 fail_async: 1001 ftl_init_fail(dev); 1002 return 0; 1003 } 1004 1005 static void 1006 _ftl_halt_defrag(void *arg) 1007 { 1008 ftl_reloc_halt(((struct spdk_ftl_dev *)arg)->reloc); 1009 } 1010 1011 static void 1012 ftl_dev_free_sync(struct spdk_ftl_dev *dev) 1013 { 1014 struct spdk_ftl_dev *iter; 1015 size_t i; 1016 1017 if (!dev) { 1018 return; 1019 } 1020 1021 pthread_mutex_lock(&g_ftl_queue_lock); 1022 STAILQ_FOREACH(iter, &g_ftl_queue, stailq) { 1023 if (iter == dev) { 1024 STAILQ_REMOVE(&g_ftl_queue, dev, spdk_ftl_dev, stailq); 1025 break; 1026 } 1027 } 1028 pthread_mutex_unlock(&g_ftl_queue_lock); 1029 1030 assert(LIST_EMPTY(&dev->wptr_list)); 1031 1032 ftl_dev_dump_bands(dev); 1033 ftl_dev_dump_stats(dev); 1034 1035 if (dev->ioch) { 1036 spdk_put_io_channel(dev->ioch); 1037 spdk_io_device_unregister(dev, NULL); 1038 } 1039 1040 if (dev->bands) { 1041 for (i = 0; i < ftl_dev_num_bands(dev); ++i) { 1042 free(dev->bands[i].chunk_buf); 1043 spdk_bit_array_free(&dev->bands[i].md.vld_map); 1044 } 1045 } 1046 1047 spdk_mempool_free(dev->lba_pool); 1048 1049 ftl_rwb_free(dev->rwb); 1050 ftl_reloc_free(dev->reloc); 1051 1052 free(dev->name); 1053 free(dev->punits); 1054 free(dev->bands); 1055 free(dev->l2p); 1056 free(dev); 1057 } 1058 1059 static int 1060 ftl_halt_poller(void *ctx) 1061 { 1062 struct spdk_ftl_dev *dev = ctx; 1063 spdk_ftl_fn halt_cb = dev->halt_cb; 1064 void *halt_arg = dev->halt_arg; 1065 1066 if (!dev->core_thread.poller && !dev->read_thread.poller) { 1067 spdk_poller_unregister(&dev->halt_poller); 1068 1069 ftl_dev_free_thread(dev, &dev->read_thread); 1070 ftl_dev_free_thread(dev, &dev->core_thread); 1071 1072 ftl_anm_unregister_device(dev); 1073 ftl_dev_free_sync(dev); 1074 1075 if (halt_cb) { 1076 halt_cb(halt_arg, 0); 1077 } 1078 } 1079 1080 return 0; 1081 } 1082 1083 static void 1084 ftl_add_halt_poller(void *ctx) 1085 { 1086 struct spdk_ftl_dev *dev = ctx; 1087 1088 _ftl_halt_defrag(dev); 1089 1090 assert(!dev->halt_poller); 1091 dev->halt_poller = spdk_poller_register(ftl_halt_poller, dev, 100); 1092 } 1093 1094 int 1095 spdk_ftl_dev_free(struct spdk_ftl_dev *dev, spdk_ftl_fn cb, void *cb_arg) 1096 { 1097 if (dev->halt_cb) { 1098 return -EBUSY; 1099 } 1100 1101 dev->halt_cb = cb; 1102 dev->halt_arg = cb_arg; 1103 dev->halt = 1; 1104 1105 ftl_rwb_disable_interleaving(dev->rwb); 1106 1107 spdk_thread_send_msg(ftl_get_core_thread(dev), ftl_add_halt_poller, dev); 1108 return 0; 1109 } 1110 1111 int 1112 spdk_ftl_module_init(const struct ftl_module_init_opts *opts, spdk_ftl_fn cb, void *cb_arg) 1113 { 1114 return ftl_anm_init(opts->anm_thread, cb, cb_arg); 1115 } 1116 1117 int 1118 spdk_ftl_module_fini(spdk_ftl_fn cb, void *cb_arg) 1119 { 1120 return ftl_anm_free(cb, cb_arg); 1121 } 1122 1123 SPDK_LOG_REGISTER_COMPONENT("ftl_init", SPDK_LOG_FTL_INIT) 1124