1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 #include "spdk/nvme.h" 36 #include "spdk/io_channel.h" 37 #include "spdk/bdev_module.h" 38 #include "spdk_internal/log.h" 39 #include "spdk/ftl.h" 40 #include "ftl_core.h" 41 #include "ftl_anm.h" 42 #include "ftl_io.h" 43 #include "ftl_reloc.h" 44 #include "ftl_rwb.h" 45 #include "ftl_band.h" 46 #include "ftl_debug.h" 47 48 #define FTL_CORE_RING_SIZE 4096 49 #define FTL_INIT_TIMEOUT 30 50 #define FTL_NSID 1 51 52 #define ftl_range_intersect(s1, e1, s2, e2) \ 53 ((s1) <= (e2) && (s2) <= (e1)) 54 55 struct ftl_admin_cmpl { 56 struct spdk_nvme_cpl status; 57 58 int complete; 59 }; 60 61 static STAILQ_HEAD(, spdk_ftl_dev) g_ftl_queue = STAILQ_HEAD_INITIALIZER(g_ftl_queue); 62 static pthread_mutex_t g_ftl_queue_lock = PTHREAD_MUTEX_INITIALIZER; 63 static const struct spdk_ftl_conf g_default_conf = { 64 .defrag = { 65 .limits = { 66 /* 5 free bands / 0 % host writes */ 67 [SPDK_FTL_LIMIT_CRIT] = { .thld = 5, .limit = 0 }, 68 /* 10 free bands / 5 % host writes */ 69 [SPDK_FTL_LIMIT_HIGH] = { .thld = 10, .limit = 5 }, 70 /* 20 free bands / 40 % host writes */ 71 [SPDK_FTL_LIMIT_LOW] = { .thld = 20, .limit = 40 }, 72 /* 40 free bands / 100 % host writes - defrag starts running */ 73 [SPDK_FTL_LIMIT_START] = { .thld = 40, .limit = 100 }, 74 }, 75 /* 10 percent valid lbks */ 76 .invalid_thld = 10, 77 }, 78 /* 20% spare lbks */ 79 .lba_rsvd = 20, 80 /* 6M write buffer */ 81 .rwb_size = 6 * 1024 * 1024, 82 /* 90% band fill threshold */ 83 .band_thld = 90, 84 /* Max 32 IO depth per band relocate */ 85 .max_reloc_qdepth = 32, 86 /* Max 3 active band relocates */ 87 .max_active_relocs = 3, 88 /* IO pool size per user thread (this should be adjusted to thread IO qdepth) */ 89 .user_io_pool_size = 2048, 90 }; 91 92 static void ftl_dev_free_sync(struct spdk_ftl_dev *dev); 93 94 static void 95 ftl_admin_cb(void *ctx, const struct spdk_nvme_cpl *cpl) 96 { 97 struct ftl_admin_cmpl *cmpl = ctx; 98 99 cmpl->complete = 1; 100 cmpl->status = *cpl; 101 } 102 103 static int 104 ftl_band_init_md(struct ftl_band *band) 105 { 106 struct ftl_md *md = &band->md; 107 108 md->vld_map = spdk_bit_array_create(ftl_num_band_lbks(band->dev)); 109 if (!md->vld_map) { 110 return -ENOMEM; 111 } 112 113 pthread_spin_init(&md->lock, PTHREAD_PROCESS_PRIVATE); 114 ftl_band_md_clear(&band->md); 115 return 0; 116 } 117 118 static int 119 ftl_check_init_opts(const struct spdk_ftl_dev_init_opts *opts, 120 const struct spdk_ocssd_geometry_data *geo) 121 { 122 struct spdk_ftl_dev *dev; 123 size_t num_punits = geo->num_pu * geo->num_grp; 124 int rc = 0; 125 126 if (opts->range.begin > opts->range.end || opts->range.end >= num_punits) { 127 return -1; 128 } 129 130 pthread_mutex_lock(&g_ftl_queue_lock); 131 132 STAILQ_FOREACH(dev, &g_ftl_queue, stailq) { 133 if (spdk_nvme_transport_id_compare(&dev->trid, &opts->trid)) { 134 continue; 135 } 136 137 if (ftl_range_intersect(opts->range.begin, opts->range.end, 138 dev->range.begin, dev->range.end)) { 139 rc = -1; 140 goto out; 141 } 142 } 143 144 out: 145 pthread_mutex_unlock(&g_ftl_queue_lock); 146 return rc; 147 } 148 149 static int 150 ftl_retrieve_bbt_page(struct spdk_ftl_dev *dev, uint64_t offset, 151 struct spdk_ocssd_chunk_information_entry *info, 152 unsigned int num_entries) 153 { 154 volatile struct ftl_admin_cmpl cmpl = {}; 155 uint32_t nsid = spdk_nvme_ns_get_id(dev->ns); 156 157 if (spdk_nvme_ctrlr_cmd_get_log_page(dev->ctrlr, SPDK_OCSSD_LOG_CHUNK_INFO, nsid, 158 info, num_entries * sizeof(*info), 159 offset * sizeof(*info), 160 ftl_admin_cb, (void *)&cmpl)) { 161 return -1; 162 } 163 164 while (!cmpl.complete) { 165 spdk_nvme_ctrlr_process_admin_completions(dev->ctrlr); 166 } 167 168 if (spdk_nvme_cpl_is_error(&cmpl.status)) { 169 SPDK_ERRLOG("Unexpected status code: [%d], status code type: [%d]\n", 170 cmpl.status.status.sc, cmpl.status.status.sct); 171 return -1; 172 } 173 174 return 0; 175 } 176 177 static int 178 ftl_retrieve_bbt(struct spdk_ftl_dev *dev, const struct ftl_punit *punit, 179 struct spdk_ocssd_chunk_information_entry *info) 180 { 181 uint32_t i = 0; 182 unsigned int num_entries = PAGE_SIZE / sizeof(*info); 183 uint64_t off = (punit->start_ppa.grp * dev->geo.num_pu + punit->start_ppa.pu) * 184 dev->geo.num_chk; 185 186 for (i = 0; i < dev->geo.num_chk; i += num_entries) { 187 if (num_entries > dev->geo.num_chk - i) { 188 num_entries = dev->geo.num_chk - i; 189 } 190 191 if (ftl_retrieve_bbt_page(dev, off + i, &info[i], num_entries)) { 192 return -1; 193 } 194 } 195 196 return 0; 197 } 198 199 static unsigned char 200 ftl_get_chunk_state(const struct spdk_ocssd_chunk_information_entry *info) 201 { 202 if (info->cs.free) { 203 return FTL_CHUNK_STATE_FREE; 204 } 205 206 if (info->cs.open) { 207 return FTL_CHUNK_STATE_OPEN; 208 } 209 210 if (info->cs.closed) { 211 return FTL_CHUNK_STATE_CLOSED; 212 } 213 214 if (info->cs.offline) { 215 return FTL_CHUNK_STATE_BAD; 216 } 217 218 assert(0 && "Invalid block state"); 219 return FTL_CHUNK_STATE_BAD; 220 } 221 222 static void 223 ftl_remove_empty_bands(struct spdk_ftl_dev *dev) 224 { 225 struct ftl_band *band, *temp_band; 226 227 /* Remove band from shut_bands list to prevent further processing */ 228 /* if all blocks on this band are bad */ 229 LIST_FOREACH_SAFE(band, &dev->shut_bands, list_entry, temp_band) { 230 if (!band->num_chunks) { 231 dev->num_bands--; 232 LIST_REMOVE(band, list_entry); 233 } 234 } 235 } 236 237 static int 238 ftl_dev_init_bands(struct spdk_ftl_dev *dev) 239 { 240 struct spdk_ocssd_chunk_information_entry *info; 241 struct ftl_band *band, *pband; 242 struct ftl_punit *punit; 243 struct ftl_chunk *chunk; 244 unsigned int i, j; 245 char buf[128]; 246 int rc = 0; 247 248 LIST_INIT(&dev->free_bands); 249 LIST_INIT(&dev->shut_bands); 250 251 dev->num_free = 0; 252 dev->num_bands = ftl_dev_num_bands(dev); 253 dev->bands = calloc(ftl_dev_num_bands(dev), sizeof(*dev->bands)); 254 if (!dev->bands) { 255 return -1; 256 } 257 258 info = calloc(dev->geo.num_chk, sizeof(*info)); 259 if (!info) { 260 return -1; 261 } 262 263 for (i = 0; i < ftl_dev_num_bands(dev); ++i) { 264 band = &dev->bands[i]; 265 band->id = i; 266 band->dev = dev; 267 band->state = FTL_BAND_STATE_CLOSED; 268 269 if (LIST_EMPTY(&dev->shut_bands)) { 270 LIST_INSERT_HEAD(&dev->shut_bands, band, list_entry); 271 } else { 272 LIST_INSERT_AFTER(pband, band, list_entry); 273 } 274 pband = band; 275 276 CIRCLEQ_INIT(&band->chunks); 277 band->chunk_buf = calloc(ftl_dev_num_punits(dev), sizeof(*band->chunk_buf)); 278 if (!band->chunk_buf) { 279 SPDK_ERRLOG("Failed to allocate block state table for band: [%u]\n", i); 280 rc = -1; 281 goto out; 282 } 283 284 rc = ftl_band_init_md(band); 285 if (rc) { 286 SPDK_ERRLOG("Failed to initialize metadata structures for band [%u]\n", i); 287 goto out; 288 } 289 } 290 291 for (i = 0; i < ftl_dev_num_punits(dev); ++i) { 292 punit = &dev->punits[i]; 293 294 rc = ftl_retrieve_bbt(dev, punit, info); 295 if (rc) { 296 SPDK_ERRLOG("Failed to retrieve bbt for @ppa: %s [%lu]\n", 297 ftl_ppa2str(punit->start_ppa, buf, sizeof(buf)), 298 ftl_ppa_addr_pack(dev, punit->start_ppa)); 299 goto out; 300 } 301 302 for (j = 0; j < ftl_dev_num_bands(dev); ++j) { 303 band = &dev->bands[j]; 304 chunk = &band->chunk_buf[i]; 305 chunk->pos = i; 306 chunk->state = ftl_get_chunk_state(&info[j]); 307 chunk->punit = punit; 308 chunk->start_ppa = punit->start_ppa; 309 chunk->start_ppa.chk = band->id; 310 311 if (chunk->state != FTL_CHUNK_STATE_BAD) { 312 band->num_chunks++; 313 CIRCLEQ_INSERT_TAIL(&band->chunks, chunk, circleq); 314 } 315 } 316 } 317 318 ftl_remove_empty_bands(dev); 319 out: 320 free(info); 321 return rc; 322 } 323 324 static int 325 ftl_dev_init_punits(struct spdk_ftl_dev *dev) 326 { 327 unsigned int i, punit; 328 329 dev->punits = calloc(ftl_dev_num_punits(dev), sizeof(*dev->punits)); 330 if (!dev->punits) { 331 return -1; 332 } 333 334 for (i = 0; i < ftl_dev_num_punits(dev); ++i) { 335 dev->punits[i].dev = dev; 336 punit = dev->range.begin + i; 337 338 dev->punits[i].start_ppa.ppa = 0; 339 dev->punits[i].start_ppa.grp = punit % dev->geo.num_grp; 340 dev->punits[i].start_ppa.pu = punit / dev->geo.num_grp; 341 } 342 343 return 0; 344 } 345 346 static int 347 ftl_dev_retrieve_geo(struct spdk_ftl_dev *dev) 348 { 349 volatile struct ftl_admin_cmpl cmpl = {}; 350 uint32_t nsid = spdk_nvme_ns_get_id(dev->ns); 351 352 if (spdk_nvme_ocssd_ctrlr_cmd_geometry(dev->ctrlr, nsid, &dev->geo, sizeof(dev->geo), 353 ftl_admin_cb, (void *)&cmpl)) { 354 SPDK_ERRLOG("Unable to retrieve geometry\n"); 355 return -1; 356 } 357 358 /* TODO: add a timeout */ 359 while (!cmpl.complete) { 360 spdk_nvme_ctrlr_process_admin_completions(dev->ctrlr); 361 } 362 363 if (spdk_nvme_cpl_is_error(&cmpl.status)) { 364 SPDK_ERRLOG("Unexpected status code: [%d], status code type: [%d]\n", 365 cmpl.status.status.sc, cmpl.status.status.sct); 366 return -1; 367 } 368 369 /* TODO: add sanity checks for the geo */ 370 dev->ppa_len = dev->geo.lbaf.grp_len + 371 dev->geo.lbaf.pu_len + 372 dev->geo.lbaf.chk_len + 373 dev->geo.lbaf.lbk_len; 374 375 dev->ppaf.lbk_offset = 0; 376 dev->ppaf.lbk_mask = (1 << dev->geo.lbaf.lbk_len) - 1; 377 dev->ppaf.chk_offset = dev->ppaf.lbk_offset + dev->geo.lbaf.lbk_len; 378 dev->ppaf.chk_mask = (1 << dev->geo.lbaf.chk_len) - 1; 379 dev->ppaf.pu_offset = dev->ppaf.chk_offset + dev->geo.lbaf.chk_len; 380 dev->ppaf.pu_mask = (1 << dev->geo.lbaf.pu_len) - 1; 381 dev->ppaf.grp_offset = dev->ppaf.pu_offset + dev->geo.lbaf.pu_len; 382 dev->ppaf.grp_mask = (1 << dev->geo.lbaf.grp_len) - 1; 383 384 /* We're using optimal write size as our xfer size */ 385 dev->xfer_size = dev->geo.ws_opt; 386 387 return 0; 388 } 389 390 static int 391 ftl_dev_nvme_init(struct spdk_ftl_dev *dev, const struct spdk_ftl_dev_init_opts *opts) 392 { 393 uint32_t block_size; 394 395 dev->ctrlr = opts->ctrlr; 396 397 if (spdk_nvme_ctrlr_get_num_ns(dev->ctrlr) != 1) { 398 SPDK_ERRLOG("Unsupported number of namespaces\n"); 399 return -1; 400 } 401 402 dev->ns = spdk_nvme_ctrlr_get_ns(dev->ctrlr, FTL_NSID); 403 dev->trid = opts->trid; 404 dev->md_size = spdk_nvme_ns_get_md_size(dev->ns); 405 406 block_size = spdk_nvme_ns_get_extended_sector_size(dev->ns); 407 if (block_size != FTL_BLOCK_SIZE) { 408 SPDK_ERRLOG("Unsupported block size (%"PRIu32")\n", block_size); 409 return -1; 410 } 411 412 if (dev->md_size % sizeof(uint32_t) != 0) { 413 /* Metadata pointer must be dword aligned */ 414 SPDK_ERRLOG("Unsupported metadata size (%zu)\n", dev->md_size); 415 return -1; 416 } 417 418 return 0; 419 } 420 421 static int 422 ftl_conf_validate(const struct spdk_ftl_conf *conf) 423 { 424 size_t i; 425 426 if (conf->defrag.invalid_thld >= 100) { 427 return -1; 428 } 429 if (conf->lba_rsvd >= 100) { 430 return -1; 431 } 432 if (conf->lba_rsvd == 0) { 433 return -1; 434 } 435 if (conf->rwb_size == 0) { 436 return -1; 437 } 438 if (conf->rwb_size % FTL_BLOCK_SIZE != 0) { 439 return -1; 440 } 441 442 for (i = 0; i < SPDK_FTL_LIMIT_MAX; ++i) { 443 if (conf->defrag.limits[i].limit > 100) { 444 return -1; 445 } 446 } 447 448 return 0; 449 } 450 451 void 452 spdk_ftl_conf_init_defaults(struct spdk_ftl_conf *conf) 453 { 454 *conf = g_default_conf; 455 } 456 457 static int 458 ftl_init_wptr_list(struct spdk_ftl_dev *dev) 459 { 460 #define POOL_NAME_LEN 128 461 char pool_name[POOL_NAME_LEN]; 462 int rc; 463 464 LIST_INIT(&dev->wptr_list); 465 LIST_INIT(&dev->flush_list); 466 467 rc = snprintf(pool_name, sizeof(pool_name), "%s-%s", dev->name, "ocssd-lba-pool"); 468 if (rc < 0 || rc >= POOL_NAME_LEN) { 469 return -ENAMETOOLONG; 470 } 471 472 /* We need to reserve at least 2 buffers for band close / open sequence 473 * alone, plus additional (8) buffers for handling write errors. 474 * TODO: This memory pool is utilized only by core thread - it introduce 475 * unnecessary overhead and should be replaced by different data structure. 476 */ 477 dev->lba_pool = spdk_mempool_create(pool_name, 2 + 8, 478 ftl_num_band_lbks(dev) * sizeof(uint64_t), 479 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 480 SPDK_ENV_SOCKET_ID_ANY); 481 if (!dev->lba_pool) { 482 return -ENOMEM; 483 } 484 485 return 0; 486 } 487 488 static size_t 489 ftl_dev_band_max_seq(struct spdk_ftl_dev *dev) 490 { 491 struct ftl_band *band; 492 size_t seq = 0; 493 494 LIST_FOREACH(band, &dev->shut_bands, list_entry) { 495 if (band->md.seq > seq) { 496 seq = band->md.seq; 497 } 498 } 499 500 return seq; 501 } 502 503 static void 504 _ftl_init_bands_state(void *ctx) 505 { 506 struct ftl_band *band, *temp_band; 507 struct spdk_ftl_dev *dev = ctx; 508 509 dev->seq = ftl_dev_band_max_seq(dev); 510 511 LIST_FOREACH_SAFE(band, &dev->shut_bands, list_entry, temp_band) { 512 if (!band->md.num_vld) { 513 ftl_band_set_state(band, FTL_BAND_STATE_FREE); 514 } 515 } 516 517 ftl_reloc_resume(dev->reloc); 518 /* Clear the limit applications as they're incremented incorrectly by */ 519 /* the initialization code */ 520 memset(dev->stats.limits, 0, sizeof(dev->stats.limits)); 521 } 522 523 static int 524 ftl_init_num_free_bands(struct spdk_ftl_dev *dev) 525 { 526 struct ftl_band *band; 527 int cnt = 0; 528 529 LIST_FOREACH(band, &dev->shut_bands, list_entry) { 530 if (band->num_chunks && !band->md.num_vld) { 531 cnt++; 532 } 533 } 534 return cnt; 535 } 536 537 static int 538 ftl_init_bands_state(struct spdk_ftl_dev *dev) 539 { 540 /* TODO: Should we abort initialization or expose read only device */ 541 /* if there is no free bands? */ 542 /* If we abort initialization should we depend on condition that */ 543 /* we have no free bands or should we have some minimal number of */ 544 /* free bands? */ 545 if (!ftl_init_num_free_bands(dev)) { 546 return -1; 547 } 548 549 spdk_thread_send_msg(ftl_get_core_thread(dev), _ftl_init_bands_state, dev); 550 return 0; 551 } 552 553 static void 554 _ftl_dev_init_thread(void *ctx) 555 { 556 struct ftl_thread *thread = ctx; 557 struct spdk_ftl_dev *dev = thread->dev; 558 559 thread->poller = spdk_poller_register(thread->poller_fn, thread, thread->period_us); 560 if (!thread->poller) { 561 SPDK_ERRLOG("Unable to register poller\n"); 562 assert(0); 563 } 564 565 if (spdk_get_thread() == ftl_get_core_thread(dev)) { 566 ftl_anm_register_device(dev, ftl_process_anm_event); 567 } 568 } 569 570 static int 571 ftl_dev_init_thread(struct spdk_ftl_dev *dev, struct ftl_thread *thread, 572 struct spdk_thread *spdk_thread, spdk_poller_fn fn, uint64_t period_us) 573 { 574 thread->dev = dev; 575 thread->poller_fn = fn; 576 thread->thread = spdk_thread; 577 thread->period_us = period_us; 578 579 thread->qpair = spdk_nvme_ctrlr_alloc_io_qpair(dev->ctrlr, NULL, 0); 580 if (!thread->qpair) { 581 SPDK_ERRLOG("Unable to initialize qpair\n"); 582 return -1; 583 } 584 585 spdk_thread_send_msg(spdk_thread, _ftl_dev_init_thread, thread); 586 return 0; 587 } 588 589 static int 590 ftl_dev_init_threads(struct spdk_ftl_dev *dev, const struct spdk_ftl_dev_init_opts *opts) 591 { 592 if (!opts->core_thread || !opts->read_thread) { 593 return -1; 594 } 595 596 if (ftl_dev_init_thread(dev, &dev->core_thread, opts->core_thread, ftl_task_core, 0)) { 597 SPDK_ERRLOG("Unable to initialize core thread\n"); 598 return -1; 599 } 600 601 if (ftl_dev_init_thread(dev, &dev->read_thread, opts->read_thread, ftl_task_read, 0)) { 602 SPDK_ERRLOG("Unable to initialize read thread\n"); 603 return -1; 604 } 605 606 return 0; 607 } 608 609 static void 610 ftl_dev_free_thread(struct spdk_ftl_dev *dev, struct ftl_thread *thread) 611 { 612 assert(thread->poller == NULL); 613 614 spdk_nvme_ctrlr_free_io_qpair(thread->qpair); 615 thread->thread = NULL; 616 thread->qpair = NULL; 617 } 618 619 static int 620 ftl_dev_l2p_alloc(struct spdk_ftl_dev *dev) 621 { 622 size_t addr_size; 623 uint64_t i; 624 625 if (dev->num_lbas == 0) { 626 SPDK_DEBUGLOG(SPDK_LOG_FTL_INIT, "Invalid l2p table size\n"); 627 return -1; 628 } 629 630 if (dev->l2p) { 631 SPDK_DEBUGLOG(SPDK_LOG_FTL_INIT, "L2p table already allocated\n"); 632 return -1; 633 } 634 635 addr_size = dev->ppa_len >= 32 ? 8 : 4; 636 dev->l2p = malloc(dev->num_lbas * addr_size); 637 if (!dev->l2p) { 638 SPDK_DEBUGLOG(SPDK_LOG_FTL_INIT, "Failed to allocate l2p table\n"); 639 return -1; 640 } 641 642 for (i = 0; i < dev->num_lbas; ++i) { 643 ftl_l2p_set(dev, i, ftl_to_ppa(FTL_PPA_INVALID)); 644 } 645 646 return 0; 647 } 648 649 static void 650 ftl_init_complete(struct spdk_ftl_dev *dev) 651 { 652 pthread_mutex_lock(&g_ftl_queue_lock); 653 STAILQ_INSERT_HEAD(&g_ftl_queue, dev, stailq); 654 pthread_mutex_unlock(&g_ftl_queue_lock); 655 656 dev->initialized = 1; 657 658 if (dev->init_cb) { 659 dev->init_cb(dev, dev->init_arg, 0); 660 } 661 662 dev->init_cb = NULL; 663 dev->init_arg = NULL; 664 } 665 666 static int 667 ftl_setup_initial_state(struct spdk_ftl_dev *dev) 668 { 669 struct spdk_ftl_conf *conf = &dev->conf; 670 size_t i; 671 672 spdk_uuid_generate(&dev->uuid); 673 674 dev->num_lbas = 0; 675 for (i = 0; i < ftl_dev_num_bands(dev); ++i) { 676 dev->num_lbas += ftl_band_num_usable_lbks(&dev->bands[i]); 677 } 678 679 dev->num_lbas = (dev->num_lbas * (100 - conf->lba_rsvd)) / 100; 680 681 if (ftl_dev_l2p_alloc(dev)) { 682 SPDK_ERRLOG("Unable to init l2p table\n"); 683 return -1; 684 } 685 686 if (ftl_init_bands_state(dev)) { 687 SPDK_ERRLOG("Unable to finish the initialization\n"); 688 return -1; 689 } 690 691 ftl_init_complete(dev); 692 return 0; 693 } 694 695 struct ftl_init_fail_ctx { 696 spdk_ftl_init_fn cb; 697 void *arg; 698 }; 699 700 static void 701 ftl_init_fail_cb(void *ctx, int status) 702 { 703 struct ftl_init_fail_ctx *fail_cb = ctx; 704 705 fail_cb->cb(NULL, fail_cb->arg, -ENODEV); 706 free(fail_cb); 707 } 708 709 static void 710 ftl_init_fail(struct spdk_ftl_dev *dev) 711 { 712 struct ftl_init_fail_ctx *fail_cb; 713 714 fail_cb = malloc(sizeof(*fail_cb)); 715 if (!fail_cb) { 716 SPDK_ERRLOG("Unable to allocate context to free the device\n"); 717 return; 718 } 719 720 fail_cb->cb = dev->init_cb; 721 fail_cb->arg = dev->init_arg; 722 dev->halt_cb = NULL; 723 724 if (spdk_ftl_dev_free(dev, ftl_init_fail_cb, fail_cb)) { 725 SPDK_ERRLOG("Unable to free the device\n"); 726 assert(0); 727 } 728 } 729 730 static void 731 ftl_restore_device_cb(struct spdk_ftl_dev *dev, struct ftl_restore *restore, int status) 732 { 733 if (status) { 734 SPDK_ERRLOG("Failed to restore the device from the SSD\n"); 735 goto error; 736 } 737 738 if (ftl_init_bands_state(dev)) { 739 SPDK_ERRLOG("Unable to finish the initialization\n"); 740 goto error; 741 } 742 743 ftl_init_complete(dev); 744 return; 745 error: 746 ftl_init_fail(dev); 747 } 748 749 static void 750 ftl_restore_md_cb(struct spdk_ftl_dev *dev, struct ftl_restore *restore, int status) 751 { 752 if (status) { 753 SPDK_ERRLOG("Failed to restore the metadata from the SSD\n"); 754 goto error; 755 } 756 757 /* After the metadata is read it should be possible to allocate the L2P */ 758 if (ftl_dev_l2p_alloc(dev)) { 759 SPDK_ERRLOG("Failed to allocate the L2P\n"); 760 goto error; 761 } 762 763 if (ftl_restore_device(restore, ftl_restore_device_cb)) { 764 SPDK_ERRLOG("Failed to start device restoration from the SSD\n"); 765 goto error; 766 } 767 768 return; 769 error: 770 ftl_init_fail(dev); 771 } 772 773 static int 774 ftl_restore_state(struct spdk_ftl_dev *dev, const struct spdk_ftl_dev_init_opts *opts) 775 { 776 dev->uuid = opts->uuid; 777 778 if (ftl_restore_md(dev, ftl_restore_md_cb)) { 779 SPDK_ERRLOG("Failed to start metadata restoration from the SSD\n"); 780 return -1; 781 } 782 783 return 0; 784 } 785 786 static int 787 ftl_io_channel_create_cb(void *io_device, void *ctx) 788 { 789 struct ftl_io_channel *ch = ctx; 790 char mempool_name[32]; 791 struct spdk_ftl_dev *dev = io_device; 792 793 snprintf(mempool_name, sizeof(mempool_name), "ftl_io_%p", ch); 794 ch->elem_size = sizeof(struct ftl_md_io); 795 ch->io_pool = spdk_mempool_create(mempool_name, 796 dev->conf.user_io_pool_size, 797 ch->elem_size, 798 0, 799 SPDK_ENV_SOCKET_ID_ANY); 800 801 if (!ch->io_pool) { 802 return -1; 803 } 804 805 return 0; 806 } 807 808 static void 809 ftl_io_channel_destroy_cb(void *io_device, void *ctx) 810 { 811 struct ftl_io_channel *ch = ctx; 812 813 spdk_mempool_free(ch->io_pool); 814 } 815 816 int 817 spdk_ftl_dev_init(const struct spdk_ftl_dev_init_opts *opts, spdk_ftl_init_fn cb, void *cb_arg) 818 { 819 struct spdk_ftl_dev *dev; 820 821 dev = calloc(1, sizeof(*dev)); 822 if (!dev) { 823 return -ENOMEM; 824 } 825 826 if (opts->conf) { 827 if (ftl_conf_validate(opts->conf)) { 828 SPDK_ERRLOG("Invalid configuration\n"); 829 goto fail_sync; 830 } 831 832 memcpy(&dev->conf, opts->conf, sizeof(dev->conf)); 833 } else { 834 spdk_ftl_conf_init_defaults(&dev->conf); 835 } 836 837 spdk_io_device_register(dev, ftl_io_channel_create_cb, ftl_io_channel_destroy_cb, 838 sizeof(struct ftl_io_channel), 839 NULL); 840 841 TAILQ_INIT(&dev->retry_queue); 842 dev->ioch = spdk_get_io_channel(dev); 843 dev->init_cb = cb; 844 dev->init_arg = cb_arg; 845 dev->range = opts->range; 846 dev->limit = SPDK_FTL_LIMIT_MAX; 847 dev->cache_bdev_desc = opts->cache_bdev_desc; 848 849 dev->name = strdup(opts->name); 850 if (!dev->name) { 851 SPDK_ERRLOG("Unable to set device name\n"); 852 goto fail_sync; 853 } 854 855 if (ftl_dev_nvme_init(dev, opts)) { 856 SPDK_ERRLOG("Unable to initialize NVMe structures\n"); 857 goto fail_sync; 858 } 859 860 /* In case of errors, we free all of the memory in ftl_dev_free_sync(), */ 861 /* so we don't have to clean up in each of the init functions. */ 862 if (ftl_dev_retrieve_geo(dev)) { 863 SPDK_ERRLOG("Unable to retrieve geometry\n"); 864 goto fail_sync; 865 } 866 867 if (ftl_check_init_opts(opts, &dev->geo)) { 868 SPDK_ERRLOG("Invalid device configuration\n"); 869 goto fail_sync; 870 } 871 872 if (ftl_dev_init_punits(dev)) { 873 SPDK_ERRLOG("Unable to initialize LUNs\n"); 874 goto fail_sync; 875 } 876 877 if (ftl_init_wptr_list(dev)) { 878 SPDK_ERRLOG("Unable to init wptr\n"); 879 goto fail_sync; 880 } 881 882 if (ftl_dev_init_bands(dev)) { 883 SPDK_ERRLOG("Unable to initialize band array\n"); 884 goto fail_sync; 885 } 886 887 dev->rwb = ftl_rwb_init(&dev->conf, dev->geo.ws_opt, dev->md_size); 888 if (!dev->rwb) { 889 SPDK_ERRLOG("Unable to initialize rwb structures\n"); 890 goto fail_sync; 891 } 892 893 dev->reloc = ftl_reloc_init(dev); 894 if (!dev->reloc) { 895 SPDK_ERRLOG("Unable to initialize reloc structures\n"); 896 goto fail_sync; 897 } 898 899 if (ftl_dev_init_threads(dev, opts)) { 900 SPDK_ERRLOG("Unable to initialize device threads\n"); 901 goto fail_sync; 902 } 903 904 if (opts->mode & SPDK_FTL_MODE_CREATE) { 905 if (ftl_setup_initial_state(dev)) { 906 SPDK_ERRLOG("Failed to setup initial state of the device\n"); 907 goto fail_async; 908 } 909 910 } else { 911 if (ftl_restore_state(dev, opts)) { 912 SPDK_ERRLOG("Unable to restore device's state from the SSD\n"); 913 goto fail_async; 914 } 915 } 916 917 return 0; 918 fail_sync: 919 ftl_dev_free_sync(dev); 920 return -ENOMEM; 921 fail_async: 922 ftl_init_fail(dev); 923 return 0; 924 } 925 926 static void 927 _ftl_halt_defrag(void *arg) 928 { 929 ftl_reloc_halt(((struct spdk_ftl_dev *)arg)->reloc); 930 } 931 932 static void 933 ftl_dev_free_sync(struct spdk_ftl_dev *dev) 934 { 935 struct spdk_ftl_dev *iter; 936 size_t i; 937 938 if (!dev) { 939 return; 940 } 941 942 pthread_mutex_lock(&g_ftl_queue_lock); 943 STAILQ_FOREACH(iter, &g_ftl_queue, stailq) { 944 if (iter == dev) { 945 STAILQ_REMOVE(&g_ftl_queue, dev, spdk_ftl_dev, stailq); 946 break; 947 } 948 } 949 pthread_mutex_unlock(&g_ftl_queue_lock); 950 951 assert(LIST_EMPTY(&dev->wptr_list)); 952 953 ftl_dev_dump_bands(dev); 954 ftl_dev_dump_stats(dev); 955 956 spdk_put_io_channel(dev->ioch); 957 spdk_io_device_unregister(dev, NULL); 958 959 if (dev->bands) { 960 for (i = 0; i < ftl_dev_num_bands(dev); ++i) { 961 free(dev->bands[i].chunk_buf); 962 spdk_bit_array_free(&dev->bands[i].md.vld_map); 963 } 964 } 965 966 spdk_mempool_free(dev->lba_pool); 967 968 ftl_rwb_free(dev->rwb); 969 ftl_reloc_free(dev->reloc); 970 971 free(dev->name); 972 free(dev->punits); 973 free(dev->bands); 974 free(dev->l2p); 975 free(dev); 976 } 977 978 static int 979 ftl_halt_poller(void *ctx) 980 { 981 struct spdk_ftl_dev *dev = ctx; 982 spdk_ftl_fn halt_cb = dev->halt_cb; 983 void *halt_arg = dev->halt_arg; 984 985 if (!dev->core_thread.poller && !dev->read_thread.poller) { 986 spdk_poller_unregister(&dev->halt_poller); 987 988 ftl_dev_free_thread(dev, &dev->read_thread); 989 ftl_dev_free_thread(dev, &dev->core_thread); 990 991 ftl_anm_unregister_device(dev); 992 ftl_dev_free_sync(dev); 993 994 if (halt_cb) { 995 halt_cb(halt_arg, 0); 996 } 997 } 998 999 return 0; 1000 } 1001 1002 static void 1003 ftl_add_halt_poller(void *ctx) 1004 { 1005 struct spdk_ftl_dev *dev = ctx; 1006 1007 _ftl_halt_defrag(dev); 1008 1009 assert(!dev->halt_poller); 1010 dev->halt_poller = spdk_poller_register(ftl_halt_poller, dev, 100); 1011 } 1012 1013 int 1014 spdk_ftl_dev_free(struct spdk_ftl_dev *dev, spdk_ftl_fn cb, void *cb_arg) 1015 { 1016 if (dev->halt_cb) { 1017 return -EBUSY; 1018 } 1019 1020 dev->halt_cb = cb; 1021 dev->halt_arg = cb_arg; 1022 dev->halt = 1; 1023 1024 spdk_thread_send_msg(ftl_get_core_thread(dev), ftl_add_halt_poller, dev); 1025 return 0; 1026 } 1027 1028 int 1029 spdk_ftl_module_init(const struct ftl_module_init_opts *opts, spdk_ftl_fn cb, void *cb_arg) 1030 { 1031 return ftl_anm_init(opts->anm_thread, cb, cb_arg); 1032 } 1033 1034 int 1035 spdk_ftl_module_fini(spdk_ftl_fn cb, void *cb_arg) 1036 { 1037 return ftl_anm_free(cb, cb_arg); 1038 } 1039 1040 SPDK_LOG_REGISTER_COMPONENT("ftl_init", SPDK_LOG_FTL_INIT) 1041