1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 #include "spdk/nvme.h" 36 #include "spdk/io_channel.h" 37 #include "spdk/bdev_module.h" 38 #include "spdk/string.h" 39 #include "spdk/likely.h" 40 #include "spdk_internal/log.h" 41 #include "spdk/ftl.h" 42 #include "ftl_core.h" 43 #include "ftl_anm.h" 44 #include "ftl_io.h" 45 #include "ftl_reloc.h" 46 #include "ftl_rwb.h" 47 #include "ftl_band.h" 48 #include "ftl_debug.h" 49 50 #define FTL_CORE_RING_SIZE 4096 51 #define FTL_INIT_TIMEOUT 30 52 #define FTL_NSID 1 53 54 #define ftl_range_intersect(s1, e1, s2, e2) \ 55 ((s1) <= (e2) && (s2) <= (e1)) 56 57 struct ftl_admin_cmpl { 58 struct spdk_nvme_cpl status; 59 60 int complete; 61 }; 62 63 static STAILQ_HEAD(, spdk_ftl_dev) g_ftl_queue = STAILQ_HEAD_INITIALIZER(g_ftl_queue); 64 static pthread_mutex_t g_ftl_queue_lock = PTHREAD_MUTEX_INITIALIZER; 65 static const struct spdk_ftl_conf g_default_conf = { 66 .defrag = { 67 .limits = { 68 /* 5 free bands / 0 % host writes */ 69 [SPDK_FTL_LIMIT_CRIT] = { .thld = 5, .limit = 0 }, 70 /* 10 free bands / 5 % host writes */ 71 [SPDK_FTL_LIMIT_HIGH] = { .thld = 10, .limit = 5 }, 72 /* 20 free bands / 40 % host writes */ 73 [SPDK_FTL_LIMIT_LOW] = { .thld = 20, .limit = 40 }, 74 /* 40 free bands / 100 % host writes - defrag starts running */ 75 [SPDK_FTL_LIMIT_START] = { .thld = 40, .limit = 100 }, 76 }, 77 /* 10 percent valid lbks */ 78 .invalid_thld = 10, 79 }, 80 /* 20% spare lbks */ 81 .lba_rsvd = 20, 82 /* 6M write buffer */ 83 .rwb_size = 6 * 1024 * 1024, 84 /* 90% band fill threshold */ 85 .band_thld = 90, 86 /* Max 32 IO depth per band relocate */ 87 .max_reloc_qdepth = 32, 88 /* Max 3 active band relocates */ 89 .max_active_relocs = 3, 90 /* IO pool size per user thread (this should be adjusted to thread IO qdepth) */ 91 .user_io_pool_size = 2048, 92 /* Number of interleaving units per ws_opt */ 93 /* 1 for default and 3 for 3D TLC NAND */ 94 .num_interleave_units = 1, 95 /* 96 * If clear ftl will return error when restoring after a dirty shutdown 97 * If set, last band will be padded, ftl will restore based only on closed bands - this 98 * will result in lost data after recovery. 99 */ 100 .allow_open_bands = false, 101 .nv_cache = { 102 /* Maximum number of concurrent requests */ 103 .max_request_cnt = 2048, 104 /* Maximum number of blocks per request */ 105 .max_request_size = 16, 106 } 107 }; 108 109 static void ftl_dev_free_sync(struct spdk_ftl_dev *dev); 110 111 static void 112 ftl_admin_cb(void *ctx, const struct spdk_nvme_cpl *cpl) 113 { 114 struct ftl_admin_cmpl *cmpl = ctx; 115 116 cmpl->complete = 1; 117 cmpl->status = *cpl; 118 } 119 120 static int 121 ftl_band_init_md(struct ftl_band *band) 122 { 123 struct ftl_lba_map *lba_map = &band->lba_map; 124 125 lba_map->vld = spdk_bit_array_create(ftl_num_band_lbks(band->dev)); 126 if (!lba_map->vld) { 127 return -ENOMEM; 128 } 129 130 pthread_spin_init(&lba_map->lock, PTHREAD_PROCESS_PRIVATE); 131 ftl_band_md_clear(band); 132 return 0; 133 } 134 135 static int 136 ftl_check_conf(const struct spdk_ftl_conf *conf, 137 const struct spdk_ocssd_geometry_data *geo) 138 { 139 size_t i; 140 141 if (conf->defrag.invalid_thld >= 100) { 142 return -1; 143 } 144 if (conf->lba_rsvd >= 100) { 145 return -1; 146 } 147 if (conf->lba_rsvd == 0) { 148 return -1; 149 } 150 if (conf->rwb_size == 0) { 151 return -1; 152 } 153 if (conf->rwb_size % FTL_BLOCK_SIZE != 0) { 154 return -1; 155 } 156 if (geo->ws_opt % conf->num_interleave_units != 0) { 157 return -1; 158 } 159 160 for (i = 0; i < SPDK_FTL_LIMIT_MAX; ++i) { 161 if (conf->defrag.limits[i].limit > 100) { 162 return -1; 163 } 164 } 165 166 return 0; 167 } 168 169 static int 170 ftl_check_init_opts(const struct spdk_ftl_dev_init_opts *opts, 171 const struct spdk_ocssd_geometry_data *geo) 172 { 173 struct spdk_ftl_dev *dev; 174 size_t num_punits = geo->num_pu * geo->num_grp; 175 int rc = 0; 176 177 if (opts->range.begin > opts->range.end || opts->range.end >= num_punits) { 178 return -1; 179 } 180 181 if (ftl_check_conf(opts->conf, geo)) { 182 return -1; 183 } 184 185 pthread_mutex_lock(&g_ftl_queue_lock); 186 187 STAILQ_FOREACH(dev, &g_ftl_queue, stailq) { 188 if (spdk_nvme_transport_id_compare(&dev->trid, &opts->trid)) { 189 continue; 190 } 191 192 if (ftl_range_intersect(opts->range.begin, opts->range.end, 193 dev->range.begin, dev->range.end)) { 194 rc = -1; 195 goto out; 196 } 197 } 198 199 out: 200 pthread_mutex_unlock(&g_ftl_queue_lock); 201 return rc; 202 } 203 204 int 205 ftl_retrieve_chunk_info(struct spdk_ftl_dev *dev, struct ftl_ppa ppa, 206 struct spdk_ocssd_chunk_information_entry *info, 207 unsigned int num_entries) 208 { 209 volatile struct ftl_admin_cmpl cmpl = {}; 210 uint32_t nsid = spdk_nvme_ns_get_id(dev->ns); 211 uint64_t offset = (ppa.grp * dev->geo.num_pu + ppa.pu) * 212 dev->geo.num_chk + ppa.chk; 213 int rc; 214 215 rc = spdk_nvme_ctrlr_cmd_get_log_page(dev->ctrlr, SPDK_OCSSD_LOG_CHUNK_INFO, nsid, 216 info, num_entries * sizeof(*info), 217 offset * sizeof(*info), 218 ftl_admin_cb, (void *)&cmpl); 219 if (spdk_unlikely(rc != 0)) { 220 SPDK_ERRLOG("spdk_nvme_ctrlr_cmd_get_log_page: %s\n", spdk_strerror(-rc)); 221 return -1; 222 } 223 224 while (!cmpl.complete) { 225 spdk_nvme_ctrlr_process_admin_completions(dev->ctrlr); 226 } 227 228 if (spdk_nvme_cpl_is_error(&cmpl.status)) { 229 SPDK_ERRLOG("Unexpected status code: [%d], status code type: [%d]\n", 230 cmpl.status.status.sc, cmpl.status.status.sct); 231 return -1; 232 } 233 234 return 0; 235 } 236 237 static int 238 ftl_retrieve_punit_chunk_info(struct spdk_ftl_dev *dev, const struct ftl_punit *punit, 239 struct spdk_ocssd_chunk_information_entry *info) 240 { 241 uint32_t i = 0; 242 unsigned int num_entries = FTL_BLOCK_SIZE / sizeof(*info); 243 struct ftl_ppa chunk_ppa = punit->start_ppa; 244 char ppa_buf[128]; 245 246 for (i = 0; i < dev->geo.num_chk; i += num_entries, chunk_ppa.chk += num_entries) { 247 if (num_entries > dev->geo.num_chk - i) { 248 num_entries = dev->geo.num_chk - i; 249 } 250 251 if (ftl_retrieve_chunk_info(dev, chunk_ppa, &info[i], num_entries)) { 252 SPDK_ERRLOG("Failed to retrieve chunk information @ppa: %s\n", 253 ftl_ppa2str(chunk_ppa, ppa_buf, sizeof(ppa_buf))); 254 return -1; 255 } 256 } 257 258 return 0; 259 } 260 261 static unsigned char 262 ftl_get_chunk_state(const struct spdk_ocssd_chunk_information_entry *info) 263 { 264 if (info->cs.free) { 265 return FTL_CHUNK_STATE_FREE; 266 } 267 268 if (info->cs.open) { 269 return FTL_CHUNK_STATE_OPEN; 270 } 271 272 if (info->cs.closed) { 273 return FTL_CHUNK_STATE_CLOSED; 274 } 275 276 if (info->cs.offline) { 277 return FTL_CHUNK_STATE_BAD; 278 } 279 280 assert(0 && "Invalid block state"); 281 return FTL_CHUNK_STATE_BAD; 282 } 283 284 static void 285 ftl_remove_empty_bands(struct spdk_ftl_dev *dev) 286 { 287 struct ftl_band *band, *temp_band; 288 289 /* Remove band from shut_bands list to prevent further processing */ 290 /* if all blocks on this band are bad */ 291 LIST_FOREACH_SAFE(band, &dev->shut_bands, list_entry, temp_band) { 292 if (!band->num_chunks) { 293 dev->num_bands--; 294 LIST_REMOVE(band, list_entry); 295 } 296 } 297 } 298 299 static int 300 ftl_dev_init_bands(struct spdk_ftl_dev *dev) 301 { 302 struct spdk_ocssd_chunk_information_entry *info; 303 struct ftl_band *band, *pband; 304 struct ftl_punit *punit; 305 struct ftl_chunk *chunk; 306 unsigned int i, j; 307 char buf[128]; 308 int rc = 0; 309 310 LIST_INIT(&dev->free_bands); 311 LIST_INIT(&dev->shut_bands); 312 313 dev->num_free = 0; 314 dev->num_bands = ftl_dev_num_bands(dev); 315 dev->bands = calloc(ftl_dev_num_bands(dev), sizeof(*dev->bands)); 316 if (!dev->bands) { 317 return -1; 318 } 319 320 info = calloc(dev->geo.num_chk, sizeof(*info)); 321 if (!info) { 322 return -1; 323 } 324 325 for (i = 0; i < ftl_dev_num_bands(dev); ++i) { 326 band = &dev->bands[i]; 327 band->id = i; 328 band->dev = dev; 329 band->state = FTL_BAND_STATE_CLOSED; 330 331 if (LIST_EMPTY(&dev->shut_bands)) { 332 LIST_INSERT_HEAD(&dev->shut_bands, band, list_entry); 333 } else { 334 LIST_INSERT_AFTER(pband, band, list_entry); 335 } 336 pband = band; 337 338 CIRCLEQ_INIT(&band->chunks); 339 band->chunk_buf = calloc(ftl_dev_num_punits(dev), sizeof(*band->chunk_buf)); 340 if (!band->chunk_buf) { 341 SPDK_ERRLOG("Failed to allocate block state table for band: [%u]\n", i); 342 rc = -1; 343 goto out; 344 } 345 346 rc = ftl_band_init_md(band); 347 if (rc) { 348 SPDK_ERRLOG("Failed to initialize metadata structures for band [%u]\n", i); 349 goto out; 350 } 351 } 352 353 for (i = 0; i < ftl_dev_num_punits(dev); ++i) { 354 punit = &dev->punits[i]; 355 356 rc = ftl_retrieve_punit_chunk_info(dev, punit, info); 357 if (rc) { 358 SPDK_ERRLOG("Failed to retrieve bbt for @ppa: %s [%lu]\n", 359 ftl_ppa2str(punit->start_ppa, buf, sizeof(buf)), 360 ftl_ppa_addr_pack(dev, punit->start_ppa)); 361 goto out; 362 } 363 364 for (j = 0; j < ftl_dev_num_bands(dev); ++j) { 365 band = &dev->bands[j]; 366 chunk = &band->chunk_buf[i]; 367 chunk->pos = i; 368 chunk->state = ftl_get_chunk_state(&info[j]); 369 chunk->punit = punit; 370 chunk->start_ppa = punit->start_ppa; 371 chunk->start_ppa.chk = band->id; 372 373 if (chunk->state != FTL_CHUNK_STATE_BAD) { 374 band->num_chunks++; 375 CIRCLEQ_INSERT_TAIL(&band->chunks, chunk, circleq); 376 } 377 } 378 } 379 380 ftl_remove_empty_bands(dev); 381 out: 382 free(info); 383 return rc; 384 } 385 386 static int 387 ftl_dev_init_punits(struct spdk_ftl_dev *dev) 388 { 389 unsigned int i, punit; 390 391 dev->punits = calloc(ftl_dev_num_punits(dev), sizeof(*dev->punits)); 392 if (!dev->punits) { 393 return -1; 394 } 395 396 for (i = 0; i < ftl_dev_num_punits(dev); ++i) { 397 dev->punits[i].dev = dev; 398 punit = dev->range.begin + i; 399 400 dev->punits[i].start_ppa.ppa = 0; 401 dev->punits[i].start_ppa.grp = punit % dev->geo.num_grp; 402 dev->punits[i].start_ppa.pu = punit / dev->geo.num_grp; 403 } 404 405 return 0; 406 } 407 408 static int 409 ftl_dev_retrieve_geo(struct spdk_ftl_dev *dev) 410 { 411 volatile struct ftl_admin_cmpl cmpl = {}; 412 uint32_t nsid = spdk_nvme_ns_get_id(dev->ns); 413 414 if (spdk_nvme_ocssd_ctrlr_cmd_geometry(dev->ctrlr, nsid, &dev->geo, sizeof(dev->geo), 415 ftl_admin_cb, (void *)&cmpl)) { 416 SPDK_ERRLOG("Unable to retrieve geometry\n"); 417 return -1; 418 } 419 420 /* TODO: add a timeout */ 421 while (!cmpl.complete) { 422 spdk_nvme_ctrlr_process_admin_completions(dev->ctrlr); 423 } 424 425 if (spdk_nvme_cpl_is_error(&cmpl.status)) { 426 SPDK_ERRLOG("Unexpected status code: [%d], status code type: [%d]\n", 427 cmpl.status.status.sc, cmpl.status.status.sct); 428 return -1; 429 } 430 431 /* TODO: add sanity checks for the geo */ 432 dev->ppa_len = dev->geo.lbaf.grp_len + 433 dev->geo.lbaf.pu_len + 434 dev->geo.lbaf.chk_len + 435 dev->geo.lbaf.lbk_len; 436 437 dev->ppaf.lbk_offset = 0; 438 dev->ppaf.lbk_mask = (1 << dev->geo.lbaf.lbk_len) - 1; 439 dev->ppaf.chk_offset = dev->ppaf.lbk_offset + dev->geo.lbaf.lbk_len; 440 dev->ppaf.chk_mask = (1 << dev->geo.lbaf.chk_len) - 1; 441 dev->ppaf.pu_offset = dev->ppaf.chk_offset + dev->geo.lbaf.chk_len; 442 dev->ppaf.pu_mask = (1 << dev->geo.lbaf.pu_len) - 1; 443 dev->ppaf.grp_offset = dev->ppaf.pu_offset + dev->geo.lbaf.pu_len; 444 dev->ppaf.grp_mask = (1 << dev->geo.lbaf.grp_len) - 1; 445 446 /* We're using optimal write size as our xfer size */ 447 dev->xfer_size = dev->geo.ws_opt; 448 449 return 0; 450 } 451 452 static int 453 ftl_dev_nvme_init(struct spdk_ftl_dev *dev, const struct spdk_ftl_dev_init_opts *opts) 454 { 455 uint32_t block_size; 456 457 dev->ctrlr = opts->ctrlr; 458 459 if (spdk_nvme_ctrlr_get_num_ns(dev->ctrlr) != 1) { 460 SPDK_ERRLOG("Unsupported number of namespaces\n"); 461 return -1; 462 } 463 464 dev->ns = spdk_nvme_ctrlr_get_ns(dev->ctrlr, FTL_NSID); 465 dev->trid = opts->trid; 466 dev->md_size = spdk_nvme_ns_get_md_size(dev->ns); 467 468 block_size = spdk_nvme_ns_get_extended_sector_size(dev->ns); 469 if (block_size != FTL_BLOCK_SIZE) { 470 SPDK_ERRLOG("Unsupported block size (%"PRIu32")\n", block_size); 471 return -1; 472 } 473 474 if (dev->md_size % sizeof(uint32_t) != 0) { 475 /* Metadata pointer must be dword aligned */ 476 SPDK_ERRLOG("Unsupported metadata size (%zu)\n", dev->md_size); 477 return -1; 478 } 479 480 return 0; 481 } 482 483 static int 484 ftl_dev_init_nv_cache(struct spdk_ftl_dev *dev, struct spdk_bdev_desc *bdev_desc) 485 { 486 struct spdk_bdev *bdev; 487 struct spdk_ftl_conf *conf = &dev->conf; 488 struct ftl_nv_cache *nv_cache = &dev->nv_cache; 489 char pool_name[128]; 490 int rc; 491 492 if (!bdev_desc) { 493 return 0; 494 } 495 496 bdev = spdk_bdev_desc_get_bdev(bdev_desc); 497 SPDK_INFOLOG(SPDK_LOG_FTL_INIT, "Using %s as write buffer cache\n", 498 spdk_bdev_get_name(bdev)); 499 500 if (spdk_bdev_get_block_size(bdev) != FTL_BLOCK_SIZE) { 501 SPDK_ERRLOG("Unsupported block size (%d)\n", spdk_bdev_get_block_size(bdev)); 502 return -1; 503 } 504 505 if (!spdk_bdev_is_md_separate(bdev)) { 506 SPDK_ERRLOG("Bdev %s doesn't support separate metadata buffer IO\n", 507 spdk_bdev_get_name(bdev)); 508 return -1; 509 } 510 511 if (spdk_bdev_get_md_size(bdev) < sizeof(uint64_t)) { 512 SPDK_ERRLOG("Bdev's %s metadata is too small (%"PRIu32")\n", 513 spdk_bdev_get_name(bdev), spdk_bdev_get_md_size(bdev)); 514 return -1; 515 } 516 517 /* The cache needs to be capable of storing at least two full bands. This requirement comes 518 * from the fact that cache works as a protection against power loss, so before the data 519 * inside the cache can be overwritten, the band it's stored on has to be closed. 520 */ 521 if (spdk_bdev_get_num_blocks(bdev) < ftl_num_band_lbks(dev) * 2) { 522 SPDK_ERRLOG("Insufficient number of blocks for write buffer cache(%"PRIu64"\n", 523 spdk_bdev_get_num_blocks(bdev)); 524 return -1; 525 } 526 527 rc = snprintf(pool_name, sizeof(pool_name), "ftl-nvpool-%p", dev); 528 if (rc < 0 || rc >= 128) { 529 return -1; 530 } 531 532 nv_cache->md_pool = spdk_mempool_create(pool_name, conf->nv_cache.max_request_cnt, 533 spdk_bdev_get_md_size(bdev) * 534 conf->nv_cache.max_request_size, 535 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 536 SPDK_ENV_SOCKET_ID_ANY); 537 if (!nv_cache->md_pool) { 538 SPDK_ERRLOG("Failed to initialize non-volatile cache metadata pool\n"); 539 return -1; 540 } 541 542 if (pthread_spin_init(&nv_cache->lock, PTHREAD_PROCESS_PRIVATE)) { 543 SPDK_ERRLOG("Failed to initialize cache lock\n"); 544 return -1; 545 } 546 547 nv_cache->bdev_desc = bdev_desc; 548 nv_cache->current_addr = 0; 549 nv_cache->num_available = spdk_bdev_get_num_blocks(bdev); 550 551 return 0; 552 } 553 554 void 555 spdk_ftl_conf_init_defaults(struct spdk_ftl_conf *conf) 556 { 557 *conf = g_default_conf; 558 } 559 560 static void 561 ftl_lba_map_request_ctor(struct spdk_mempool *mp, void *opaque, void *obj, unsigned obj_idx) 562 { 563 struct ftl_lba_map_request *request = obj; 564 struct spdk_ftl_dev *dev = opaque; 565 566 request->segments = spdk_bit_array_create(spdk_divide_round_up( 567 ftl_num_band_lbks(dev), FTL_NUM_LBA_IN_BLOCK)); 568 } 569 570 static int 571 ftl_init_lba_map_pools(struct spdk_ftl_dev *dev) 572 { 573 #define POOL_NAME_LEN 128 574 char pool_name[POOL_NAME_LEN]; 575 int rc; 576 577 rc = snprintf(pool_name, sizeof(pool_name), "%s-%s", dev->name, "ocssd-lba-pool"); 578 if (rc < 0 || rc >= POOL_NAME_LEN) { 579 return -ENAMETOOLONG; 580 } 581 582 /* We need to reserve at least 2 buffers for band close / open sequence 583 * alone, plus additional (8) buffers for handling write errors. 584 * TODO: This memory pool is utilized only by core thread - it introduce 585 * unnecessary overhead and should be replaced by different data structure. 586 */ 587 dev->lba_pool = spdk_mempool_create(pool_name, 2 + 8, 588 ftl_lba_map_pool_elem_size(dev), 589 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 590 SPDK_ENV_SOCKET_ID_ANY); 591 if (!dev->lba_pool) { 592 return -ENOMEM; 593 } 594 595 rc = snprintf(pool_name, sizeof(pool_name), "%s-%s", dev->name, "ocssd-lbareq-pool"); 596 if (rc < 0 || rc >= POOL_NAME_LEN) { 597 return -ENAMETOOLONG; 598 } 599 600 dev->lba_request_pool = spdk_mempool_create_ctor(pool_name, 601 dev->conf.max_reloc_qdepth * dev->conf.max_active_relocs, 602 sizeof(struct ftl_lba_map_request), 603 SPDK_MEMPOOL_DEFAULT_CACHE_SIZE, 604 SPDK_ENV_SOCKET_ID_ANY, 605 ftl_lba_map_request_ctor, 606 dev); 607 if (!dev->lba_request_pool) { 608 return -ENOMEM; 609 } 610 611 return 0; 612 } 613 614 static void 615 ftl_init_wptr_list(struct spdk_ftl_dev *dev) 616 { 617 LIST_INIT(&dev->wptr_list); 618 LIST_INIT(&dev->flush_list); 619 } 620 621 static size_t 622 ftl_dev_band_max_seq(struct spdk_ftl_dev *dev) 623 { 624 struct ftl_band *band; 625 size_t seq = 0; 626 627 LIST_FOREACH(band, &dev->shut_bands, list_entry) { 628 if (band->seq > seq) { 629 seq = band->seq; 630 } 631 } 632 633 return seq; 634 } 635 636 static void 637 _ftl_init_bands_state(void *ctx) 638 { 639 struct ftl_band *band, *temp_band; 640 struct spdk_ftl_dev *dev = ctx; 641 642 dev->seq = ftl_dev_band_max_seq(dev); 643 644 LIST_FOREACH_SAFE(band, &dev->shut_bands, list_entry, temp_band) { 645 if (!band->lba_map.num_vld) { 646 ftl_band_set_state(band, FTL_BAND_STATE_FREE); 647 } 648 } 649 650 ftl_reloc_resume(dev->reloc); 651 /* Clear the limit applications as they're incremented incorrectly by */ 652 /* the initialization code */ 653 memset(dev->stats.limits, 0, sizeof(dev->stats.limits)); 654 } 655 656 static int 657 ftl_init_num_free_bands(struct spdk_ftl_dev *dev) 658 { 659 struct ftl_band *band; 660 int cnt = 0; 661 662 LIST_FOREACH(band, &dev->shut_bands, list_entry) { 663 if (band->num_chunks && !band->lba_map.num_vld) { 664 cnt++; 665 } 666 } 667 return cnt; 668 } 669 670 static int 671 ftl_init_bands_state(struct spdk_ftl_dev *dev) 672 { 673 /* TODO: Should we abort initialization or expose read only device */ 674 /* if there is no free bands? */ 675 /* If we abort initialization should we depend on condition that */ 676 /* we have no free bands or should we have some minimal number of */ 677 /* free bands? */ 678 if (!ftl_init_num_free_bands(dev)) { 679 return -1; 680 } 681 682 spdk_thread_send_msg(ftl_get_core_thread(dev), _ftl_init_bands_state, dev); 683 return 0; 684 } 685 686 static void 687 _ftl_dev_init_thread(void *ctx) 688 { 689 struct ftl_thread *thread = ctx; 690 struct spdk_ftl_dev *dev = thread->dev; 691 692 thread->poller = spdk_poller_register(thread->poller_fn, thread, thread->period_us); 693 if (!thread->poller) { 694 SPDK_ERRLOG("Unable to register poller\n"); 695 assert(0); 696 } 697 698 if (spdk_get_thread() == ftl_get_core_thread(dev)) { 699 ftl_anm_register_device(dev, ftl_process_anm_event); 700 } 701 } 702 703 static int 704 ftl_dev_init_thread(struct spdk_ftl_dev *dev, struct ftl_thread *thread, 705 struct spdk_thread *spdk_thread, spdk_poller_fn fn, uint64_t period_us) 706 { 707 thread->dev = dev; 708 thread->poller_fn = fn; 709 thread->thread = spdk_thread; 710 thread->period_us = period_us; 711 712 thread->qpair = spdk_nvme_ctrlr_alloc_io_qpair(dev->ctrlr, NULL, 0); 713 if (!thread->qpair) { 714 SPDK_ERRLOG("Unable to initialize qpair\n"); 715 return -1; 716 } 717 718 spdk_thread_send_msg(spdk_thread, _ftl_dev_init_thread, thread); 719 return 0; 720 } 721 722 static int 723 ftl_dev_init_threads(struct spdk_ftl_dev *dev, const struct spdk_ftl_dev_init_opts *opts) 724 { 725 if (!opts->core_thread || !opts->read_thread) { 726 return -1; 727 } 728 729 if (ftl_dev_init_thread(dev, &dev->core_thread, opts->core_thread, ftl_task_core, 0)) { 730 SPDK_ERRLOG("Unable to initialize core thread\n"); 731 return -1; 732 } 733 734 if (ftl_dev_init_thread(dev, &dev->read_thread, opts->read_thread, ftl_task_read, 0)) { 735 SPDK_ERRLOG("Unable to initialize read thread\n"); 736 return -1; 737 } 738 739 return 0; 740 } 741 742 static void 743 ftl_dev_free_thread(struct spdk_ftl_dev *dev, struct ftl_thread *thread) 744 { 745 assert(thread->poller == NULL); 746 747 spdk_nvme_ctrlr_free_io_qpair(thread->qpair); 748 thread->thread = NULL; 749 thread->qpair = NULL; 750 } 751 752 static int 753 ftl_dev_l2p_alloc(struct spdk_ftl_dev *dev) 754 { 755 size_t addr_size; 756 uint64_t i; 757 758 if (dev->num_lbas == 0) { 759 SPDK_DEBUGLOG(SPDK_LOG_FTL_INIT, "Invalid l2p table size\n"); 760 return -1; 761 } 762 763 if (dev->l2p) { 764 SPDK_DEBUGLOG(SPDK_LOG_FTL_INIT, "L2p table already allocated\n"); 765 return -1; 766 } 767 768 addr_size = dev->ppa_len >= 32 ? 8 : 4; 769 dev->l2p = malloc(dev->num_lbas * addr_size); 770 if (!dev->l2p) { 771 SPDK_DEBUGLOG(SPDK_LOG_FTL_INIT, "Failed to allocate l2p table\n"); 772 return -1; 773 } 774 775 for (i = 0; i < dev->num_lbas; ++i) { 776 ftl_l2p_set(dev, i, ftl_to_ppa(FTL_PPA_INVALID)); 777 } 778 779 return 0; 780 } 781 782 static void 783 ftl_init_complete(struct spdk_ftl_dev *dev) 784 { 785 pthread_mutex_lock(&g_ftl_queue_lock); 786 STAILQ_INSERT_HEAD(&g_ftl_queue, dev, stailq); 787 pthread_mutex_unlock(&g_ftl_queue_lock); 788 789 dev->initialized = 1; 790 791 if (dev->init_cb) { 792 dev->init_cb(dev, dev->init_arg, 0); 793 } 794 795 dev->init_cb = NULL; 796 dev->init_arg = NULL; 797 } 798 799 static int 800 ftl_setup_initial_state(struct spdk_ftl_dev *dev) 801 { 802 struct spdk_ftl_conf *conf = &dev->conf; 803 size_t i; 804 805 spdk_uuid_generate(&dev->uuid); 806 807 dev->num_lbas = 0; 808 for (i = 0; i < ftl_dev_num_bands(dev); ++i) { 809 dev->num_lbas += ftl_band_num_usable_lbks(&dev->bands[i]); 810 } 811 812 dev->num_lbas = (dev->num_lbas * (100 - conf->lba_rsvd)) / 100; 813 814 if (ftl_dev_l2p_alloc(dev)) { 815 SPDK_ERRLOG("Unable to init l2p table\n"); 816 return -1; 817 } 818 819 if (ftl_init_bands_state(dev)) { 820 SPDK_ERRLOG("Unable to finish the initialization\n"); 821 return -1; 822 } 823 824 ftl_init_complete(dev); 825 return 0; 826 } 827 828 struct ftl_init_fail_ctx { 829 spdk_ftl_init_fn cb; 830 void *arg; 831 }; 832 833 static void 834 ftl_init_fail_cb(void *ctx, int status) 835 { 836 struct ftl_init_fail_ctx *fail_cb = ctx; 837 838 fail_cb->cb(NULL, fail_cb->arg, -ENODEV); 839 free(fail_cb); 840 } 841 842 static void 843 ftl_init_fail(struct spdk_ftl_dev *dev) 844 { 845 struct ftl_init_fail_ctx *fail_cb; 846 847 fail_cb = malloc(sizeof(*fail_cb)); 848 if (!fail_cb) { 849 SPDK_ERRLOG("Unable to allocate context to free the device\n"); 850 return; 851 } 852 853 fail_cb->cb = dev->init_cb; 854 fail_cb->arg = dev->init_arg; 855 dev->halt_cb = NULL; 856 857 if (spdk_ftl_dev_free(dev, ftl_init_fail_cb, fail_cb)) { 858 SPDK_ERRLOG("Unable to free the device\n"); 859 assert(0); 860 } 861 } 862 863 static void 864 ftl_restore_device_cb(struct spdk_ftl_dev *dev, struct ftl_restore *restore, int status) 865 { 866 if (status) { 867 SPDK_ERRLOG("Failed to restore the device from the SSD\n"); 868 goto error; 869 } 870 871 if (ftl_init_bands_state(dev)) { 872 SPDK_ERRLOG("Unable to finish the initialization\n"); 873 goto error; 874 } 875 876 ftl_init_complete(dev); 877 return; 878 error: 879 ftl_init_fail(dev); 880 } 881 882 static void 883 ftl_restore_md_cb(struct spdk_ftl_dev *dev, struct ftl_restore *restore, int status) 884 { 885 if (status) { 886 SPDK_ERRLOG("Failed to restore the metadata from the SSD\n"); 887 goto error; 888 } 889 890 /* After the metadata is read it should be possible to allocate the L2P */ 891 if (ftl_dev_l2p_alloc(dev)) { 892 SPDK_ERRLOG("Failed to allocate the L2P\n"); 893 goto error; 894 } 895 896 if (ftl_restore_device(restore, ftl_restore_device_cb)) { 897 SPDK_ERRLOG("Failed to start device restoration from the SSD\n"); 898 goto error; 899 } 900 901 return; 902 error: 903 ftl_init_fail(dev); 904 } 905 906 static int 907 ftl_restore_state(struct spdk_ftl_dev *dev, const struct spdk_ftl_dev_init_opts *opts) 908 { 909 dev->uuid = opts->uuid; 910 911 if (ftl_restore_md(dev, ftl_restore_md_cb)) { 912 SPDK_ERRLOG("Failed to start metadata restoration from the SSD\n"); 913 return -1; 914 } 915 916 return 0; 917 } 918 919 static int 920 ftl_io_channel_create_cb(void *io_device, void *ctx) 921 { 922 struct spdk_ftl_dev *dev = io_device; 923 struct ftl_io_channel *ioch = ctx; 924 char mempool_name[32]; 925 926 snprintf(mempool_name, sizeof(mempool_name), "ftl_io_%p", ioch); 927 ioch->cache_ioch = NULL; 928 ioch->dev = dev; 929 ioch->elem_size = sizeof(struct ftl_md_io); 930 ioch->io_pool = spdk_mempool_create(mempool_name, 931 dev->conf.user_io_pool_size, 932 ioch->elem_size, 933 0, 934 SPDK_ENV_SOCKET_ID_ANY); 935 if (!ioch->io_pool) { 936 SPDK_ERRLOG("Failed to create IO channel's IO pool\n"); 937 return -1; 938 } 939 940 if (dev->nv_cache.bdev_desc) { 941 ioch->cache_ioch = spdk_bdev_get_io_channel(dev->nv_cache.bdev_desc); 942 if (!ioch->cache_ioch) { 943 SPDK_ERRLOG("Failed to create cache IO channel\n"); 944 spdk_mempool_free(ioch->io_pool); 945 return -1; 946 } 947 } 948 949 return 0; 950 } 951 952 static void 953 ftl_io_channel_destroy_cb(void *io_device, void *ctx) 954 { 955 struct ftl_io_channel *ioch = ctx; 956 957 spdk_mempool_free(ioch->io_pool); 958 959 if (ioch->cache_ioch) { 960 spdk_put_io_channel(ioch->cache_ioch); 961 } 962 } 963 964 static int 965 ftl_dev_init_io_channel(struct spdk_ftl_dev *dev) 966 { 967 spdk_io_device_register(dev, ftl_io_channel_create_cb, ftl_io_channel_destroy_cb, 968 sizeof(struct ftl_io_channel), 969 NULL); 970 971 dev->ioch = spdk_get_io_channel(dev); 972 if (!dev->ioch) { 973 spdk_io_device_unregister(dev, NULL); 974 return -1; 975 } 976 977 return 0; 978 } 979 980 int 981 spdk_ftl_dev_init(const struct spdk_ftl_dev_init_opts *_opts, spdk_ftl_init_fn cb, void *cb_arg) 982 { 983 struct spdk_ftl_dev *dev; 984 struct spdk_ftl_dev_init_opts opts = *_opts; 985 986 dev = calloc(1, sizeof(*dev)); 987 if (!dev) { 988 return -ENOMEM; 989 } 990 991 if (!opts.conf) { 992 opts.conf = &g_default_conf; 993 } 994 995 TAILQ_INIT(&dev->retry_queue); 996 dev->conf = *opts.conf; 997 dev->init_cb = cb; 998 dev->init_arg = cb_arg; 999 dev->range = opts.range; 1000 dev->limit = SPDK_FTL_LIMIT_MAX; 1001 1002 dev->name = strdup(opts.name); 1003 if (!dev->name) { 1004 SPDK_ERRLOG("Unable to set device name\n"); 1005 goto fail_sync; 1006 } 1007 1008 if (ftl_dev_nvme_init(dev, &opts)) { 1009 SPDK_ERRLOG("Unable to initialize NVMe structures\n"); 1010 goto fail_sync; 1011 } 1012 1013 /* In case of errors, we free all of the memory in ftl_dev_free_sync(), */ 1014 /* so we don't have to clean up in each of the init functions. */ 1015 if (ftl_dev_retrieve_geo(dev)) { 1016 SPDK_ERRLOG("Unable to retrieve geometry\n"); 1017 goto fail_sync; 1018 } 1019 1020 if (ftl_check_init_opts(&opts, &dev->geo)) { 1021 SPDK_ERRLOG("Invalid device configuration\n"); 1022 goto fail_sync; 1023 } 1024 1025 if (ftl_dev_init_punits(dev)) { 1026 SPDK_ERRLOG("Unable to initialize LUNs\n"); 1027 goto fail_sync; 1028 } 1029 1030 if (ftl_init_lba_map_pools(dev)) { 1031 SPDK_ERRLOG("Unable to init LBA map pools\n"); 1032 goto fail_sync; 1033 } 1034 1035 ftl_init_wptr_list(dev); 1036 1037 if (ftl_dev_init_bands(dev)) { 1038 SPDK_ERRLOG("Unable to initialize band array\n"); 1039 goto fail_sync; 1040 } 1041 1042 if (ftl_dev_init_nv_cache(dev, opts.cache_bdev_desc)) { 1043 SPDK_ERRLOG("Unable to initialize persistent cache\n"); 1044 goto fail_sync; 1045 } 1046 1047 dev->rwb = ftl_rwb_init(&dev->conf, dev->geo.ws_opt, dev->md_size, ftl_dev_num_punits(dev)); 1048 if (!dev->rwb) { 1049 SPDK_ERRLOG("Unable to initialize rwb structures\n"); 1050 goto fail_sync; 1051 } 1052 1053 dev->reloc = ftl_reloc_init(dev); 1054 if (!dev->reloc) { 1055 SPDK_ERRLOG("Unable to initialize reloc structures\n"); 1056 goto fail_sync; 1057 } 1058 1059 if (ftl_dev_init_io_channel(dev)) { 1060 SPDK_ERRLOG("Unable to initialize IO channels\n"); 1061 goto fail_sync; 1062 } 1063 1064 if (ftl_dev_init_threads(dev, &opts)) { 1065 SPDK_ERRLOG("Unable to initialize device threads\n"); 1066 goto fail_sync; 1067 } 1068 1069 if (opts.mode & SPDK_FTL_MODE_CREATE) { 1070 if (ftl_setup_initial_state(dev)) { 1071 SPDK_ERRLOG("Failed to setup initial state of the device\n"); 1072 goto fail_async; 1073 } 1074 } else { 1075 if (ftl_restore_state(dev, &opts)) { 1076 SPDK_ERRLOG("Unable to restore device's state from the SSD\n"); 1077 goto fail_async; 1078 } 1079 } 1080 1081 return 0; 1082 fail_sync: 1083 ftl_dev_free_sync(dev); 1084 return -ENOMEM; 1085 fail_async: 1086 ftl_init_fail(dev); 1087 return 0; 1088 } 1089 1090 static void 1091 _ftl_halt_defrag(void *arg) 1092 { 1093 ftl_reloc_halt(((struct spdk_ftl_dev *)arg)->reloc); 1094 } 1095 1096 static void 1097 ftl_lba_map_request_dtor(struct spdk_mempool *mp, void *opaque, void *obj, unsigned obj_idx) 1098 { 1099 struct ftl_lba_map_request *request = obj; 1100 1101 spdk_bit_array_free(&request->segments); 1102 } 1103 1104 static void 1105 ftl_dev_free_sync(struct spdk_ftl_dev *dev) 1106 { 1107 struct spdk_ftl_dev *iter; 1108 size_t i; 1109 1110 if (!dev) { 1111 return; 1112 } 1113 1114 pthread_mutex_lock(&g_ftl_queue_lock); 1115 STAILQ_FOREACH(iter, &g_ftl_queue, stailq) { 1116 if (iter == dev) { 1117 STAILQ_REMOVE(&g_ftl_queue, dev, spdk_ftl_dev, stailq); 1118 break; 1119 } 1120 } 1121 pthread_mutex_unlock(&g_ftl_queue_lock); 1122 1123 assert(LIST_EMPTY(&dev->wptr_list)); 1124 1125 ftl_dev_dump_bands(dev); 1126 ftl_dev_dump_stats(dev); 1127 1128 if (dev->ioch) { 1129 spdk_put_io_channel(dev->ioch); 1130 spdk_io_device_unregister(dev, NULL); 1131 } 1132 1133 if (dev->bands) { 1134 for (i = 0; i < ftl_dev_num_bands(dev); ++i) { 1135 free(dev->bands[i].chunk_buf); 1136 spdk_bit_array_free(&dev->bands[i].lba_map.vld); 1137 } 1138 } 1139 1140 spdk_mempool_free(dev->lba_pool); 1141 spdk_mempool_free(dev->nv_cache.md_pool); 1142 if (dev->lba_request_pool) { 1143 spdk_mempool_obj_iter(dev->lba_request_pool, ftl_lba_map_request_dtor, NULL); 1144 } 1145 spdk_mempool_free(dev->lba_request_pool); 1146 1147 ftl_rwb_free(dev->rwb); 1148 ftl_reloc_free(dev->reloc); 1149 1150 free(dev->name); 1151 free(dev->punits); 1152 free(dev->bands); 1153 free(dev->l2p); 1154 free(dev); 1155 } 1156 1157 static int 1158 ftl_halt_poller(void *ctx) 1159 { 1160 struct spdk_ftl_dev *dev = ctx; 1161 spdk_ftl_fn halt_cb = dev->halt_cb; 1162 void *halt_arg = dev->halt_arg; 1163 1164 if (!dev->core_thread.poller && !dev->read_thread.poller) { 1165 spdk_poller_unregister(&dev->halt_poller); 1166 1167 ftl_dev_free_thread(dev, &dev->read_thread); 1168 ftl_dev_free_thread(dev, &dev->core_thread); 1169 1170 ftl_anm_unregister_device(dev); 1171 ftl_dev_free_sync(dev); 1172 1173 if (halt_cb) { 1174 halt_cb(halt_arg, 0); 1175 } 1176 } 1177 1178 return 0; 1179 } 1180 1181 static void 1182 ftl_add_halt_poller(void *ctx) 1183 { 1184 struct spdk_ftl_dev *dev = ctx; 1185 1186 _ftl_halt_defrag(dev); 1187 1188 assert(!dev->halt_poller); 1189 dev->halt_poller = spdk_poller_register(ftl_halt_poller, dev, 100); 1190 } 1191 1192 int 1193 spdk_ftl_dev_free(struct spdk_ftl_dev *dev, spdk_ftl_fn cb, void *cb_arg) 1194 { 1195 if (dev->halt_cb) { 1196 return -EBUSY; 1197 } 1198 1199 dev->halt_cb = cb; 1200 dev->halt_arg = cb_arg; 1201 dev->halt = 1; 1202 1203 ftl_rwb_disable_interleaving(dev->rwb); 1204 1205 spdk_thread_send_msg(ftl_get_core_thread(dev), ftl_add_halt_poller, dev); 1206 return 0; 1207 } 1208 1209 int 1210 spdk_ftl_module_init(const struct ftl_module_init_opts *opts, spdk_ftl_fn cb, void *cb_arg) 1211 { 1212 return ftl_anm_init(opts->anm_thread, cb, cb_arg); 1213 } 1214 1215 int 1216 spdk_ftl_module_fini(spdk_ftl_fn cb, void *cb_arg) 1217 { 1218 return ftl_anm_free(cb, cb_arg); 1219 } 1220 1221 SPDK_LOG_REGISTER_COMPONENT("ftl_init", SPDK_LOG_FTL_INIT) 1222