1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/bdev_module.h" 7 #include "spdk/crc32.h" 8 9 #include "ftl_internal.h" 10 #include "ftl_band.h" 11 #include "ftl_core.h" 12 #include "ftl_layout.h" 13 #include "ftl_nv_cache_io.h" 14 #include "ftl_writer.h" 15 #include "mngt/ftl_mngt.h" 16 17 struct ftl_p2l_ckpt { 18 TAILQ_ENTRY(ftl_p2l_ckpt) link; 19 union ftl_md_vss *vss_md_page; 20 struct ftl_md *md; 21 struct ftl_layout_region *layout_region; 22 uint64_t num_pages; 23 24 #if defined(DEBUG) 25 uint64_t dbg_bmp_sz; 26 void *dbg_bmp; 27 struct ftl_bitmap *bmp; 28 #endif 29 }; 30 31 static struct ftl_p2l_ckpt * 32 ftl_p2l_ckpt_new(struct spdk_ftl_dev *dev, int region_type) 33 { 34 struct ftl_p2l_ckpt *ckpt; 35 36 ckpt = calloc(1, sizeof(struct ftl_p2l_ckpt)); 37 if (!ckpt) { 38 return NULL; 39 } 40 41 ckpt->vss_md_page = ftl_md_vss_buf_alloc(&dev->layout.region[region_type], 42 dev->layout.region[region_type].num_entries); 43 ckpt->layout_region = &dev->layout.region[region_type]; 44 ckpt->md = dev->layout.md[region_type]; 45 ckpt->num_pages = spdk_divide_round_up(ftl_get_num_blocks_in_band(dev), FTL_NUM_LBA_IN_BLOCK); 46 47 if (!ckpt->vss_md_page) { 48 free(ckpt); 49 return NULL; 50 } 51 52 #if defined(DEBUG) 53 /* The bitmap size must be a multiple of word size (8b) - round up */ 54 ckpt->dbg_bmp_sz = spdk_divide_round_up(ckpt->num_pages, 8); 55 56 ckpt->dbg_bmp = calloc(1, ckpt->dbg_bmp_sz); 57 assert(ckpt->dbg_bmp); 58 ckpt->bmp = ftl_bitmap_create(ckpt->dbg_bmp, ckpt->dbg_bmp_sz); 59 assert(ckpt->bmp); 60 #endif 61 62 return ckpt; 63 } 64 65 static void 66 ftl_p2l_ckpt_destroy(struct ftl_p2l_ckpt *ckpt) 67 { 68 #if defined(DEBUG) 69 ftl_bitmap_destroy(ckpt->bmp); 70 free(ckpt->dbg_bmp); 71 #endif 72 spdk_dma_free(ckpt->vss_md_page); 73 free(ckpt); 74 } 75 76 int 77 ftl_p2l_ckpt_init(struct spdk_ftl_dev *dev) 78 { 79 int region_type; 80 struct ftl_p2l_ckpt *ckpt; 81 82 TAILQ_INIT(&dev->p2l_ckpt.free); 83 TAILQ_INIT(&dev->p2l_ckpt.inuse); 84 for (region_type = FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MIN; 85 region_type <= FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MAX; 86 region_type++) { 87 ckpt = ftl_p2l_ckpt_new(dev, region_type); 88 if (!ckpt) { 89 return -1; 90 } 91 TAILQ_INSERT_TAIL(&dev->p2l_ckpt.free, ckpt, link); 92 } 93 return 0; 94 } 95 96 void 97 ftl_p2l_ckpt_deinit(struct spdk_ftl_dev *dev) 98 { 99 struct ftl_p2l_ckpt *ckpt, *ckpt_next; 100 101 TAILQ_FOREACH_SAFE(ckpt, &dev->p2l_ckpt.free, link, ckpt_next) { 102 TAILQ_REMOVE(&dev->p2l_ckpt.free, ckpt, link); 103 ftl_p2l_ckpt_destroy(ckpt); 104 } 105 106 TAILQ_FOREACH_SAFE(ckpt, &dev->p2l_ckpt.inuse, link, ckpt_next) { 107 TAILQ_REMOVE(&dev->p2l_ckpt.inuse, ckpt, link); 108 ftl_p2l_ckpt_destroy(ckpt); 109 } 110 } 111 112 struct ftl_p2l_ckpt * 113 ftl_p2l_ckpt_acquire(struct spdk_ftl_dev *dev) 114 { 115 struct ftl_p2l_ckpt *ckpt; 116 117 ckpt = TAILQ_FIRST(&dev->p2l_ckpt.free); 118 assert(ckpt); 119 TAILQ_REMOVE(&dev->p2l_ckpt.free, ckpt, link); 120 TAILQ_INSERT_TAIL(&dev->p2l_ckpt.inuse, ckpt, link); 121 return ckpt; 122 } 123 124 void 125 ftl_p2l_ckpt_release(struct spdk_ftl_dev *dev, struct ftl_p2l_ckpt *ckpt) 126 { 127 assert(ckpt); 128 #if defined(DEBUG) 129 memset(ckpt->dbg_bmp, 0, ckpt->dbg_bmp_sz); 130 #endif 131 TAILQ_REMOVE(&dev->p2l_ckpt.inuse, ckpt, link); 132 TAILQ_INSERT_TAIL(&dev->p2l_ckpt.free, ckpt, link); 133 } 134 135 static void 136 ftl_p2l_ckpt_issue_end(int status, void *arg) 137 { 138 struct ftl_rq *rq = arg; 139 assert(rq); 140 141 if (status) { 142 #ifdef SPDK_FTL_RETRY_ON_ERROR 143 /* retry */ 144 ftl_md_persist_entry_retry(&rq->md_persist_entry_ctx); 145 return; 146 #else 147 ftl_abort(); 148 #endif 149 } 150 151 assert(rq->io.band->queue_depth > 0); 152 rq->io.band->queue_depth--; 153 154 rq->owner.cb(rq); 155 } 156 157 void 158 ftl_p2l_ckpt_issue(struct ftl_rq *rq) 159 { 160 struct ftl_rq_entry *iter = rq->entries; 161 ftl_addr addr = rq->io.addr; 162 struct ftl_p2l_ckpt *ckpt = NULL; 163 struct ftl_p2l_ckpt_page *map_page; 164 union ftl_md_vss *md_page; 165 struct ftl_band *band; 166 uint64_t band_offs, p2l_map_page_no, i; 167 168 assert(rq); 169 band = rq->io.band; 170 ckpt = band->p2l_map.p2l_ckpt; 171 assert(ckpt); 172 173 /* Derive the P2L map page no */ 174 band_offs = ftl_band_block_offset_from_addr(band, rq->io.addr); 175 p2l_map_page_no = band_offs / FTL_NUM_LBA_IN_BLOCK; 176 assert((band_offs + rq->num_blocks - 1) / FTL_NUM_LBA_IN_BLOCK == p2l_map_page_no); 177 assert(p2l_map_page_no < ckpt->num_pages); 178 179 /* Get the corresponding P2L map page - the underlying stored data is the same as in the end metadata of band P2L (ftl_p2l_map_entry), 180 * however we're interested in a whole page (4KiB) worth of content 181 */ 182 map_page = ((struct ftl_p2l_ckpt_page *)band->p2l_map.band_map) + p2l_map_page_no; 183 assert(map_page); 184 185 /* Set up the md */ 186 md_page = &ckpt->vss_md_page[p2l_map_page_no]; 187 md_page->p2l_ckpt.seq_id = band->md->seq; 188 assert(rq->num_blocks == FTL_NUM_LBA_IN_BLOCK); 189 190 /* Update the band P2L map */ 191 for (i = 0; i < rq->num_blocks; i++, iter++) { 192 if (iter->lba != FTL_LBA_INVALID) { 193 /* This is compaction or reloc */ 194 assert(!ftl_addr_in_nvc(rq->dev, addr)); 195 ftl_band_set_p2l(band, iter->lba, addr, iter->seq_id); 196 } 197 addr = ftl_band_next_addr(band, addr, 1); 198 } 199 200 #if defined(DEBUG) 201 ftl_bitmap_set(ckpt->bmp, p2l_map_page_no); 202 #endif 203 204 md_page->p2l_ckpt.p2l_checksum = spdk_crc32c_update(map_page, 205 rq->num_blocks * sizeof(struct ftl_p2l_map_entry), 0); 206 /* Save the P2L map entry */ 207 ftl_md_persist_entry(ckpt->md, p2l_map_page_no, map_page, md_page, ftl_p2l_ckpt_issue_end, 208 rq, &rq->md_persist_entry_ctx); 209 } 210 211 #if defined(DEBUG) 212 static void 213 ftl_p2l_validate_pages(struct ftl_band *band, struct ftl_p2l_ckpt *ckpt, 214 uint64_t page_begin, uint64_t page_end, bool val) 215 { 216 uint64_t page_no; 217 218 for (page_no = page_begin; page_no < page_end; page_no++) { 219 assert(ftl_bitmap_get(ckpt->bmp, page_no) == val); 220 } 221 } 222 223 void 224 ftl_p2l_validate_ckpt(struct ftl_band *band) 225 { 226 struct ftl_p2l_ckpt *ckpt = band->p2l_map.p2l_ckpt; 227 uint64_t num_blks_tail_md = ftl_tail_md_num_blocks(band->dev); 228 uint64_t num_pages_tail_md = num_blks_tail_md / FTL_NUM_LBA_IN_BLOCK; 229 230 if (!ckpt) { 231 return; 232 } 233 234 assert(num_blks_tail_md % FTL_NUM_LBA_IN_BLOCK == 0); 235 236 /* all data pages written */ 237 ftl_p2l_validate_pages(band, ckpt, 238 0, ckpt->num_pages - num_pages_tail_md, true); 239 240 /* tail md pages not written */ 241 ftl_p2l_validate_pages(band, ckpt, ckpt->num_pages - num_pages_tail_md, 242 ckpt->num_pages, false); 243 } 244 #endif 245 246 static struct ftl_band * 247 ftl_get_band_from_region(struct spdk_ftl_dev *dev, enum ftl_layout_region_type type) 248 { 249 struct ftl_band *band = NULL; 250 uint64_t i; 251 252 assert(type >= FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MIN); 253 assert(type <= FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MAX); 254 255 for (i = 0; i < ftl_get_num_bands(dev); i++) { 256 band = &dev->bands[i]; 257 if ((band->md->state == FTL_BAND_STATE_OPEN || 258 band->md->state == FTL_BAND_STATE_FULL) && 259 band->md->p2l_md_region == type) { 260 return band; 261 } 262 } 263 264 return NULL; 265 } 266 267 static void ftl_mngt_persist_band_p2l(struct ftl_mngt_process *mngt, struct ftl_p2l_sync_ctx *ctx); 268 269 static void 270 ftl_p2l_ckpt_persist_end(int status, void *arg) 271 { 272 struct ftl_mngt_process *mngt = arg; 273 struct ftl_p2l_sync_ctx *ctx; 274 275 assert(mngt); 276 277 if (status) { 278 ftl_mngt_fail_step(mngt); 279 return; 280 } 281 282 ctx = ftl_mngt_get_step_ctx(mngt); 283 ctx->page_start++; 284 285 if (ctx->page_start == ctx->page_end) { 286 ctx->md_region++; 287 ftl_mngt_continue_step(mngt); 288 } else { 289 ftl_mngt_persist_band_p2l(mngt, ctx); 290 } 291 } 292 293 static void 294 ftl_mngt_persist_band_p2l(struct ftl_mngt_process *mngt, struct ftl_p2l_sync_ctx *ctx) 295 { 296 struct ftl_band *band = ctx->band; 297 union ftl_md_vss *md_page; 298 struct ftl_p2l_ckpt_page *map_page; 299 struct ftl_p2l_ckpt *ckpt; 300 301 ckpt = band->p2l_map.p2l_ckpt; 302 303 map_page = ((struct ftl_p2l_ckpt_page *)band->p2l_map.band_map) + ctx->page_start; 304 305 md_page = &ckpt->vss_md_page[ctx->page_start]; 306 md_page->p2l_ckpt.seq_id = band->md->seq; 307 md_page->p2l_ckpt.p2l_checksum = spdk_crc32c_update(map_page, 308 FTL_NUM_LBA_IN_BLOCK * sizeof(struct ftl_p2l_map_entry), 0); 309 310 /* Save the P2L map entry */ 311 ftl_md_persist_entry(ckpt->md, ctx->page_start, map_page, md_page, 312 ftl_p2l_ckpt_persist_end, mngt, &band->md_persist_entry_ctx); 313 } 314 315 void 316 ftl_mngt_persist_bands_p2l(struct ftl_mngt_process *mngt) 317 { 318 struct ftl_p2l_sync_ctx *ctx = ftl_mngt_get_step_ctx(mngt); 319 struct ftl_band *band; 320 uint64_t band_offs, p2l_map_page_no; 321 322 if (ctx->md_region > FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MAX) { 323 ftl_mngt_next_step(mngt); 324 return; 325 } 326 327 band = ftl_get_band_from_region(ftl_mngt_get_dev(mngt), ctx->md_region); 328 329 /* No band has the md region assigned (shutdown happened before next_band was assigned) */ 330 if (!band) { 331 ctx->page_start = 0; 332 ctx->page_end = 0; 333 ctx->md_region++; 334 ftl_mngt_continue_step(mngt); 335 return; 336 } 337 338 band_offs = ftl_band_block_offset_from_addr(band, band->md->iter.addr); 339 p2l_map_page_no = band_offs / FTL_NUM_LBA_IN_BLOCK; 340 341 ctx->page_start = 0; 342 ctx->page_end = p2l_map_page_no; 343 ctx->band = band; 344 345 /* Band wasn't written to - no need to sync its P2L */ 346 if (ctx->page_end == 0) { 347 ctx->md_region++; 348 ftl_mngt_continue_step(mngt); 349 return; 350 } 351 352 ftl_mngt_persist_band_p2l(mngt, ctx); 353 } 354 355 int 356 ftl_mngt_p2l_ckpt_get_seq_id(struct spdk_ftl_dev *dev, int md_region) 357 { 358 struct ftl_layout *layout = &dev->layout; 359 struct ftl_md *md = layout->md[md_region]; 360 union ftl_md_vss *page_md_buf = ftl_md_get_vss_buffer(md); 361 uint64_t page_no, seq_id = 0; 362 363 for (page_no = 0; page_no < layout->p2l.ckpt_pages; page_no++, page_md_buf++) { 364 if (seq_id < page_md_buf->p2l_ckpt.seq_id) { 365 seq_id = page_md_buf->p2l_ckpt.seq_id; 366 } 367 } 368 return seq_id; 369 } 370 371 int 372 ftl_mngt_p2l_ckpt_restore(struct ftl_band *band, uint32_t md_region, uint64_t seq_id) 373 { 374 struct ftl_layout *layout = &band->dev->layout; 375 struct ftl_md *md = layout->md[md_region]; 376 union ftl_md_vss *page_md_buf = ftl_md_get_vss_buffer(md); 377 struct ftl_p2l_ckpt_page *page = ftl_md_get_buffer(md); 378 struct ftl_p2l_ckpt_page *map_page; 379 uint64_t page_no, page_max = 0; 380 bool page_found = false; 381 382 assert(band->md->p2l_md_region == md_region); 383 if (band->md->p2l_md_region != md_region) { 384 return -EINVAL; 385 } 386 387 assert(band->md->seq == seq_id); 388 if (band->md->seq != seq_id) { 389 return -EINVAL; 390 } 391 392 for (page_no = 0; page_no < layout->p2l.ckpt_pages; page_no++, page++, page_md_buf++) { 393 if (page_md_buf->p2l_ckpt.seq_id != seq_id) { 394 continue; 395 } 396 397 page_max = page_no; 398 page_found = true; 399 400 /* Get the corresponding P2L map page - the underlying stored data is the same as in the end metadata of band P2L (ftl_p2l_map_entry), 401 * however we're interested in a whole page (4KiB) worth of content 402 */ 403 map_page = ((struct ftl_p2l_ckpt_page *)band->p2l_map.band_map) + page_no; 404 405 if (page_md_buf->p2l_ckpt.p2l_checksum && 406 page_md_buf->p2l_ckpt.p2l_checksum != spdk_crc32c_update(page, 407 FTL_NUM_LBA_IN_BLOCK * sizeof(struct ftl_p2l_map_entry), 0)) { 408 ftl_stats_crc_error(band->dev, FTL_STATS_TYPE_MD_NV_CACHE); 409 return -EINVAL; 410 } 411 412 /* Restore the page from P2L checkpoint */ 413 *map_page = *page; 414 } 415 416 assert(page_found); 417 if (!page_found) { 418 return -EINVAL; 419 } 420 421 /* Restore check point in band P2L map */ 422 band->p2l_map.p2l_ckpt = ftl_p2l_ckpt_acquire_region_type( 423 band->dev, md_region); 424 425 #ifdef DEBUG 426 /* Set check point valid map for validation */ 427 struct ftl_p2l_ckpt *ckpt = band->p2l_map.p2l_ckpt ; 428 for (uint64_t i = 0; i <= page_max; i++) { 429 ftl_bitmap_set(ckpt->bmp, i); 430 } 431 #endif 432 433 ftl_band_iter_init(band); 434 ftl_band_iter_set(band, (page_max + 1) * FTL_NUM_LBA_IN_BLOCK); 435 436 return 0; 437 } 438 439 enum ftl_layout_region_type 440 ftl_p2l_ckpt_region_type(const struct ftl_p2l_ckpt *ckpt) { 441 return ckpt->layout_region->type; 442 } 443 444 struct ftl_p2l_ckpt * 445 ftl_p2l_ckpt_acquire_region_type(struct spdk_ftl_dev *dev, uint32_t region_type) 446 { 447 struct ftl_p2l_ckpt *ckpt = NULL; 448 449 TAILQ_FOREACH(ckpt, &dev->p2l_ckpt.free, link) { 450 if (ckpt->layout_region->type == region_type) { 451 break; 452 } 453 } 454 455 assert(ckpt); 456 457 TAILQ_REMOVE(&dev->p2l_ckpt.free, ckpt, link); 458 TAILQ_INSERT_TAIL(&dev->p2l_ckpt.inuse, ckpt, link); 459 460 return ckpt; 461 } 462 463 int 464 ftl_mngt_p2l_ckpt_restore_clean(struct ftl_band *band) 465 { 466 struct spdk_ftl_dev *dev = band->dev; 467 struct ftl_layout *layout = &dev->layout; 468 struct ftl_p2l_ckpt_page *page, *map_page; 469 enum ftl_layout_region_type md_region = band->md->p2l_md_region; 470 uint64_t page_no; 471 uint64_t num_written_pages; 472 union ftl_md_vss *page_md_buf; 473 474 if (md_region < FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MIN || 475 md_region > FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MAX) { 476 return -EINVAL; 477 } 478 479 assert(band->md->iter.offset % FTL_NUM_LBA_IN_BLOCK == 0); 480 num_written_pages = band->md->iter.offset / FTL_NUM_LBA_IN_BLOCK; 481 482 /* Associate band with md region before shutdown */ 483 if (!band->p2l_map.p2l_ckpt) { 484 band->p2l_map.p2l_ckpt = ftl_p2l_ckpt_acquire_region_type(dev, md_region); 485 } 486 487 /* Band was opened but no data was written */ 488 if (band->md->iter.offset == 0) { 489 return 0; 490 } 491 492 page_no = 0; 493 494 /* Restore P2L map up to last written page */ 495 page_md_buf = ftl_md_get_vss_buffer(layout->md[md_region]); 496 page = ftl_md_get_buffer(layout->md[md_region]); 497 498 for (; page_no < num_written_pages; page_no++, page++, page_md_buf++) { 499 if (page_md_buf->p2l_ckpt.seq_id != band->md->seq) { 500 assert(page_md_buf->p2l_ckpt.seq_id == band->md->seq); 501 } 502 503 /* Get the corresponding P2L map page */ 504 map_page = ((struct ftl_p2l_ckpt_page *)band->p2l_map.band_map) + page_no; 505 506 /* Restore the page from P2L checkpoint */ 507 *map_page = *page; 508 509 #if defined(DEBUG) 510 assert(ftl_bitmap_get(band->p2l_map.p2l_ckpt->bmp, page_no) == false); 511 ftl_bitmap_set(band->p2l_map.p2l_ckpt->bmp, page_no); 512 #endif 513 } 514 515 assert(page_md_buf->p2l_ckpt.seq_id < band->md->seq); 516 517 return 0; 518 } 519 520 void 521 ftl_mngt_p2l_ckpt_restore_shm_clean(struct ftl_band *band) 522 { 523 struct spdk_ftl_dev *dev = band->dev; 524 enum ftl_layout_region_type md_region = band->md->p2l_md_region; 525 526 /* Associate band with md region before shutdown */ 527 if (!band->p2l_map.p2l_ckpt) { 528 band->p2l_map.p2l_ckpt = ftl_p2l_ckpt_acquire_region_type(dev, md_region); 529 } 530 531 #if defined(DEBUG) 532 uint64_t page_no; 533 uint64_t num_written_pages; 534 535 assert(band->md->iter.offset % FTL_NUM_LBA_IN_BLOCK == 0); 536 num_written_pages = band->md->iter.offset / FTL_NUM_LBA_IN_BLOCK; 537 538 /* Band was opened but no data was written */ 539 if (band->md->iter.offset == 0) { 540 return; 541 } 542 543 /* Set page number to first data page - skip head md */ 544 page_no = 0; 545 546 for (; page_no < num_written_pages; page_no++) { 547 assert(ftl_bitmap_get(band->p2l_map.p2l_ckpt->bmp, page_no) == false); 548 ftl_bitmap_set(band->p2l_map.p2l_ckpt->bmp, page_no); 549 } 550 #endif 551 } 552