1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/bdev_module.h" 7 #include "spdk/crc32.h" 8 9 #include "ftl_internal.h" 10 #include "ftl_band.h" 11 #include "ftl_core.h" 12 #include "ftl_layout.h" 13 #include "ftl_nv_cache_io.h" 14 #include "ftl_writer.h" 15 #include "mngt/ftl_mngt.h" 16 17 struct ftl_p2l_ckpt { 18 TAILQ_ENTRY(ftl_p2l_ckpt) link; 19 union ftl_md_vss *vss_md_page; 20 struct ftl_md *md; 21 struct ftl_layout_region *layout_region; 22 uint64_t num_pages; 23 24 #if defined(DEBUG) 25 uint64_t dbg_bmp_sz; 26 void *dbg_bmp; 27 struct ftl_bitmap *bmp; 28 #endif 29 }; 30 31 static struct ftl_p2l_ckpt * 32 ftl_p2l_ckpt_new(struct spdk_ftl_dev *dev, int region_type) 33 { 34 struct ftl_p2l_ckpt *ckpt; 35 36 ckpt = calloc(1, sizeof(struct ftl_p2l_ckpt)); 37 if (!ckpt) { 38 return NULL; 39 } 40 41 ckpt->vss_md_page = ftl_md_vss_buf_alloc(&dev->layout.region[region_type], 42 dev->layout.region[region_type].num_entries); 43 ckpt->layout_region = &dev->layout.region[region_type]; 44 ckpt->md = dev->layout.md[region_type]; 45 ckpt->num_pages = spdk_divide_round_up(ftl_get_num_blocks_in_band(dev), FTL_NUM_LBA_IN_BLOCK); 46 47 if (!ckpt->vss_md_page) { 48 free(ckpt); 49 return NULL; 50 } 51 52 #if defined(DEBUG) 53 /* The bitmap size must be a multiple of word size (8b) - round up */ 54 ckpt->dbg_bmp_sz = spdk_divide_round_up(ckpt->num_pages, 8); 55 56 ckpt->dbg_bmp = calloc(1, ckpt->dbg_bmp_sz); 57 assert(ckpt->dbg_bmp); 58 ckpt->bmp = ftl_bitmap_create(ckpt->dbg_bmp, ckpt->dbg_bmp_sz); 59 assert(ckpt->bmp); 60 #endif 61 62 return ckpt; 63 } 64 65 static void 66 ftl_p2l_ckpt_destroy(struct ftl_p2l_ckpt *ckpt) 67 { 68 #if defined(DEBUG) 69 ftl_bitmap_destroy(ckpt->bmp); 70 free(ckpt->dbg_bmp); 71 #endif 72 spdk_dma_free(ckpt->vss_md_page); 73 free(ckpt); 74 } 75 76 int 77 ftl_p2l_ckpt_init(struct spdk_ftl_dev *dev) 78 { 79 int region_type; 80 struct ftl_p2l_ckpt *ckpt; 81 82 TAILQ_INIT(&dev->p2l_ckpt.free); 83 TAILQ_INIT(&dev->p2l_ckpt.inuse); 84 for (region_type = FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MIN; 85 region_type <= FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MAX; 86 region_type++) { 87 ckpt = ftl_p2l_ckpt_new(dev, region_type); 88 if (!ckpt) { 89 return -1; 90 } 91 TAILQ_INSERT_TAIL(&dev->p2l_ckpt.free, ckpt, link); 92 } 93 return 0; 94 } 95 96 void 97 ftl_p2l_ckpt_deinit(struct spdk_ftl_dev *dev) 98 { 99 struct ftl_p2l_ckpt *ckpt, *ckpt_next; 100 101 TAILQ_FOREACH_SAFE(ckpt, &dev->p2l_ckpt.free, link, ckpt_next) { 102 TAILQ_REMOVE(&dev->p2l_ckpt.free, ckpt, link); 103 ftl_p2l_ckpt_destroy(ckpt); 104 } 105 106 TAILQ_FOREACH_SAFE(ckpt, &dev->p2l_ckpt.inuse, link, ckpt_next) { 107 TAILQ_REMOVE(&dev->p2l_ckpt.inuse, ckpt, link); 108 ftl_p2l_ckpt_destroy(ckpt); 109 } 110 } 111 112 struct ftl_p2l_ckpt * 113 ftl_p2l_ckpt_acquire(struct spdk_ftl_dev *dev) 114 { 115 struct ftl_p2l_ckpt *ckpt; 116 117 ckpt = TAILQ_FIRST(&dev->p2l_ckpt.free); 118 assert(ckpt); 119 TAILQ_REMOVE(&dev->p2l_ckpt.free, ckpt, link); 120 TAILQ_INSERT_TAIL(&dev->p2l_ckpt.inuse, ckpt, link); 121 return ckpt; 122 } 123 124 void 125 ftl_p2l_ckpt_release(struct spdk_ftl_dev *dev, struct ftl_p2l_ckpt *ckpt) 126 { 127 assert(ckpt); 128 #if defined(DEBUG) 129 memset(ckpt->dbg_bmp, 0, ckpt->dbg_bmp_sz); 130 #endif 131 TAILQ_REMOVE(&dev->p2l_ckpt.inuse, ckpt, link); 132 TAILQ_INSERT_TAIL(&dev->p2l_ckpt.free, ckpt, link); 133 } 134 135 static void 136 ftl_p2l_ckpt_issue_end(int status, void *arg) 137 { 138 struct ftl_rq *rq = arg; 139 assert(rq); 140 141 if (status) { 142 /* retry */ 143 ftl_md_persist_entry_retry(&rq->md_persist_entry_ctx); 144 return; 145 } 146 147 assert(rq->io.band->queue_depth > 0); 148 rq->io.band->queue_depth--; 149 150 rq->owner.cb(rq); 151 } 152 153 void 154 ftl_p2l_ckpt_issue(struct ftl_rq *rq) 155 { 156 struct ftl_rq_entry *iter = rq->entries; 157 ftl_addr addr = rq->io.addr; 158 struct ftl_p2l_ckpt *ckpt = NULL; 159 struct ftl_p2l_ckpt_page *map_page; 160 union ftl_md_vss *md_page; 161 struct ftl_band *band; 162 uint64_t band_offs, p2l_map_page_no, i; 163 164 assert(rq); 165 band = rq->io.band; 166 ckpt = band->p2l_map.p2l_ckpt; 167 assert(ckpt); 168 169 /* Derive the P2L map page no */ 170 band_offs = ftl_band_block_offset_from_addr(band, rq->io.addr); 171 p2l_map_page_no = band_offs / FTL_NUM_LBA_IN_BLOCK; 172 assert((band_offs + rq->num_blocks - 1) / FTL_NUM_LBA_IN_BLOCK == p2l_map_page_no); 173 assert(p2l_map_page_no < ckpt->num_pages); 174 175 /* Get the corresponding P2L map page - the underlying stored data is the same as in the end metadata of band P2L (ftl_p2l_map_entry), 176 * however we're interested in a whole page (4KiB) worth of content 177 */ 178 map_page = ((struct ftl_p2l_ckpt_page *)band->p2l_map.band_map) + p2l_map_page_no; 179 assert(map_page); 180 181 /* Set up the md */ 182 md_page = &ckpt->vss_md_page[p2l_map_page_no]; 183 md_page->p2l_ckpt.seq_id = band->md->seq; 184 assert(rq->num_blocks == FTL_NUM_LBA_IN_BLOCK); 185 186 /* Update the band P2L map */ 187 for (i = 0; i < rq->num_blocks; i++, iter++) { 188 if (iter->lba != FTL_LBA_INVALID) { 189 /* This is compaction or reloc */ 190 assert(!ftl_addr_in_nvc(rq->dev, addr)); 191 ftl_band_set_p2l(band, iter->lba, addr, iter->seq_id); 192 } 193 addr = ftl_band_next_addr(band, addr, 1); 194 } 195 196 #if defined(DEBUG) 197 ftl_bitmap_set(ckpt->bmp, p2l_map_page_no); 198 #endif 199 200 md_page->p2l_ckpt.p2l_checksum = spdk_crc32c_update(map_page, 201 rq->num_blocks * sizeof(struct ftl_p2l_map_entry), 0); 202 /* Save the P2L map entry */ 203 ftl_md_persist_entry(ckpt->md, p2l_map_page_no, map_page, md_page, ftl_p2l_ckpt_issue_end, 204 rq, &rq->md_persist_entry_ctx); 205 } 206 207 #if defined(DEBUG) 208 static void 209 ftl_p2l_validate_pages(struct ftl_band *band, struct ftl_p2l_ckpt *ckpt, 210 uint64_t page_begin, uint64_t page_end, bool val) 211 { 212 uint64_t page_no; 213 214 for (page_no = page_begin; page_no < page_end; page_no++) { 215 assert(ftl_bitmap_get(ckpt->bmp, page_no) == val); 216 } 217 } 218 219 void 220 ftl_p2l_validate_ckpt(struct ftl_band *band) 221 { 222 struct ftl_p2l_ckpt *ckpt = band->p2l_map.p2l_ckpt; 223 uint64_t num_blks_tail_md = ftl_tail_md_num_blocks(band->dev); 224 uint64_t num_pages_tail_md = num_blks_tail_md / FTL_NUM_LBA_IN_BLOCK; 225 226 if (!ckpt) { 227 return; 228 } 229 230 assert(num_blks_tail_md % FTL_NUM_LBA_IN_BLOCK == 0); 231 232 /* all data pages written */ 233 ftl_p2l_validate_pages(band, ckpt, 234 0, ckpt->num_pages - num_pages_tail_md, true); 235 236 /* tail md pages not written */ 237 ftl_p2l_validate_pages(band, ckpt, ckpt->num_pages - num_pages_tail_md, 238 ckpt->num_pages, false); 239 } 240 #endif 241 242 static struct ftl_band * 243 ftl_get_band_from_region(struct spdk_ftl_dev *dev, enum ftl_layout_region_type type) 244 { 245 struct ftl_band *band = NULL; 246 uint64_t i; 247 248 assert(type >= FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MIN); 249 assert(type <= FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MAX); 250 251 for (i = 0; i < ftl_get_num_bands(dev); i++) { 252 band = &dev->bands[i]; 253 if ((band->md->state == FTL_BAND_STATE_OPEN || 254 band->md->state == FTL_BAND_STATE_FULL) && 255 band->md->p2l_md_region == type) { 256 return band; 257 } 258 } 259 260 return NULL; 261 } 262 263 static void ftl_mngt_persist_band_p2l(struct ftl_mngt_process *mngt, struct ftl_p2l_sync_ctx *ctx); 264 265 static void 266 ftl_p2l_ckpt_persist_end(int status, void *arg) 267 { 268 struct ftl_mngt_process *mngt = arg; 269 struct ftl_p2l_sync_ctx *ctx; 270 271 assert(mngt); 272 273 if (status) { 274 ftl_mngt_fail_step(mngt); 275 return; 276 } 277 278 ctx = ftl_mngt_get_step_ctx(mngt); 279 ctx->page_start++; 280 281 if (ctx->page_start == ctx->page_end) { 282 ctx->md_region++; 283 ftl_mngt_continue_step(mngt); 284 } else { 285 ftl_mngt_persist_band_p2l(mngt, ctx); 286 } 287 } 288 289 static void 290 ftl_mngt_persist_band_p2l(struct ftl_mngt_process *mngt, struct ftl_p2l_sync_ctx *ctx) 291 { 292 struct ftl_band *band = ctx->band; 293 union ftl_md_vss *md_page; 294 struct ftl_p2l_ckpt_page *map_page; 295 struct ftl_p2l_ckpt *ckpt; 296 297 ckpt = band->p2l_map.p2l_ckpt; 298 299 map_page = ((struct ftl_p2l_ckpt_page *)band->p2l_map.band_map) + ctx->page_start; 300 301 md_page = &ckpt->vss_md_page[ctx->page_start]; 302 md_page->p2l_ckpt.seq_id = band->md->seq; 303 md_page->p2l_ckpt.p2l_checksum = spdk_crc32c_update(map_page, 304 FTL_NUM_LBA_IN_BLOCK * sizeof(struct ftl_p2l_map_entry), 0); 305 306 /* Save the P2L map entry */ 307 ftl_md_persist_entry(ckpt->md, ctx->page_start, map_page, md_page, 308 ftl_p2l_ckpt_persist_end, mngt, &band->md_persist_entry_ctx); 309 } 310 311 void 312 ftl_mngt_persist_bands_p2l(struct ftl_mngt_process *mngt) 313 { 314 struct ftl_p2l_sync_ctx *ctx = ftl_mngt_get_step_ctx(mngt); 315 struct ftl_band *band; 316 uint64_t band_offs, p2l_map_page_no; 317 318 if (ctx->md_region > FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MAX) { 319 ftl_mngt_next_step(mngt); 320 return; 321 } 322 323 band = ftl_get_band_from_region(ftl_mngt_get_dev(mngt), ctx->md_region); 324 325 /* No band has the md region assigned (shutdown happened before next_band was assigned) */ 326 if (!band) { 327 ctx->page_start = 0; 328 ctx->page_end = 0; 329 ctx->md_region++; 330 ftl_mngt_continue_step(mngt); 331 return; 332 } 333 334 band_offs = ftl_band_block_offset_from_addr(band, band->md->iter.addr); 335 p2l_map_page_no = band_offs / FTL_NUM_LBA_IN_BLOCK; 336 337 ctx->page_start = 0; 338 ctx->page_end = p2l_map_page_no; 339 ctx->band = band; 340 341 /* Band wasn't written to - no need to sync its P2L */ 342 if (ctx->page_end == 0) { 343 ctx->md_region++; 344 ftl_mngt_continue_step(mngt); 345 return; 346 } 347 348 ftl_mngt_persist_band_p2l(mngt, ctx); 349 } 350 351 int 352 ftl_mngt_p2l_ckpt_get_seq_id(struct spdk_ftl_dev *dev, int md_region) 353 { 354 struct ftl_layout *layout = &dev->layout; 355 struct ftl_md *md = layout->md[md_region]; 356 union ftl_md_vss *page_md_buf = ftl_md_get_vss_buffer(md); 357 uint64_t page_no, seq_id = 0; 358 359 for (page_no = 0; page_no < layout->p2l.ckpt_pages; page_no++, page_md_buf++) { 360 if (seq_id < page_md_buf->p2l_ckpt.seq_id) { 361 seq_id = page_md_buf->p2l_ckpt.seq_id; 362 } 363 } 364 return seq_id; 365 } 366 367 int 368 ftl_mngt_p2l_ckpt_restore(struct ftl_band *band, uint32_t md_region, uint64_t seq_id) 369 { 370 struct ftl_layout *layout = &band->dev->layout; 371 struct ftl_md *md = layout->md[md_region]; 372 union ftl_md_vss *page_md_buf = ftl_md_get_vss_buffer(md); 373 struct ftl_p2l_ckpt_page *page = ftl_md_get_buffer(md); 374 struct ftl_p2l_ckpt_page *map_page; 375 uint64_t page_no, page_max = 0; 376 bool page_found = false; 377 378 assert(band->md->p2l_md_region == md_region); 379 if (band->md->p2l_md_region != md_region) { 380 return -EINVAL; 381 } 382 383 assert(band->md->seq == seq_id); 384 if (band->md->seq != seq_id) { 385 return -EINVAL; 386 } 387 388 for (page_no = 0; page_no < layout->p2l.ckpt_pages; page_no++, page++, page_md_buf++) { 389 if (page_md_buf->p2l_ckpt.seq_id != seq_id) { 390 continue; 391 } 392 393 page_max = page_no; 394 page_found = true; 395 396 /* Get the corresponding P2L map page - the underlying stored data is the same as in the end metadata of band P2L (ftl_p2l_map_entry), 397 * however we're interested in a whole page (4KiB) worth of content 398 */ 399 map_page = ((struct ftl_p2l_ckpt_page *)band->p2l_map.band_map) + page_no; 400 401 if (page_md_buf->p2l_ckpt.p2l_checksum && 402 page_md_buf->p2l_ckpt.p2l_checksum != spdk_crc32c_update(page, 403 FTL_NUM_LBA_IN_BLOCK * sizeof(struct ftl_p2l_map_entry), 0)) { 404 return -EINVAL; 405 } 406 407 /* Restore the page from P2L checkpoint */ 408 *map_page = *page; 409 } 410 411 assert(page_found); 412 if (!page_found) { 413 return -EINVAL; 414 } 415 416 /* Restore check point in band P2L map */ 417 band->p2l_map.p2l_ckpt = ftl_p2l_ckpt_acquire_region_type( 418 band->dev, md_region); 419 420 #ifdef DEBUG 421 /* Set check point valid map for validation */ 422 struct ftl_p2l_ckpt *ckpt = band->p2l_map.p2l_ckpt ; 423 for (uint64_t i = 0; i <= page_max; i++) { 424 ftl_bitmap_set(ckpt->bmp, i); 425 } 426 #endif 427 428 ftl_band_iter_init(band); 429 ftl_band_iter_set(band, (page_max + 1) * FTL_NUM_LBA_IN_BLOCK); 430 431 return 0; 432 } 433 434 enum ftl_layout_region_type 435 ftl_p2l_ckpt_region_type(const struct ftl_p2l_ckpt *ckpt) { 436 return ckpt->layout_region->type; 437 } 438 439 struct ftl_p2l_ckpt * 440 ftl_p2l_ckpt_acquire_region_type(struct spdk_ftl_dev *dev, uint32_t region_type) 441 { 442 struct ftl_p2l_ckpt *ckpt = NULL; 443 444 TAILQ_FOREACH(ckpt, &dev->p2l_ckpt.free, link) { 445 if (ckpt->layout_region->type == region_type) { 446 break; 447 } 448 } 449 450 assert(ckpt); 451 452 TAILQ_REMOVE(&dev->p2l_ckpt.free, ckpt, link); 453 TAILQ_INSERT_TAIL(&dev->p2l_ckpt.inuse, ckpt, link); 454 455 return ckpt; 456 } 457 458 int 459 ftl_mngt_p2l_ckpt_restore_clean(struct ftl_band *band) 460 { 461 struct spdk_ftl_dev *dev = band->dev; 462 struct ftl_layout *layout = &dev->layout; 463 struct ftl_p2l_ckpt_page *page, *map_page; 464 enum ftl_layout_region_type md_region = band->md->p2l_md_region; 465 uint64_t page_no; 466 uint64_t num_written_pages; 467 union ftl_md_vss *page_md_buf; 468 469 if (md_region < FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MIN || 470 md_region > FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MAX) { 471 return -EINVAL; 472 } 473 474 assert(band->md->iter.offset % FTL_NUM_LBA_IN_BLOCK == 0); 475 num_written_pages = band->md->iter.offset / FTL_NUM_LBA_IN_BLOCK; 476 477 /* Associate band with md region before shutdown */ 478 if (!band->p2l_map.p2l_ckpt) { 479 band->p2l_map.p2l_ckpt = ftl_p2l_ckpt_acquire_region_type(dev, md_region); 480 } 481 482 /* Band was opened but no data was written */ 483 if (band->md->iter.offset == 0) { 484 return 0; 485 } 486 487 page_no = 0; 488 489 /* Restore P2L map up to last written page */ 490 page_md_buf = ftl_md_get_vss_buffer(layout->md[md_region]); 491 page = ftl_md_get_buffer(layout->md[md_region]); 492 493 for (; page_no < num_written_pages; page_no++, page++, page_md_buf++) { 494 if (page_md_buf->p2l_ckpt.seq_id != band->md->seq) { 495 assert(page_md_buf->p2l_ckpt.seq_id == band->md->seq); 496 } 497 498 /* Get the corresponding P2L map page */ 499 map_page = ((struct ftl_p2l_ckpt_page *)band->p2l_map.band_map) + page_no; 500 501 /* Restore the page from P2L checkpoint */ 502 *map_page = *page; 503 504 #if defined(DEBUG) 505 assert(ftl_bitmap_get(band->p2l_map.p2l_ckpt->bmp, page_no) == false); 506 ftl_bitmap_set(band->p2l_map.p2l_ckpt->bmp, page_no); 507 #endif 508 } 509 510 assert(page_md_buf->p2l_ckpt.seq_id < band->md->seq); 511 512 return 0; 513 } 514 515 void 516 ftl_mngt_p2l_ckpt_restore_shm_clean(struct ftl_band *band) 517 { 518 struct spdk_ftl_dev *dev = band->dev; 519 enum ftl_layout_region_type md_region = band->md->p2l_md_region; 520 521 /* Associate band with md region before shutdown */ 522 if (!band->p2l_map.p2l_ckpt) { 523 band->p2l_map.p2l_ckpt = ftl_p2l_ckpt_acquire_region_type(dev, md_region); 524 } 525 526 #if defined(DEBUG) 527 uint64_t page_no; 528 uint64_t num_written_pages; 529 530 assert(band->md->iter.offset % FTL_NUM_LBA_IN_BLOCK == 0); 531 num_written_pages = band->md->iter.offset / FTL_NUM_LBA_IN_BLOCK; 532 533 /* Band was opened but no data was written */ 534 if (band->md->iter.offset == 0) { 535 return; 536 } 537 538 /* Set page number to first data page - skip head md */ 539 page_no = 0; 540 541 for (; page_no < num_written_pages; page_no++) { 542 assert(ftl_bitmap_get(band->p2l_map.p2l_ckpt->bmp, page_no) == false); 543 ftl_bitmap_set(band->p2l_map.p2l_ckpt->bmp, page_no); 544 } 545 #endif 546 } 547