1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2023 Solidigm All Rights Reserved 3 * Copyright (C) 2022 Intel Corporation. 4 * All rights reserved. 5 */ 6 7 #include "spdk/bdev.h" 8 9 #include "ftl_core.h" 10 #include "ftl_utils.h" 11 #include "ftl_band.h" 12 #include "ftl_layout.h" 13 #include "ftl_nv_cache.h" 14 #include "ftl_sb.h" 15 16 #define FTL_NV_CACHE_CHUNK_DATA_SIZE(blocks) ((uint64_t)blocks * FTL_BLOCK_SIZE) 17 #define FTL_NV_CACHE_CHUNK_SIZE(blocks) \ 18 (FTL_NV_CACHE_CHUNK_DATA_SIZE(blocks) + (2 * FTL_NV_CACHE_CHUNK_MD_SIZE)) 19 20 static inline float 21 blocks2mib(uint64_t blocks) 22 { 23 float result; 24 25 result = blocks; 26 result *= FTL_BLOCK_SIZE; 27 result /= 1024UL; 28 result /= 1024UL; 29 30 return result; 31 } 32 33 static uint64_t 34 superblock_region_size(struct spdk_ftl_dev *dev) 35 { 36 const struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(dev->base_bdev_desc); 37 uint64_t wus = spdk_bdev_get_write_unit_size(bdev) * FTL_BLOCK_SIZE; 38 39 if (wus > FTL_SUPERBLOCK_SIZE) { 40 return wus; 41 } else { 42 return wus * spdk_divide_round_up(FTL_SUPERBLOCK_SIZE, wus); 43 } 44 } 45 46 static uint64_t 47 superblock_region_blocks(struct spdk_ftl_dev *dev) 48 { 49 return superblock_region_size(dev) / FTL_BLOCK_SIZE; 50 } 51 52 static inline uint64_t 53 blocks_region(struct spdk_ftl_dev *dev, uint64_t bytes) 54 { 55 const uint64_t alignment = superblock_region_size(dev); 56 uint64_t result; 57 58 result = spdk_divide_round_up(bytes, alignment); 59 result *= alignment; 60 result /= FTL_BLOCK_SIZE; 61 62 return result; 63 } 64 65 static void 66 dump_region(struct spdk_ftl_dev *dev, struct ftl_layout_region *region) 67 { 68 assert(!(region->current.offset % superblock_region_blocks(dev))); 69 assert(!(region->current.blocks % superblock_region_blocks(dev))); 70 71 FTL_NOTICELOG(dev, "Region %s\n", region->name); 72 FTL_NOTICELOG(dev, " offset: %.2f MiB\n", 73 blocks2mib(region->current.offset)); 74 FTL_NOTICELOG(dev, " blocks: %.2f MiB\n", 75 blocks2mib(region->current.blocks)); 76 } 77 78 int 79 ftl_validate_regions(struct spdk_ftl_dev *dev, struct ftl_layout *layout) 80 { 81 uint64_t i, j; 82 83 /* Validate if regions doesn't overlap each other */ 84 /* TODO: major upgrades: keep track of and validate free_nvc/free_btm regions */ 85 for (i = 0; i < FTL_LAYOUT_REGION_TYPE_MAX; i++) { 86 struct ftl_layout_region *r1 = &layout->region[i]; 87 88 for (j = 0; j < FTL_LAYOUT_REGION_TYPE_MAX; j++) { 89 struct ftl_layout_region *r2 = &layout->region[j]; 90 91 if (r1->bdev_desc != r2->bdev_desc) { 92 continue; 93 } 94 95 if (i == j) { 96 continue; 97 } 98 99 uint64_t r1_begin = r1->current.offset; 100 uint64_t r1_end = r1->current.offset + r1->current.blocks - 1; 101 uint64_t r2_begin = r2->current.offset; 102 uint64_t r2_end = r2->current.offset + r2->current.blocks - 1; 103 104 if (spdk_max(r1_begin, r2_begin) <= spdk_min(r1_end, r2_end)) { 105 FTL_ERRLOG(dev, "Layout initialization ERROR, two regions overlap, " 106 "%s and %s\n", r1->name, r2->name); 107 return -1; 108 } 109 } 110 } 111 112 return 0; 113 } 114 115 static uint64_t 116 get_num_user_lbas(struct spdk_ftl_dev *dev) 117 { 118 uint64_t blocks; 119 120 blocks = dev->num_bands * ftl_get_num_blocks_in_band(dev); 121 blocks = (blocks * (100 - dev->conf.overprovisioning)) / 100; 122 123 return blocks; 124 } 125 126 static void 127 set_region_bdev_nvc(struct ftl_layout_region *reg, struct spdk_ftl_dev *dev) 128 { 129 reg->bdev_desc = dev->nv_cache.bdev_desc; 130 reg->ioch = dev->nv_cache.cache_ioch; 131 reg->vss_blksz = dev->nv_cache.md_size; 132 } 133 134 static void 135 set_region_bdev_btm(struct ftl_layout_region *reg, struct spdk_ftl_dev *dev) 136 { 137 reg->bdev_desc = dev->base_bdev_desc; 138 reg->ioch = dev->base_ioch; 139 reg->vss_blksz = 0; 140 } 141 142 static int 143 setup_layout_nvc(struct spdk_ftl_dev *dev) 144 { 145 int region_type; 146 uint64_t left, offset = 0, l2p_blocks; 147 struct ftl_layout *layout = &dev->layout; 148 struct ftl_layout_region *region, *mirror; 149 static const char *p2l_region_name[] = { 150 "p2l0", 151 "p2l1", 152 "p2l2", 153 "p2l3" 154 }; 155 156 /* Skip the superblock region. Already init`d in ftl_layout_setup_superblock */ 157 region = &layout->region[FTL_LAYOUT_REGION_TYPE_SB]; 158 offset += region->current.blocks; 159 160 /* Initialize L2P region */ 161 if (offset >= layout->nvc.total_blocks) { 162 goto error; 163 } 164 region = &layout->region[FTL_LAYOUT_REGION_TYPE_L2P]; 165 region->type = FTL_LAYOUT_REGION_TYPE_L2P; 166 region->name = "l2p"; 167 region->current.version = 0; 168 region->prev.version = 0; 169 region->current.offset = offset; 170 region->current.blocks = blocks_region(dev, layout->l2p.addr_size * dev->num_lbas); 171 set_region_bdev_nvc(region, dev); 172 offset += region->current.blocks; 173 174 /* Initialize band info metadata */ 175 if (offset >= layout->nvc.total_blocks) { 176 goto error; 177 } 178 region = &layout->region[FTL_LAYOUT_REGION_TYPE_BAND_MD]; 179 region->type = FTL_LAYOUT_REGION_TYPE_BAND_MD; 180 region->mirror_type = FTL_LAYOUT_REGION_TYPE_BAND_MD_MIRROR; 181 region->name = "band_md"; 182 region->current.version = region->prev.version = FTL_BAND_VERSION_CURRENT; 183 region->current.offset = offset; 184 region->current.blocks = blocks_region(dev, ftl_get_num_bands(dev) * sizeof(struct ftl_band_md)); 185 region->entry_size = sizeof(struct ftl_band_md) / FTL_BLOCK_SIZE; 186 region->num_entries = ftl_get_num_bands(dev); 187 set_region_bdev_nvc(region, dev); 188 offset += region->current.blocks; 189 190 /* Initialize band info metadata mirror */ 191 if (offset >= layout->nvc.total_blocks) { 192 goto error; 193 } 194 mirror = &layout->region[FTL_LAYOUT_REGION_TYPE_BAND_MD_MIRROR]; 195 *mirror = *region; 196 mirror->type = FTL_LAYOUT_REGION_TYPE_BAND_MD_MIRROR; 197 mirror->mirror_type = FTL_LAYOUT_REGION_TYPE_INVALID; 198 mirror->name = "band_md_mirror"; 199 mirror->current.offset += region->current.blocks; 200 offset += mirror->current.blocks; 201 202 if (offset >= layout->nvc.total_blocks) { 203 goto error; 204 } 205 206 /* 207 * Initialize P2L checkpointing regions 208 */ 209 SPDK_STATIC_ASSERT(SPDK_COUNTOF(p2l_region_name) == FTL_LAYOUT_REGION_TYPE_P2L_COUNT, 210 "Incorrect # of P2L region names"); 211 for (region_type = FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MIN; 212 region_type <= FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MAX; 213 region_type++) { 214 if (offset >= layout->nvc.total_blocks) { 215 goto error; 216 } 217 region = &layout->region[region_type]; 218 region->type = region_type; 219 region->name = p2l_region_name[region_type - FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MIN]; 220 region->current.version = FTL_P2L_VERSION_CURRENT; 221 region->prev.version = FTL_P2L_VERSION_CURRENT; 222 region->current.offset = offset; 223 region->current.blocks = blocks_region(dev, layout->p2l.ckpt_pages * FTL_BLOCK_SIZE); 224 region->entry_size = 1; 225 region->num_entries = region->current.blocks; 226 set_region_bdev_nvc(region, dev); 227 offset += region->current.blocks; 228 } 229 230 /* 231 * Initialize trim metadata region 232 */ 233 if (offset >= layout->nvc.total_blocks) { 234 goto error; 235 } 236 l2p_blocks = layout->region[FTL_LAYOUT_REGION_TYPE_L2P].current.blocks; 237 region = &layout->region[FTL_LAYOUT_REGION_TYPE_TRIM_MD]; 238 region->type = FTL_LAYOUT_REGION_TYPE_TRIM_MD; 239 region->mirror_type = FTL_LAYOUT_REGION_TYPE_TRIM_MD_MIRROR; 240 region->name = "trim_md"; 241 region->current.version = 0; 242 region->prev.version = 0; 243 region->current.offset = offset; 244 region->current.blocks = blocks_region(dev, l2p_blocks * sizeof(uint64_t)); 245 region->entry_size = 1; 246 region->num_entries = region->current.blocks; 247 set_region_bdev_nvc(region, dev); 248 offset += region->current.blocks; 249 250 /* Initialize trim metadata mirror region */ 251 if (offset >= layout->nvc.total_blocks) { 252 goto error; 253 } 254 mirror = &layout->region[FTL_LAYOUT_REGION_TYPE_TRIM_MD_MIRROR]; 255 *mirror = *region; 256 mirror->type = FTL_LAYOUT_REGION_TYPE_TRIM_MD_MIRROR; 257 mirror->mirror_type = FTL_LAYOUT_REGION_TYPE_INVALID; 258 mirror->name = "trim_md_mirror"; 259 mirror->current.offset += region->current.blocks; 260 offset += mirror->current.blocks; 261 262 /* 263 * Initialize NV Cache metadata 264 */ 265 if (offset >= layout->nvc.total_blocks) { 266 goto error; 267 } 268 269 left = layout->nvc.total_blocks - offset; 270 layout->nvc.chunk_data_blocks = 271 FTL_NV_CACHE_CHUNK_DATA_SIZE(ftl_get_num_blocks_in_band(dev)) / FTL_BLOCK_SIZE; 272 layout->nvc.chunk_meta_size = FTL_NV_CACHE_CHUNK_MD_SIZE; 273 layout->nvc.chunk_count = (left * FTL_BLOCK_SIZE) / 274 FTL_NV_CACHE_CHUNK_SIZE(ftl_get_num_blocks_in_band(dev)); 275 layout->nvc.chunk_tail_md_num_blocks = ftl_nv_cache_chunk_tail_md_num_blocks(&dev->nv_cache); 276 277 if (0 == layout->nvc.chunk_count) { 278 goto error; 279 } 280 region = &layout->region[FTL_LAYOUT_REGION_TYPE_NVC_MD]; 281 region->type = FTL_LAYOUT_REGION_TYPE_NVC_MD; 282 region->mirror_type = FTL_LAYOUT_REGION_TYPE_NVC_MD_MIRROR; 283 region->name = "nvc_md"; 284 region->current.version = region->prev.version = FTL_NVC_VERSION_CURRENT; 285 region->current.offset = offset; 286 region->current.blocks = blocks_region(dev, layout->nvc.chunk_count * 287 sizeof(struct ftl_nv_cache_chunk_md)); 288 region->entry_size = sizeof(struct ftl_nv_cache_chunk_md) / FTL_BLOCK_SIZE; 289 region->num_entries = layout->nvc.chunk_count; 290 set_region_bdev_nvc(region, dev); 291 offset += region->current.blocks; 292 293 /* 294 * Initialize NV Cache metadata mirror 295 */ 296 mirror = &layout->region[FTL_LAYOUT_REGION_TYPE_NVC_MD_MIRROR]; 297 *mirror = *region; 298 mirror->type = FTL_LAYOUT_REGION_TYPE_NVC_MD_MIRROR; 299 mirror->mirror_type = FTL_LAYOUT_REGION_TYPE_INVALID; 300 mirror->name = "nvc_md_mirror"; 301 mirror->current.offset += region->current.blocks; 302 offset += mirror->current.blocks; 303 304 /* 305 * Initialize data region on NV cache 306 */ 307 if (offset >= layout->nvc.total_blocks) { 308 goto error; 309 } 310 region = &layout->region[FTL_LAYOUT_REGION_TYPE_DATA_NVC]; 311 region->type = FTL_LAYOUT_REGION_TYPE_DATA_NVC; 312 region->name = "data_nvc"; 313 region->current.version = region->prev.version = 0; 314 region->current.offset = offset; 315 region->current.blocks = layout->nvc.chunk_count * layout->nvc.chunk_data_blocks; 316 set_region_bdev_nvc(region, dev); 317 offset += region->current.blocks; 318 319 left = layout->nvc.total_blocks - offset; 320 if (left > layout->nvc.chunk_data_blocks) { 321 FTL_ERRLOG(dev, "Error when setup NV cache layout\n"); 322 return -1; 323 } 324 325 if (offset > layout->nvc.total_blocks) { 326 FTL_ERRLOG(dev, "Error when setup NV cache layout\n"); 327 goto error; 328 } 329 330 return 0; 331 332 error: 333 FTL_ERRLOG(dev, "Insufficient NV Cache capacity to preserve metadata\n"); 334 return -1; 335 } 336 337 static ftl_addr 338 layout_base_offset(struct spdk_ftl_dev *dev) 339 { 340 ftl_addr addr; 341 342 addr = dev->num_bands * ftl_get_num_blocks_in_band(dev); 343 return addr; 344 } 345 346 static int 347 setup_layout_base(struct spdk_ftl_dev *dev) 348 { 349 uint64_t left, offset; 350 struct ftl_layout *layout = &dev->layout; 351 struct ftl_layout_region *region; 352 uint64_t data_base_alignment = 8 * ftl_bitmap_buffer_alignment; 353 /* Allocating a ftl_bitmap requires a 8B input buffer alignment, since we're reusing the global valid map md buffer 354 * this means that each band starting address needs to be aligned too - each device sector takes 1b in the valid map, 355 * so 64 sectors (8*8) is the needed alignment 356 */ 357 358 layout->base.num_usable_blocks = ftl_get_num_blocks_in_band(dev); 359 layout->base.user_blocks = ftl_band_user_blocks(dev->bands); 360 361 /* Base device layout is following: 362 * - superblock 363 * - data 364 * - valid map 365 */ 366 offset = layout->region[FTL_LAYOUT_REGION_TYPE_SB_BASE].current.blocks; 367 offset = SPDK_ALIGN_CEIL(offset, data_base_alignment); 368 369 /* Setup data region on base device */ 370 region = &layout->region[FTL_LAYOUT_REGION_TYPE_DATA_BASE]; 371 region->type = FTL_LAYOUT_REGION_TYPE_DATA_BASE; 372 region->name = "data_btm"; 373 region->current.version = region->prev.version = 0; 374 region->current.offset = offset; 375 region->current.blocks = layout_base_offset(dev); 376 set_region_bdev_btm(region, dev); 377 378 offset += region->current.blocks; 379 380 /* Setup validity map */ 381 region = &layout->region[FTL_LAYOUT_REGION_TYPE_VALID_MAP]; 382 region->type = FTL_LAYOUT_REGION_TYPE_VALID_MAP; 383 region->name = "vmap"; 384 region->current.version = region->prev.version = 0; 385 region->current.offset = offset; 386 region->current.blocks = blocks_region(dev, spdk_divide_round_up( 387 layout->base.total_blocks + layout->nvc.total_blocks, 8)); 388 set_region_bdev_btm(region, dev); 389 offset += region->current.blocks; 390 391 /* Checking for underflow */ 392 left = layout->base.total_blocks - offset; 393 if (left > layout->base.total_blocks) { 394 FTL_ERRLOG(dev, "Error when setup base device layout\n"); 395 return -1; 396 } 397 398 if (offset > layout->base.total_blocks) { 399 FTL_ERRLOG(dev, "Error when setup base device layout\n"); 400 return -1; 401 } 402 403 return 0; 404 } 405 406 int 407 ftl_layout_setup(struct spdk_ftl_dev *dev) 408 { 409 const struct spdk_bdev *bdev; 410 struct ftl_layout *layout = &dev->layout; 411 uint64_t i; 412 uint64_t num_lbas; 413 414 bdev = spdk_bdev_desc_get_bdev(dev->base_bdev_desc); 415 layout->base.total_blocks = spdk_bdev_get_num_blocks(bdev); 416 417 bdev = spdk_bdev_desc_get_bdev(dev->nv_cache.bdev_desc); 418 layout->nvc.total_blocks = spdk_bdev_get_num_blocks(bdev); 419 420 /* Initialize mirrors types */ 421 for (i = 0; i < FTL_LAYOUT_REGION_TYPE_MAX; ++i) { 422 if (i == FTL_LAYOUT_REGION_TYPE_SB) { 423 /* Super block has been already initialized */ 424 continue; 425 } 426 427 layout->region[i].mirror_type = FTL_LAYOUT_REGION_TYPE_INVALID; 428 } 429 430 /* 431 * Initialize L2P information 432 */ 433 num_lbas = get_num_user_lbas(dev); 434 if (dev->num_lbas == 0) { 435 assert(dev->conf.mode & SPDK_FTL_MODE_CREATE); 436 dev->num_lbas = num_lbas; 437 dev->sb->lba_cnt = num_lbas; 438 } else if (dev->num_lbas != num_lbas) { 439 FTL_ERRLOG(dev, "Mismatched FTL num_lbas\n"); 440 return -EINVAL; 441 } 442 layout->l2p.addr_length = spdk_u64log2(layout->base.total_blocks + layout->nvc.total_blocks) + 1; 443 layout->l2p.addr_size = layout->l2p.addr_length > 32 ? 8 : 4; 444 layout->l2p.lbas_in_page = FTL_BLOCK_SIZE / layout->l2p.addr_size; 445 446 /* Setup P2L ckpt */ 447 layout->p2l.ckpt_pages = spdk_divide_round_up(ftl_get_num_blocks_in_band(dev), dev->xfer_size); 448 449 if (setup_layout_nvc(dev)) { 450 return -EINVAL; 451 } 452 453 if (setup_layout_base(dev)) { 454 return -EINVAL; 455 } 456 457 if (ftl_validate_regions(dev, layout)) { 458 return -EINVAL; 459 } 460 461 FTL_NOTICELOG(dev, "Base device capacity: %.2f MiB\n", 462 blocks2mib(layout->base.total_blocks)); 463 FTL_NOTICELOG(dev, "NV cache device capacity: %.2f MiB\n", 464 blocks2mib(layout->nvc.total_blocks)); 465 FTL_NOTICELOG(dev, "L2P entries: %"PRIu64"\n", dev->num_lbas); 466 FTL_NOTICELOG(dev, "L2P address size: %"PRIu64"\n", layout->l2p.addr_size); 467 FTL_NOTICELOG(dev, "P2L checkpoint pages: %"PRIu64"\n", layout->p2l.ckpt_pages); 468 469 return 0; 470 } 471 472 int 473 ftl_layout_setup_superblock(struct spdk_ftl_dev *dev) 474 { 475 struct ftl_layout *layout = &dev->layout; 476 struct ftl_layout_region *region = &layout->region[FTL_LAYOUT_REGION_TYPE_SB]; 477 uint64_t total_blocks, offset, left; 478 479 assert(layout->md[FTL_LAYOUT_REGION_TYPE_SB] == NULL); 480 481 /* Initialize superblock region */ 482 region->type = FTL_LAYOUT_REGION_TYPE_SB; 483 region->mirror_type = FTL_LAYOUT_REGION_TYPE_SB_BASE; 484 region->name = "sb"; 485 region->current.version = FTL_SB_VERSION_CURRENT; 486 region->prev.version = FTL_SB_VERSION_CURRENT; 487 region->current.offset = 0; 488 489 region->current.blocks = superblock_region_blocks(dev); 490 region->vss_blksz = 0; 491 region->bdev_desc = dev->nv_cache.bdev_desc; 492 region->ioch = dev->nv_cache.cache_ioch; 493 494 assert(region->bdev_desc != NULL); 495 assert(region->ioch != NULL); 496 497 region = &layout->region[FTL_LAYOUT_REGION_TYPE_SB_BASE]; 498 region->type = FTL_LAYOUT_REGION_TYPE_SB_BASE; 499 region->mirror_type = FTL_LAYOUT_REGION_TYPE_INVALID; 500 region->name = "sb_mirror"; 501 region->current.version = FTL_SB_VERSION_CURRENT; 502 region->prev.version = FTL_SB_VERSION_CURRENT; 503 region->current.offset = 0; 504 region->current.blocks = superblock_region_blocks(dev); 505 set_region_bdev_btm(region, dev); 506 507 /* Check if SB can be stored at the end of base device */ 508 total_blocks = spdk_bdev_get_num_blocks( 509 spdk_bdev_desc_get_bdev(dev->base_bdev_desc)); 510 offset = region->current.offset + region->current.blocks; 511 left = total_blocks - offset; 512 if ((left > total_blocks) || (offset > total_blocks)) { 513 FTL_ERRLOG(dev, "Error when setup base device super block\n"); 514 return -1; 515 } 516 517 return 0; 518 } 519 520 void 521 ftl_layout_dump(struct spdk_ftl_dev *dev) 522 { 523 struct ftl_layout *layout = &dev->layout; 524 int i; 525 526 FTL_NOTICELOG(dev, "NV cache layout:\n"); 527 for (i = 0; i < FTL_LAYOUT_REGION_TYPE_MAX; ++i) { 528 if (layout->region[i].bdev_desc == dev->nv_cache.bdev_desc) { 529 dump_region(dev, &layout->region[i]); 530 } 531 } 532 FTL_NOTICELOG(dev, "Base device layout:\n"); 533 for (i = 0; i < FTL_LAYOUT_REGION_TYPE_MAX; ++i) { 534 if (layout->region[i].bdev_desc == dev->base_bdev_desc) { 535 dump_region(dev, &layout->region[i]); 536 } 537 } 538 } 539 540 uint64_t 541 ftl_layout_base_md_blocks(struct spdk_ftl_dev *dev) 542 { 543 const struct spdk_bdev *bdev; 544 uint64_t md_blocks = 0, total_blocks = 0; 545 546 bdev = spdk_bdev_desc_get_bdev(dev->base_bdev_desc); 547 total_blocks += spdk_bdev_get_num_blocks(bdev); 548 549 bdev = spdk_bdev_desc_get_bdev(dev->nv_cache.bdev_desc); 550 total_blocks += spdk_bdev_get_num_blocks(bdev); 551 552 /* Count space needed for validity map */ 553 md_blocks += blocks_region(dev, spdk_divide_round_up(total_blocks, 8)); 554 555 /* Count space needed for superblock */ 556 md_blocks += superblock_region_blocks(dev); 557 return md_blocks; 558 } 559