1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2023 Solidigm All Rights Reserved 3 * Copyright (C) 2022 Intel Corporation. 4 * All rights reserved. 5 */ 6 7 #include "spdk/bdev.h" 8 9 #include "ftl_core.h" 10 #include "ftl_utils.h" 11 #include "ftl_band.h" 12 #include "ftl_layout.h" 13 #include "ftl_nv_cache.h" 14 #include "ftl_sb.h" 15 #include "nvc/ftl_nvc_dev.h" 16 17 #define FTL_NV_CACHE_CHUNK_DATA_SIZE(blocks) ((uint64_t)blocks * FTL_BLOCK_SIZE) 18 #define FTL_NV_CACHE_CHUNK_SIZE(blocks) \ 19 (FTL_NV_CACHE_CHUNK_DATA_SIZE(blocks) + (2 * FTL_NV_CACHE_CHUNK_MD_SIZE)) 20 21 static inline float 22 blocks2mib(uint64_t blocks) 23 { 24 float result; 25 26 result = blocks; 27 result *= FTL_BLOCK_SIZE; 28 result /= 1024UL; 29 result /= 1024UL; 30 31 return result; 32 } 33 34 static uint64_t 35 superblock_region_size(struct spdk_ftl_dev *dev) 36 { 37 const struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(dev->base_bdev_desc); 38 uint64_t wus = spdk_bdev_get_write_unit_size(bdev) * FTL_BLOCK_SIZE; 39 40 if (wus > FTL_SUPERBLOCK_SIZE) { 41 return wus; 42 } else { 43 return wus * spdk_divide_round_up(FTL_SUPERBLOCK_SIZE, wus); 44 } 45 } 46 47 static uint64_t 48 superblock_region_blocks(struct spdk_ftl_dev *dev) 49 { 50 return superblock_region_size(dev) / FTL_BLOCK_SIZE; 51 } 52 53 uint64_t 54 ftl_md_region_blocks(struct spdk_ftl_dev *dev, uint64_t bytes) 55 { 56 const uint64_t alignment = superblock_region_size(dev); 57 uint64_t result; 58 59 result = spdk_divide_round_up(bytes, alignment); 60 result *= alignment; 61 result /= FTL_BLOCK_SIZE; 62 63 return result; 64 } 65 66 const char * 67 ftl_md_region_name(enum ftl_layout_region_type reg_type) 68 { 69 static const char *md_region_name[FTL_LAYOUT_REGION_TYPE_MAX] = { 70 [FTL_LAYOUT_REGION_TYPE_SB] = "sb", 71 [FTL_LAYOUT_REGION_TYPE_SB_BASE] = "sb_mirror", 72 [FTL_LAYOUT_REGION_TYPE_L2P] = "l2p", 73 [FTL_LAYOUT_REGION_TYPE_BAND_MD] = "band_md", 74 [FTL_LAYOUT_REGION_TYPE_BAND_MD_MIRROR] = "band_md_mirror", 75 [FTL_LAYOUT_REGION_TYPE_VALID_MAP] = "vmap", 76 [FTL_LAYOUT_REGION_TYPE_NVC_MD] = "nvc_md", 77 [FTL_LAYOUT_REGION_TYPE_NVC_MD_MIRROR] = "nvc_md_mirror", 78 [FTL_LAYOUT_REGION_TYPE_DATA_NVC] = "data_nvc", 79 [FTL_LAYOUT_REGION_TYPE_DATA_BASE] = "data_btm", 80 [FTL_LAYOUT_REGION_TYPE_P2L_CKPT_GC] = "p2l0", 81 [FTL_LAYOUT_REGION_TYPE_P2L_CKPT_GC_NEXT] = "p2l1", 82 [FTL_LAYOUT_REGION_TYPE_P2L_CKPT_COMP] = "p2l2", 83 [FTL_LAYOUT_REGION_TYPE_P2L_CKPT_COMP_NEXT] = "p2l3", 84 [FTL_LAYOUT_REGION_TYPE_TRIM_MD] = "trim_md", 85 [FTL_LAYOUT_REGION_TYPE_TRIM_MD_MIRROR] = "trim_md_mirror", 86 }; 87 const char *reg_name = md_region_name[reg_type]; 88 89 assert(reg_type < FTL_LAYOUT_REGION_TYPE_MAX); 90 assert(reg_name != NULL); 91 return reg_name; 92 } 93 94 static void 95 dump_region(struct spdk_ftl_dev *dev, struct ftl_layout_region *region) 96 { 97 assert(!(region->current.offset % superblock_region_blocks(dev))); 98 assert(!(region->current.blocks % superblock_region_blocks(dev))); 99 100 FTL_NOTICELOG(dev, "Region %s\n", region->name); 101 FTL_NOTICELOG(dev, " offset: %.2f MiB\n", 102 blocks2mib(region->current.offset)); 103 FTL_NOTICELOG(dev, " blocks: %.2f MiB\n", 104 blocks2mib(region->current.blocks)); 105 } 106 107 int 108 ftl_validate_regions(struct spdk_ftl_dev *dev, struct ftl_layout *layout) 109 { 110 uint64_t i, j; 111 112 /* Validate if regions doesn't overlap each other */ 113 /* TODO: major upgrades: keep track of and validate free_nvc/free_btm regions */ 114 for (i = 0; i < FTL_LAYOUT_REGION_TYPE_MAX; i++) { 115 struct ftl_layout_region *r1 = &layout->region[i]; 116 117 for (j = 0; j < FTL_LAYOUT_REGION_TYPE_MAX; j++) { 118 struct ftl_layout_region *r2 = &layout->region[j]; 119 120 if (r1->bdev_desc != r2->bdev_desc) { 121 continue; 122 } 123 124 if (i == j) { 125 continue; 126 } 127 128 uint64_t r1_begin = r1->current.offset; 129 uint64_t r1_end = r1->current.offset + r1->current.blocks - 1; 130 uint64_t r2_begin = r2->current.offset; 131 uint64_t r2_end = r2->current.offset + r2->current.blocks - 1; 132 133 if (spdk_max(r1_begin, r2_begin) <= spdk_min(r1_end, r2_end)) { 134 FTL_ERRLOG(dev, "Layout initialization ERROR, two regions overlap, " 135 "%s and %s\n", r1->name, r2->name); 136 return -1; 137 } 138 } 139 } 140 141 return 0; 142 } 143 144 static uint64_t 145 get_num_user_lbas(struct spdk_ftl_dev *dev) 146 { 147 uint64_t blocks; 148 149 blocks = dev->num_bands * ftl_get_num_blocks_in_band(dev); 150 blocks = (blocks * (100 - dev->conf.overprovisioning)) / 100; 151 152 return blocks; 153 } 154 155 static int 156 setup_layout_nvc(struct spdk_ftl_dev *dev) 157 { 158 int region_type; 159 uint64_t left, offset = 0, l2p_blocks; 160 struct ftl_layout *layout = &dev->layout; 161 const struct ftl_md_layout_ops *md_ops = &dev->nv_cache.nvc_desc->ops.md_layout_ops; 162 163 /* Initialize L2P region */ 164 if (!md_ops->region_create(dev, FTL_LAYOUT_REGION_TYPE_L2P, 0, layout->l2p.addr_size, 165 dev->num_lbas)) { 166 goto error; 167 } 168 169 /* Initialize band info metadata */ 170 if (!md_ops->region_create(dev, FTL_LAYOUT_REGION_TYPE_BAND_MD, FTL_BAND_VERSION_CURRENT, 171 sizeof(struct ftl_band_md), ftl_get_num_bands(dev))) { 172 goto error; 173 } 174 175 /* Initialize band info metadata mirror */ 176 if (!md_ops->region_create(dev, FTL_LAYOUT_REGION_TYPE_BAND_MD_MIRROR, FTL_BAND_VERSION_CURRENT, 177 sizeof(struct ftl_band_md), ftl_get_num_bands(dev))) { 178 goto error; 179 } 180 layout->region[FTL_LAYOUT_REGION_TYPE_BAND_MD].mirror_type = FTL_LAYOUT_REGION_TYPE_BAND_MD_MIRROR; 181 182 /* 183 * Initialize P2L checkpointing regions 184 */ 185 for (region_type = FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MIN; 186 region_type <= FTL_LAYOUT_REGION_TYPE_P2L_CKPT_MAX; 187 region_type++) { 188 if (!md_ops->region_create(dev, region_type, FTL_P2L_VERSION_CURRENT, FTL_BLOCK_SIZE, 189 layout->p2l.ckpt_pages)) { 190 goto error; 191 } 192 } 193 194 /* 195 * Initialize trim metadata region 196 */ 197 l2p_blocks = layout->region[FTL_LAYOUT_REGION_TYPE_L2P].current.blocks; 198 if (!md_ops->region_create(dev, FTL_LAYOUT_REGION_TYPE_TRIM_MD, 0, sizeof(uint64_t), l2p_blocks)) { 199 goto error; 200 } 201 202 /* Initialize trim metadata mirror region */ 203 if (!md_ops->region_create(dev, FTL_LAYOUT_REGION_TYPE_TRIM_MD_MIRROR, 0, sizeof(uint64_t), 204 l2p_blocks)) { 205 goto error; 206 } 207 layout->region[FTL_LAYOUT_REGION_TYPE_TRIM_MD].mirror_type = FTL_LAYOUT_REGION_TYPE_TRIM_MD_MIRROR; 208 209 /* 210 * Initialize NV Cache metadata 211 */ 212 offset = layout->region[FTL_LAYOUT_REGION_TYPE_TRIM_MD_MIRROR].current.offset + 213 layout->region[FTL_LAYOUT_REGION_TYPE_TRIM_MD_MIRROR].current.blocks; 214 left = layout->nvc.total_blocks - offset; 215 layout->nvc.chunk_data_blocks = 216 FTL_NV_CACHE_CHUNK_DATA_SIZE(ftl_get_num_blocks_in_band(dev)) / FTL_BLOCK_SIZE; 217 layout->nvc.chunk_meta_size = FTL_NV_CACHE_CHUNK_MD_SIZE; 218 layout->nvc.chunk_count = (left * FTL_BLOCK_SIZE) / 219 FTL_NV_CACHE_CHUNK_SIZE(ftl_get_num_blocks_in_band(dev)); 220 layout->nvc.chunk_tail_md_num_blocks = ftl_nv_cache_chunk_tail_md_num_blocks(&dev->nv_cache); 221 222 if (0 == layout->nvc.chunk_count) { 223 goto error; 224 } 225 if (!md_ops->region_create(dev, FTL_LAYOUT_REGION_TYPE_NVC_MD, FTL_NVC_VERSION_CURRENT, 226 sizeof(struct ftl_nv_cache_chunk_md), layout->nvc.chunk_count)) { 227 goto error; 228 } 229 230 /* 231 * Initialize NV Cache metadata mirror 232 */ 233 if (!md_ops->region_create(dev, FTL_LAYOUT_REGION_TYPE_NVC_MD_MIRROR, FTL_NVC_VERSION_CURRENT, 234 sizeof(struct ftl_nv_cache_chunk_md), layout->nvc.chunk_count)) { 235 goto error; 236 } 237 layout->region[FTL_LAYOUT_REGION_TYPE_NVC_MD].mirror_type = FTL_LAYOUT_REGION_TYPE_NVC_MD_MIRROR; 238 239 /* 240 * Initialize data region on NV cache 241 */ 242 if (!md_ops->region_create(dev, FTL_LAYOUT_REGION_TYPE_DATA_NVC, 0, 243 layout->nvc.chunk_data_blocks * FTL_BLOCK_SIZE, layout->nvc.chunk_count)) { 244 goto error; 245 } 246 247 return 0; 248 249 error: 250 FTL_ERRLOG(dev, "Insufficient NV Cache capacity to preserve metadata\n"); 251 return -1; 252 } 253 254 static ftl_addr 255 layout_base_offset(struct spdk_ftl_dev *dev) 256 { 257 ftl_addr addr; 258 259 addr = dev->num_bands * ftl_get_num_blocks_in_band(dev); 260 return addr; 261 } 262 263 static int 264 setup_layout_base(struct spdk_ftl_dev *dev) 265 { 266 struct ftl_layout *layout = &dev->layout; 267 const struct ftl_md_layout_ops *md_ops = &dev->base_type->ops.md_layout_ops; 268 uint64_t valid_map_size; 269 270 layout->base.num_usable_blocks = ftl_get_num_blocks_in_band(dev); 271 layout->base.user_blocks = ftl_band_user_blocks(dev->bands); 272 273 /* Base device layout is as follows: 274 * - superblock 275 * - data 276 * - valid map 277 */ 278 if (!md_ops->region_create(dev, FTL_LAYOUT_REGION_TYPE_DATA_BASE, 0, FTL_BLOCK_SIZE, 279 layout_base_offset(dev))) { 280 return -1; 281 } 282 283 valid_map_size = spdk_divide_round_up(layout->base.total_blocks + layout->nvc.total_blocks, 8); 284 if (!md_ops->region_create(dev, FTL_LAYOUT_REGION_TYPE_VALID_MAP, 0, FTL_BLOCK_SIZE, 285 ftl_md_region_blocks(dev, valid_map_size))) { 286 return -1; 287 } 288 289 return 0; 290 } 291 292 int 293 ftl_layout_setup(struct spdk_ftl_dev *dev) 294 { 295 struct ftl_layout *layout = &dev->layout; 296 uint64_t i; 297 uint64_t num_lbas; 298 299 /* Initialize mirrors types */ 300 for (i = 0; i < FTL_LAYOUT_REGION_TYPE_MAX; ++i) { 301 if (i == FTL_LAYOUT_REGION_TYPE_SB) { 302 /* Super block has been already initialized */ 303 continue; 304 } 305 306 layout->region[i].mirror_type = FTL_LAYOUT_REGION_TYPE_INVALID; 307 } 308 309 /* 310 * Initialize L2P information 311 */ 312 num_lbas = get_num_user_lbas(dev); 313 if (dev->num_lbas == 0) { 314 assert(dev->conf.mode & SPDK_FTL_MODE_CREATE); 315 dev->num_lbas = num_lbas; 316 dev->sb->lba_cnt = num_lbas; 317 } else if (dev->num_lbas != num_lbas) { 318 FTL_ERRLOG(dev, "Mismatched FTL num_lbas\n"); 319 return -EINVAL; 320 } 321 layout->l2p.addr_length = spdk_u64log2(layout->base.total_blocks + layout->nvc.total_blocks) + 1; 322 layout->l2p.addr_size = layout->l2p.addr_length > 32 ? 8 : 4; 323 layout->l2p.lbas_in_page = FTL_BLOCK_SIZE / layout->l2p.addr_size; 324 325 /* Setup P2L ckpt */ 326 layout->p2l.ckpt_pages = spdk_divide_round_up(ftl_get_num_blocks_in_band(dev), dev->xfer_size); 327 328 if (setup_layout_nvc(dev)) { 329 return -EINVAL; 330 } 331 332 if (setup_layout_base(dev)) { 333 return -EINVAL; 334 } 335 336 if (ftl_validate_regions(dev, layout)) { 337 return -EINVAL; 338 } 339 340 FTL_NOTICELOG(dev, "Base device capacity: %.2f MiB\n", 341 blocks2mib(layout->base.total_blocks)); 342 FTL_NOTICELOG(dev, "NV cache device capacity: %.2f MiB\n", 343 blocks2mib(layout->nvc.total_blocks)); 344 FTL_NOTICELOG(dev, "L2P entries: %"PRIu64"\n", dev->num_lbas); 345 FTL_NOTICELOG(dev, "L2P address size: %"PRIu64"\n", layout->l2p.addr_size); 346 FTL_NOTICELOG(dev, "P2L checkpoint pages: %"PRIu64"\n", layout->p2l.ckpt_pages); 347 348 return 0; 349 } 350 351 int 352 ftl_layout_setup_superblock(struct spdk_ftl_dev *dev) 353 { 354 const struct spdk_bdev *bdev; 355 struct ftl_layout *layout = &dev->layout; 356 struct ftl_layout_region *region = &layout->region[FTL_LAYOUT_REGION_TYPE_SB]; 357 uint64_t total_blocks, offset, left; 358 const struct ftl_md_layout_ops *md_ops = &dev->nv_cache.nvc_desc->ops.md_layout_ops; 359 const struct ftl_md_layout_ops *base_md_ops = &dev->base_type->ops.md_layout_ops; 360 361 assert(layout->md[FTL_LAYOUT_REGION_TYPE_SB] == NULL); 362 363 bdev = spdk_bdev_desc_get_bdev(dev->base_bdev_desc); 364 layout->base.total_blocks = spdk_bdev_get_num_blocks(bdev); 365 366 bdev = spdk_bdev_desc_get_bdev(dev->nv_cache.bdev_desc); 367 layout->nvc.total_blocks = spdk_bdev_get_num_blocks(bdev); 368 369 /* Initialize superblock region */ 370 if (!md_ops->region_create(dev, FTL_LAYOUT_REGION_TYPE_SB, FTL_SB_VERSION_CURRENT, 371 superblock_region_size(dev), 1)) { 372 FTL_ERRLOG(dev, "Error when setting up primary super block\n"); 373 return -1; 374 } 375 376 assert(region->bdev_desc != NULL); 377 assert(region->ioch != NULL); 378 assert(region->current.offset == 0); 379 380 if (!base_md_ops->region_create(dev, FTL_LAYOUT_REGION_TYPE_SB_BASE, FTL_SB_VERSION_CURRENT, 381 superblock_region_size(dev), 1)) { 382 FTL_ERRLOG(dev, "Error when setting up secondary super block\n"); 383 return -1; 384 } 385 layout->region[FTL_LAYOUT_REGION_TYPE_SB].mirror_type = FTL_LAYOUT_REGION_TYPE_SB_BASE; 386 387 region = &layout->region[FTL_LAYOUT_REGION_TYPE_SB_BASE]; 388 assert(region->current.offset == 0); 389 390 /* Check if SB can be stored at the end of base device */ 391 total_blocks = spdk_bdev_get_num_blocks( 392 spdk_bdev_desc_get_bdev(dev->base_bdev_desc)); 393 offset = region->current.offset + region->current.blocks; 394 left = total_blocks - offset; 395 if ((left > total_blocks) || (offset > total_blocks)) { 396 FTL_ERRLOG(dev, "Error when setup base device super block\n"); 397 return -1; 398 } 399 400 return 0; 401 } 402 403 void 404 ftl_layout_dump(struct spdk_ftl_dev *dev) 405 { 406 struct ftl_layout *layout = &dev->layout; 407 int i; 408 409 FTL_NOTICELOG(dev, "NV cache layout:\n"); 410 for (i = 0; i < FTL_LAYOUT_REGION_TYPE_MAX; ++i) { 411 if (layout->region[i].bdev_desc == dev->nv_cache.bdev_desc) { 412 dump_region(dev, &layout->region[i]); 413 } 414 } 415 FTL_NOTICELOG(dev, "Base device layout:\n"); 416 for (i = 0; i < FTL_LAYOUT_REGION_TYPE_MAX; ++i) { 417 if (layout->region[i].bdev_desc == dev->base_bdev_desc) { 418 dump_region(dev, &layout->region[i]); 419 } 420 } 421 } 422 423 uint64_t 424 ftl_layout_base_md_blocks(struct spdk_ftl_dev *dev) 425 { 426 const struct spdk_bdev *bdev; 427 uint64_t md_blocks = 0, total_blocks = 0; 428 429 bdev = spdk_bdev_desc_get_bdev(dev->base_bdev_desc); 430 total_blocks += spdk_bdev_get_num_blocks(bdev); 431 432 bdev = spdk_bdev_desc_get_bdev(dev->nv_cache.bdev_desc); 433 total_blocks += spdk_bdev_get_num_blocks(bdev); 434 435 /* Count space needed for validity map */ 436 md_blocks += ftl_md_region_blocks(dev, spdk_divide_round_up(total_blocks, 8)); 437 438 /* Count space needed for superblock */ 439 md_blocks += superblock_region_blocks(dev); 440 return md_blocks; 441 } 442