1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "ftl_core.h" 7 #include "ftl_mngt_steps.h" 8 #include "ftl_band.h" 9 #include "ftl_internal.h" 10 11 static int 12 ftl_band_init_md(struct ftl_band *band) 13 { 14 struct spdk_ftl_dev *dev = band->dev; 15 struct ftl_p2l_map *p2l_map = &band->p2l_map; 16 struct ftl_md *band_info_md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_BAND_MD]; 17 struct ftl_md *valid_map_md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_VALID_MAP]; 18 uint64_t band_num_blocks = ftl_get_num_blocks_in_band(band->dev); 19 size_t band_valid_map_bytes; 20 struct ftl_band_md *band_md = ftl_md_get_buffer(band_info_md); 21 22 if (band_num_blocks % (ftl_bitmap_buffer_alignment * 8)) { 23 FTL_ERRLOG(dev, "The number of blocks in band is not divisible by bitmap word bits\n"); 24 return -EINVAL; 25 } 26 band_valid_map_bytes = band_num_blocks / 8; 27 28 p2l_map->valid = ftl_bitmap_create(ftl_md_get_buffer(valid_map_md) + 29 band->start_addr / 8, band_valid_map_bytes); 30 if (!p2l_map->valid) { 31 return -ENOMEM; 32 } 33 34 band->md = &band_md[band->id]; 35 if (!ftl_fast_startup(dev)) { 36 band->md->df_p2l_map = FTL_DF_OBJ_ID_INVALID; 37 } 38 39 return 0; 40 } 41 42 static int 43 ftl_dev_init_bands(struct spdk_ftl_dev *dev) 44 { 45 struct ftl_band *band; 46 uint64_t i, blocks, md_blocks, md_bands; 47 48 /* Calculate initial number of bands */ 49 blocks = spdk_bdev_get_num_blocks(spdk_bdev_desc_get_bdev(dev->base_bdev_desc)); 50 dev->num_bands = blocks / ftl_get_num_blocks_in_band(dev); 51 52 /* Calculate number of bands considering base device metadata size requirement */ 53 md_blocks = ftl_layout_base_md_blocks(dev); 54 md_bands = spdk_divide_round_up(md_blocks, dev->num_blocks_in_band); 55 56 if (dev->num_bands > md_bands) { 57 /* Save a band worth of space for metadata */ 58 dev->num_bands -= md_bands; 59 } else { 60 FTL_ERRLOG(dev, "Base device too small to store metadata\n"); 61 return -1; 62 } 63 64 TAILQ_INIT(&dev->free_bands); 65 TAILQ_INIT(&dev->shut_bands); 66 67 dev->num_free = 0; 68 dev->bands = calloc(ftl_get_num_bands(dev), sizeof(*dev->bands)); 69 if (!dev->bands) { 70 return -ENOMEM; 71 } 72 73 for (i = 0; i < ftl_get_num_bands(dev); ++i) { 74 band = &dev->bands[i]; 75 band->id = i; 76 band->dev = dev; 77 78 /* Adding to shut_bands is necessary - see ftl_restore_band_close_cb() */ 79 TAILQ_INSERT_TAIL(&dev->shut_bands, band, queue_entry); 80 } 81 82 return 0; 83 } 84 85 static int 86 ftl_dev_init_bands_md(struct spdk_ftl_dev *dev) 87 { 88 uint64_t i; 89 int rc = 0; 90 91 for (i = 0; i < ftl_get_num_bands(dev); ++i) { 92 rc = ftl_band_init_md(&dev->bands[i]); 93 if (rc) { 94 FTL_ERRLOG(dev, "Failed to initialize metadata structures for band [%lu]\n", i); 95 break; 96 } 97 } 98 99 return rc; 100 } 101 102 static void 103 ftl_dev_deinit_bands(struct spdk_ftl_dev *dev) 104 { 105 free(dev->bands); 106 } 107 108 static void 109 ftl_dev_deinit_bands_md(struct spdk_ftl_dev *dev) 110 { 111 if (dev->bands) { 112 uint64_t i; 113 for (i = 0; i < dev->num_bands; ++i) { 114 struct ftl_band *band = &dev->bands[i]; 115 116 ftl_bitmap_destroy(band->p2l_map.valid); 117 band->p2l_map.valid = NULL; 118 119 band->md = NULL; 120 } 121 } 122 } 123 124 void 125 ftl_mngt_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 126 { 127 if (ftl_dev_init_bands(dev)) { 128 ftl_mngt_fail_step(mngt); 129 } else { 130 ftl_mngt_next_step(mngt); 131 } 132 } 133 134 void 135 ftl_mngt_init_bands_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 136 { 137 if (ftl_dev_init_bands_md(dev)) { 138 ftl_mngt_fail_step(mngt); 139 } else { 140 ftl_mngt_next_step(mngt); 141 } 142 } 143 144 void 145 ftl_mngt_deinit_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 146 { 147 ftl_dev_deinit_bands(dev); 148 ftl_mngt_next_step(mngt); 149 } 150 151 void 152 ftl_mngt_deinit_bands_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 153 { 154 ftl_dev_deinit_bands_md(dev); 155 ftl_mngt_next_step(mngt); 156 } 157 158 /* 159 * For grouping multiple logical bands (1GiB) to make any IOs more sequential from the drive's 160 * perspective. Improves WAF. 161 */ 162 #define BASE_BDEV_RECLAIM_UNIT_SIZE (72 * GiB) 163 164 static void 165 decorate_bands(struct spdk_ftl_dev *dev) 166 { 167 struct ftl_band *band; 168 uint64_t i, num_to_drop, phys_id = 0; 169 uint64_t num_blocks, num_bands; 170 uint64_t num_blocks_in_band = ftl_get_num_blocks_in_band(dev); 171 uint64_t reclaim_unit_num_blocks = BASE_BDEV_RECLAIM_UNIT_SIZE / FTL_BLOCK_SIZE; 172 uint32_t num_logical_in_phys = 2; 173 174 assert(reclaim_unit_num_blocks % num_blocks_in_band == 0); 175 176 num_blocks = spdk_bdev_get_num_blocks(spdk_bdev_desc_get_bdev(dev->base_bdev_desc)); 177 178 /* For base bdev bigger than 1TB take reclaim uint size for grouping GC bands */ 179 if (num_blocks > (TiB / FTL_BLOCK_SIZE)) { 180 assert(reclaim_unit_num_blocks < num_blocks); 181 num_logical_in_phys = reclaim_unit_num_blocks / num_blocks_in_band; 182 } 183 184 num_to_drop = ftl_get_num_bands(dev) % num_logical_in_phys; 185 186 i = 0; 187 while (i < ftl_get_num_bands(dev) - num_to_drop) { 188 band = &dev->bands[i]; 189 190 band->phys_id = phys_id; 191 i++; 192 if (i % num_logical_in_phys == 0) { 193 phys_id++; 194 } 195 } 196 197 /* Mark not aligned logical bands as broken */ 198 num_bands = ftl_get_num_bands(dev); 199 while (i < num_bands) { 200 band = &dev->bands[i]; 201 dev->num_bands--; 202 TAILQ_REMOVE(&dev->shut_bands, band, queue_entry); 203 i++; 204 } 205 206 dev->num_logical_bands_in_physical = num_logical_in_phys; 207 } 208 209 void 210 ftl_mngt_decorate_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 211 { 212 decorate_bands(dev); 213 ftl_mngt_next_step(mngt); 214 } 215 216 void 217 ftl_mngt_initialize_band_address(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 218 { 219 struct ftl_band *band; 220 struct ftl_md *data_md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_DATA_BASE]; 221 uint64_t i; 222 223 for (i = 0; i < ftl_get_num_bands(dev); i++) { 224 band = &dev->bands[i]; 225 band->start_addr = data_md->region->current.offset + i * dev->num_blocks_in_band; 226 band->tail_md_addr = ftl_band_tail_md_addr(band); 227 } 228 229 ftl_mngt_next_step(mngt); 230 } 231 232 void 233 ftl_recover_max_seq(struct spdk_ftl_dev *dev) 234 { 235 struct ftl_band *band; 236 uint64_t band_close_seq_id = 0, band_open_seq_id = 0; 237 uint64_t chunk_close_seq_id = 0, chunk_open_seq_id = 0; 238 uint64_t max = 0; 239 240 TAILQ_FOREACH(band, &dev->shut_bands, queue_entry) { 241 band_open_seq_id = spdk_max(band_open_seq_id, band->md->seq); 242 band_close_seq_id = spdk_max(band_close_seq_id, band->md->close_seq_id); 243 } 244 ftl_nv_cache_get_max_seq_id(&dev->nv_cache, &chunk_open_seq_id, &chunk_close_seq_id); 245 246 247 dev->nv_cache.last_seq_id = chunk_close_seq_id; 248 dev->writer_gc.last_seq_id = band_close_seq_id; 249 dev->writer_user.last_seq_id = band_close_seq_id; 250 251 max = spdk_max(max, band_open_seq_id); 252 max = spdk_max(max, band_close_seq_id); 253 max = spdk_max(max, chunk_open_seq_id); 254 max = spdk_max(max, chunk_close_seq_id); 255 256 dev->sb->seq_id = max; 257 } 258 259 static int 260 _band_cmp(const void *_a, const void *_b) 261 { 262 struct ftl_band *a, *b; 263 264 a = *((struct ftl_band **)_a); 265 b = *((struct ftl_band **)_b); 266 267 return a->md->seq - b->md->seq; 268 } 269 270 static struct ftl_band * 271 next_high_prio_band(struct spdk_ftl_dev *dev) 272 { 273 struct ftl_band *result = NULL, *band; 274 uint64_t validity = UINT64_MAX; 275 276 TAILQ_FOREACH(band, &dev->shut_bands, queue_entry) { 277 if (band->p2l_map.num_valid < validity) { 278 result = band; 279 validity = result->p2l_map.num_valid; 280 } 281 } 282 283 return result; 284 } 285 286 static int 287 finalize_init_gc(struct spdk_ftl_dev *dev) 288 { 289 struct ftl_band *band; 290 uint64_t free_blocks, blocks_to_move; 291 292 ftl_band_init_gc_iter(dev); 293 dev->sb_shm->gc_info.band_id_high_prio = FTL_BAND_ID_INVALID; 294 295 if (0 == dev->num_free) { 296 /* Get number of available blocks in writer */ 297 free_blocks = ftl_writer_get_free_blocks(&dev->writer_gc); 298 299 /* 300 * First, check a band candidate to GC 301 */ 302 band = ftl_band_search_next_to_reloc(dev); 303 ftl_bug(NULL == band); 304 blocks_to_move = band->p2l_map.num_valid; 305 if (blocks_to_move <= free_blocks) { 306 /* This GC band can be moved */ 307 return 0; 308 } 309 310 /* 311 * The GC candidate cannot be moved because no enough space. We need to find 312 * another band. 313 */ 314 band = next_high_prio_band(dev); 315 ftl_bug(NULL == band); 316 317 if (band->p2l_map.num_valid > free_blocks) { 318 FTL_ERRLOG(dev, "CRITICAL ERROR, no more free bands and cannot start\n"); 319 return -1; 320 } else { 321 /* GC needs to start using this band */ 322 dev->sb_shm->gc_info.band_id_high_prio = band->id; 323 } 324 } 325 326 return 0; 327 } 328 329 void 330 ftl_mngt_finalize_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 331 { 332 struct ftl_band *band, *temp_band, *open_bands[FTL_MAX_OPEN_BANDS]; 333 struct ftl_writer *writer = NULL; 334 uint64_t i, num_open = 0, num_shut = 0; 335 uint64_t offset; 336 bool fast_startup = ftl_fast_startup(dev); 337 338 ftl_recover_max_seq(dev); 339 340 TAILQ_FOREACH_SAFE(band, &dev->free_bands, queue_entry, temp_band) { 341 band->md->df_p2l_map = FTL_DF_OBJ_ID_INVALID; 342 } 343 344 TAILQ_FOREACH_SAFE(band, &dev->shut_bands, queue_entry, temp_band) { 345 if (band->md->state == FTL_BAND_STATE_OPEN || 346 band->md->state == FTL_BAND_STATE_FULL) { 347 TAILQ_REMOVE(&dev->shut_bands, band, queue_entry); 348 open_bands[num_open++] = band; 349 assert(num_open <= FTL_MAX_OPEN_BANDS); 350 continue; 351 } 352 353 if (dev->conf.mode & SPDK_FTL_MODE_CREATE) { 354 TAILQ_REMOVE(&dev->shut_bands, band, queue_entry); 355 assert(band->md->state == FTL_BAND_STATE_FREE); 356 band->md->state = FTL_BAND_STATE_CLOSED; 357 ftl_band_set_state(band, FTL_BAND_STATE_FREE); 358 } else { 359 num_shut++; 360 } 361 362 band->md->df_p2l_map = FTL_DF_OBJ_ID_INVALID; 363 } 364 365 /* Assign open bands to writers and alloc necessary resources */ 366 qsort(open_bands, num_open, sizeof(open_bands[0]), _band_cmp); 367 368 for (i = 0; i < num_open; ++i) { 369 band = open_bands[i]; 370 371 if (band->md->type == FTL_BAND_TYPE_COMPACTION) { 372 writer = &dev->writer_user; 373 } else if (band->md->type == FTL_BAND_TYPE_GC) { 374 writer = &dev->writer_gc; 375 } else { 376 assert(false); 377 } 378 379 if (band->md->state == FTL_BAND_STATE_FULL) { 380 TAILQ_INSERT_TAIL(&writer->full_bands, band, queue_entry); 381 } else { 382 if (writer->band == NULL) { 383 writer->band = band; 384 } else { 385 writer->next_band = band; 386 } 387 } 388 389 writer->num_bands++; 390 ftl_band_set_owner(band, ftl_writer_band_state_change, writer); 391 392 if (fast_startup) { 393 FTL_NOTICELOG(dev, "SHM: band open P2L map df_id 0x%"PRIx64"\n", band->md->df_p2l_map); 394 if (ftl_band_open_p2l_map(band)) { 395 ftl_mngt_fail_step(mngt); 396 return; 397 } 398 399 offset = band->md->iter.offset; 400 ftl_band_iter_init(band); 401 ftl_band_iter_set(band, offset); 402 ftl_mngt_p2l_ckpt_restore_shm_clean(band); 403 } else if (dev->sb->clean) { 404 band->md->df_p2l_map = FTL_DF_OBJ_ID_INVALID; 405 if (ftl_band_alloc_p2l_map(band)) { 406 ftl_mngt_fail_step(mngt); 407 return; 408 } 409 410 offset = band->md->iter.offset; 411 ftl_band_iter_init(band); 412 ftl_band_iter_set(band, offset); 413 414 if (ftl_mngt_p2l_ckpt_restore_clean(band)) { 415 ftl_mngt_fail_step(mngt); 416 return; 417 } 418 } 419 } 420 421 if (fast_startup) { 422 ftl_mempool_initialize_ext(dev->p2l_pool); 423 } 424 425 426 /* Recalculate number of free bands */ 427 dev->num_free = 0; 428 TAILQ_FOREACH(band, &dev->free_bands, queue_entry) { 429 assert(band->md->state == FTL_BAND_STATE_FREE); 430 dev->num_free++; 431 } 432 ftl_apply_limits(dev); 433 434 if ((num_shut + num_open + dev->num_free) != ftl_get_num_bands(dev)) { 435 FTL_ERRLOG(dev, "ERROR, band list inconsistent state\n"); 436 ftl_mngt_fail_step(mngt); 437 return; 438 } 439 440 if (finalize_init_gc(dev)) { 441 ftl_mngt_fail_step(mngt); 442 } else { 443 ftl_mngt_next_step(mngt); 444 } 445 } 446