1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "ftl_core.h" 7 #include "ftl_mngt_steps.h" 8 #include "ftl_band.h" 9 #include "ftl_internal.h" 10 11 static int 12 ftl_band_init_md(struct ftl_band *band) 13 { 14 struct spdk_ftl_dev *dev = band->dev; 15 struct ftl_p2l_map *p2l_map = &band->p2l_map; 16 struct ftl_md *band_info_md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_BAND_MD]; 17 struct ftl_md *valid_map_md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_VALID_MAP]; 18 uint64_t band_num_blocks = ftl_get_num_blocks_in_band(band->dev); 19 size_t band_valid_map_bytes; 20 struct ftl_band_md *band_md = ftl_md_get_buffer(band_info_md); 21 22 if (band_num_blocks % (ftl_bitmap_buffer_alignment * 8)) { 23 FTL_ERRLOG(dev, "The number of blocks in band is not divisible by bitmap word bits\n"); 24 return -EINVAL; 25 } 26 band_valid_map_bytes = band_num_blocks / 8; 27 28 p2l_map->valid = ftl_bitmap_create(ftl_md_get_buffer(valid_map_md) + 29 band->start_addr / 8, band_valid_map_bytes); 30 if (!p2l_map->valid) { 31 return -ENOMEM; 32 } 33 34 band->md = &band_md[band->id]; 35 if (!ftl_fast_startup(dev)) { 36 band->md->df_p2l_map = FTL_DF_OBJ_ID_INVALID; 37 } 38 39 return 0; 40 } 41 42 static int 43 ftl_dev_init_bands(struct spdk_ftl_dev *dev) 44 { 45 struct ftl_band *band; 46 uint64_t i, blocks, md_blocks, md_bands; 47 48 /* Calculate initial number of bands */ 49 blocks = spdk_bdev_get_num_blocks(spdk_bdev_desc_get_bdev(dev->base_bdev_desc)); 50 dev->num_bands = blocks / ftl_get_num_blocks_in_band(dev); 51 52 /* Calculate number of bands considering base device metadata size requirement */ 53 md_blocks = ftl_layout_base_md_blocks(dev); 54 md_bands = spdk_divide_round_up(md_blocks, dev->num_blocks_in_band); 55 56 if (dev->num_bands > md_bands) { 57 /* Save a band worth of space for metadata */ 58 dev->num_bands -= md_bands; 59 } else { 60 FTL_ERRLOG(dev, "Base device too small to store metadata\n"); 61 return -1; 62 } 63 64 TAILQ_INIT(&dev->free_bands); 65 TAILQ_INIT(&dev->shut_bands); 66 67 dev->num_free = 0; 68 dev->bands = calloc(ftl_get_num_bands(dev), sizeof(*dev->bands)); 69 if (!dev->bands) { 70 return -ENOMEM; 71 } 72 73 for (i = 0; i < ftl_get_num_bands(dev); ++i) { 74 band = &dev->bands[i]; 75 band->id = i; 76 band->dev = dev; 77 78 /* Adding to shut_bands is necessary - see ftl_restore_band_close_cb() */ 79 TAILQ_INSERT_TAIL(&dev->shut_bands, band, queue_entry); 80 } 81 82 return 0; 83 } 84 85 static int 86 ftl_dev_init_bands_md(struct spdk_ftl_dev *dev) 87 { 88 uint64_t i; 89 int rc = 0; 90 91 for (i = 0; i < ftl_get_num_bands(dev); ++i) { 92 rc = ftl_band_init_md(&dev->bands[i]); 93 if (rc) { 94 FTL_ERRLOG(dev, "Failed to initialize metadata structures for band [%lu]\n", i); 95 break; 96 } 97 } 98 99 return rc; 100 } 101 102 static void 103 ftl_dev_deinit_bands(struct spdk_ftl_dev *dev) 104 { 105 free(dev->bands); 106 } 107 108 static void 109 ftl_dev_deinit_bands_md(struct spdk_ftl_dev *dev) 110 { 111 if (dev->bands) { 112 uint64_t i; 113 for (i = 0; i < dev->num_bands; ++i) { 114 struct ftl_band *band = &dev->bands[i]; 115 116 ftl_bitmap_destroy(band->p2l_map.valid); 117 band->p2l_map.valid = NULL; 118 119 band->md = NULL; 120 } 121 } 122 } 123 124 void 125 ftl_mngt_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 126 { 127 if (ftl_dev_init_bands(dev)) { 128 ftl_mngt_fail_step(mngt); 129 } else { 130 ftl_mngt_next_step(mngt); 131 } 132 } 133 134 void 135 ftl_mngt_init_bands_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 136 { 137 if (ftl_dev_init_bands_md(dev)) { 138 ftl_mngt_fail_step(mngt); 139 } else { 140 ftl_mngt_next_step(mngt); 141 } 142 } 143 144 void 145 ftl_mngt_deinit_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 146 { 147 ftl_dev_deinit_bands(dev); 148 ftl_mngt_next_step(mngt); 149 } 150 151 void 152 ftl_mngt_deinit_bands_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 153 { 154 ftl_dev_deinit_bands_md(dev); 155 ftl_mngt_next_step(mngt); 156 } 157 158 /* 159 * For grouping multiple logical bands (1GiB) to make any IOs more sequential from the drive's 160 * perspective. Improves WAF. 161 */ 162 #define BASE_BDEV_RECLAIM_UNIT_SIZE (72 * GiB) 163 164 static void 165 decorate_bands(struct spdk_ftl_dev *dev) 166 { 167 struct ftl_band *band; 168 uint64_t i, num_to_drop, phys_id = 0; 169 uint64_t num_blocks, num_bands; 170 uint64_t num_blocks_in_band = ftl_get_num_blocks_in_band(dev); 171 uint64_t reclaim_unit_num_blocks = BASE_BDEV_RECLAIM_UNIT_SIZE / FTL_BLOCK_SIZE; 172 uint32_t num_logical_in_phys = 2; 173 174 assert(reclaim_unit_num_blocks % num_blocks_in_band == 0); 175 176 num_blocks = spdk_bdev_get_num_blocks(spdk_bdev_desc_get_bdev(dev->base_bdev_desc)); 177 178 /* For base bdev bigger than 1TB take reclaim uint size for grouping GC bands */ 179 if (num_blocks > (TiB / FTL_BLOCK_SIZE)) { 180 assert(reclaim_unit_num_blocks < num_blocks); 181 num_logical_in_phys = reclaim_unit_num_blocks / num_blocks_in_band; 182 } 183 184 num_to_drop = ftl_get_num_bands(dev) % num_logical_in_phys; 185 186 i = 0; 187 while (i < ftl_get_num_bands(dev) - num_to_drop) { 188 band = &dev->bands[i]; 189 190 band->phys_id = phys_id; 191 i++; 192 if (i % num_logical_in_phys == 0) { 193 phys_id++; 194 } 195 } 196 197 /* Mark not aligned logical bands as broken */ 198 num_bands = ftl_get_num_bands(dev); 199 while (i < num_bands) { 200 band = &dev->bands[i]; 201 dev->num_bands--; 202 TAILQ_REMOVE(&dev->shut_bands, band, queue_entry); 203 i++; 204 } 205 206 dev->num_logical_bands_in_physical = num_logical_in_phys; 207 } 208 209 void 210 ftl_mngt_decorate_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 211 { 212 decorate_bands(dev); 213 ftl_mngt_next_step(mngt); 214 } 215 216 void 217 ftl_mngt_initialize_band_address(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 218 { 219 struct ftl_band *band; 220 struct ftl_md *data_md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_DATA_BASE]; 221 uint64_t i; 222 223 for (i = 0; i < ftl_get_num_bands(dev); i++) { 224 band = &dev->bands[i]; 225 band->start_addr = data_md->region->current.offset + i * dev->num_blocks_in_band; 226 band->tail_md_addr = ftl_band_tail_md_addr(band); 227 } 228 229 ftl_mngt_next_step(mngt); 230 } 231 232 void 233 ftl_recover_max_seq(struct spdk_ftl_dev *dev) 234 { 235 struct ftl_band *band; 236 uint64_t band_close_seq_id = 0, band_open_seq_id = 0; 237 uint64_t chunk_close_seq_id = 0, chunk_open_seq_id = 0; 238 uint64_t max = 0; 239 240 TAILQ_FOREACH(band, &dev->shut_bands, queue_entry) { 241 band_open_seq_id = spdk_max(band_open_seq_id, band->md->seq); 242 band_close_seq_id = spdk_max(band_close_seq_id, band->md->close_seq_id); 243 } 244 ftl_nv_cache_get_max_seq_id(&dev->nv_cache, &chunk_open_seq_id, &chunk_close_seq_id); 245 246 247 dev->nv_cache.last_seq_id = chunk_close_seq_id; 248 dev->writer_gc.last_seq_id = band_close_seq_id; 249 dev->writer_user.last_seq_id = band_close_seq_id; 250 251 max = spdk_max(max, band_open_seq_id); 252 max = spdk_max(max, band_close_seq_id); 253 max = spdk_max(max, chunk_open_seq_id); 254 max = spdk_max(max, chunk_close_seq_id); 255 256 dev->sb->seq_id = max; 257 } 258 259 static int 260 _band_cmp(const void *_a, const void *_b) 261 { 262 struct ftl_band *a, *b; 263 264 a = *((struct ftl_band **)_a); 265 b = *((struct ftl_band **)_b); 266 267 return a->md->seq - b->md->seq; 268 } 269 270 static struct ftl_band * 271 next_high_prio_band(struct spdk_ftl_dev *dev) 272 { 273 struct ftl_band *result = NULL, *band; 274 uint64_t validity = UINT64_MAX; 275 276 TAILQ_FOREACH(band, &dev->shut_bands, queue_entry) { 277 if (band->p2l_map.num_valid < validity) { 278 result = band; 279 validity = result->p2l_map.num_valid; 280 } 281 } 282 283 return result; 284 } 285 286 static int 287 finalize_init_gc(struct spdk_ftl_dev *dev) 288 { 289 struct ftl_band *band; 290 uint64_t free_blocks, blocks_to_move; 291 292 ftl_band_init_gc_iter(dev); 293 dev->sb_shm->gc_info.band_id_high_prio = FTL_BAND_ID_INVALID; 294 295 if (0 == dev->num_free) { 296 /* Get number of available blocks in writer */ 297 free_blocks = ftl_writer_get_free_blocks(&dev->writer_gc); 298 299 /* 300 * First, check a band candidate to GC 301 */ 302 band = ftl_band_search_next_to_reloc(dev); 303 ftl_bug(NULL == band); 304 blocks_to_move = band->p2l_map.num_valid; 305 if (blocks_to_move <= free_blocks) { 306 /* This GC band can be moved */ 307 return 0; 308 } 309 310 /* 311 * The GC candidate cannot be moved because no enough space. We need to find 312 * another band. 313 */ 314 band = next_high_prio_band(dev); 315 ftl_bug(NULL == band); 316 317 if (band->p2l_map.num_valid > free_blocks) { 318 FTL_ERRLOG(dev, "CRITICAL ERROR, no more free bands and cannot start\n"); 319 return -1; 320 } else { 321 /* GC needs to start using this band */ 322 dev->sb_shm->gc_info.band_id_high_prio = band->id; 323 } 324 } 325 326 return 0; 327 } 328 329 static void 330 ftl_property_dump_base_dev(struct spdk_ftl_dev *dev, const struct ftl_property *property, 331 struct spdk_json_write_ctx *w) 332 { 333 uint64_t i; 334 struct ftl_band *band; 335 336 spdk_json_write_named_array_begin(w, "bands"); 337 for (i = 0, band = dev->bands; i < ftl_get_num_bands(dev); i++, band++) { 338 spdk_json_write_object_begin(w); 339 spdk_json_write_named_uint64(w, "id", i); 340 spdk_json_write_named_string(w, "state", ftl_band_get_state_name(band)); 341 spdk_json_write_named_double(w, "validity", 1.0 - ftl_band_invalidity(band)); 342 spdk_json_write_object_end(w); 343 } 344 spdk_json_write_array_end(w); 345 } 346 347 void 348 ftl_mngt_finalize_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 349 { 350 struct ftl_band *band, *temp_band, *open_bands[FTL_MAX_OPEN_BANDS]; 351 struct ftl_writer *writer = NULL; 352 uint64_t i, num_open = 0, num_shut = 0; 353 uint64_t offset; 354 bool fast_startup = ftl_fast_startup(dev); 355 356 ftl_recover_max_seq(dev); 357 ftl_property_register(dev, "base_device", NULL, 0, NULL, NULL, ftl_property_dump_base_dev, NULL, 358 NULL, true); 359 360 TAILQ_FOREACH_SAFE(band, &dev->free_bands, queue_entry, temp_band) { 361 band->md->df_p2l_map = FTL_DF_OBJ_ID_INVALID; 362 } 363 364 TAILQ_FOREACH_SAFE(band, &dev->shut_bands, queue_entry, temp_band) { 365 if (band->md->state == FTL_BAND_STATE_OPEN || 366 band->md->state == FTL_BAND_STATE_FULL) { 367 TAILQ_REMOVE(&dev->shut_bands, band, queue_entry); 368 open_bands[num_open++] = band; 369 assert(num_open <= FTL_MAX_OPEN_BANDS); 370 continue; 371 } 372 373 if (dev->conf.mode & SPDK_FTL_MODE_CREATE) { 374 TAILQ_REMOVE(&dev->shut_bands, band, queue_entry); 375 assert(band->md->state == FTL_BAND_STATE_FREE); 376 band->md->state = FTL_BAND_STATE_CLOSED; 377 ftl_band_set_state(band, FTL_BAND_STATE_FREE); 378 } else { 379 num_shut++; 380 } 381 382 band->md->df_p2l_map = FTL_DF_OBJ_ID_INVALID; 383 } 384 385 /* Assign open bands to writers and alloc necessary resources */ 386 qsort(open_bands, num_open, sizeof(open_bands[0]), _band_cmp); 387 388 for (i = 0; i < num_open; ++i) { 389 band = open_bands[i]; 390 391 if (band->md->type == FTL_BAND_TYPE_COMPACTION) { 392 writer = &dev->writer_user; 393 } else if (band->md->type == FTL_BAND_TYPE_GC) { 394 writer = &dev->writer_gc; 395 } else { 396 assert(false); 397 } 398 399 if (band->md->state == FTL_BAND_STATE_FULL) { 400 TAILQ_INSERT_TAIL(&writer->full_bands, band, queue_entry); 401 } else { 402 if (writer->band == NULL) { 403 writer->band = band; 404 } else { 405 writer->next_band = band; 406 } 407 } 408 409 writer->num_bands++; 410 ftl_band_set_owner(band, ftl_writer_band_state_change, writer); 411 412 if (fast_startup) { 413 FTL_NOTICELOG(dev, "SHM: band open P2L map df_id 0x%"PRIx64"\n", band->md->df_p2l_map); 414 if (ftl_band_open_p2l_map(band)) { 415 ftl_mngt_fail_step(mngt); 416 return; 417 } 418 419 offset = band->md->iter.offset; 420 ftl_band_iter_init(band); 421 ftl_band_iter_set(band, offset); 422 ftl_mngt_p2l_ckpt_restore_shm_clean(band); 423 } else if (dev->sb->clean) { 424 band->md->df_p2l_map = FTL_DF_OBJ_ID_INVALID; 425 if (ftl_band_alloc_p2l_map(band)) { 426 ftl_mngt_fail_step(mngt); 427 return; 428 } 429 430 offset = band->md->iter.offset; 431 ftl_band_iter_init(band); 432 ftl_band_iter_set(band, offset); 433 434 if (ftl_mngt_p2l_ckpt_restore_clean(band)) { 435 ftl_mngt_fail_step(mngt); 436 return; 437 } 438 } 439 } 440 441 if (fast_startup) { 442 ftl_mempool_initialize_ext(dev->p2l_pool); 443 } 444 445 446 /* Recalculate number of free bands */ 447 dev->num_free = 0; 448 TAILQ_FOREACH(band, &dev->free_bands, queue_entry) { 449 assert(band->md->state == FTL_BAND_STATE_FREE); 450 dev->num_free++; 451 } 452 ftl_apply_limits(dev); 453 454 if ((num_shut + num_open + dev->num_free) != ftl_get_num_bands(dev)) { 455 FTL_ERRLOG(dev, "ERROR, band list inconsistent state\n"); 456 ftl_mngt_fail_step(mngt); 457 return; 458 } 459 460 if (finalize_init_gc(dev)) { 461 ftl_mngt_fail_step(mngt); 462 } else { 463 ftl_mngt_next_step(mngt); 464 } 465 } 466