1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "ftl_core.h" 7 #include "ftl_mngt_steps.h" 8 #include "ftl_band.h" 9 #include "ftl_internal.h" 10 11 static int 12 ftl_band_init_md(struct ftl_band *band) 13 { 14 struct spdk_ftl_dev *dev = band->dev; 15 struct ftl_p2l_map *p2l_map = &band->p2l_map; 16 struct ftl_md *band_info_md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_BAND_MD]; 17 struct ftl_md *valid_map_md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_VALID_MAP]; 18 uint64_t band_num_blocks = ftl_get_num_blocks_in_band(band->dev); 19 size_t band_valid_map_bytes; 20 struct ftl_band_md *band_md = ftl_md_get_buffer(band_info_md); 21 22 if (band_num_blocks % (ftl_bitmap_buffer_alignment * 8)) { 23 FTL_ERRLOG(dev, "The number of blocks in band is not divisible by bitmap word bits\n"); 24 return -EINVAL; 25 } 26 band_valid_map_bytes = band_num_blocks / 8; 27 28 p2l_map->valid = ftl_bitmap_create(ftl_md_get_buffer(valid_map_md) + 29 band->start_addr / 8, band_valid_map_bytes); 30 if (!p2l_map->valid) { 31 return -ENOMEM; 32 } 33 34 band->md = &band_md[band->id]; 35 if (!ftl_fast_startup(dev)) { 36 band->md->df_p2l_map = FTL_DF_OBJ_ID_INVALID; 37 } 38 39 return 0; 40 } 41 42 static int 43 ftl_dev_init_bands(struct spdk_ftl_dev *dev) 44 { 45 struct ftl_band *band; 46 uint64_t i; 47 48 TAILQ_INIT(&dev->free_bands); 49 TAILQ_INIT(&dev->shut_bands); 50 51 dev->num_free = 0; 52 dev->bands = calloc(ftl_get_num_bands(dev), sizeof(*dev->bands)); 53 if (!dev->bands) { 54 return -ENOMEM; 55 } 56 57 for (i = 0; i < ftl_get_num_bands(dev); ++i) { 58 band = &dev->bands[i]; 59 band->id = i; 60 band->dev = dev; 61 62 /* Adding to shut_bands is necessary - see ftl_restore_band_close_cb() */ 63 TAILQ_INSERT_TAIL(&dev->shut_bands, band, queue_entry); 64 } 65 66 return 0; 67 } 68 69 static int 70 ftl_dev_init_bands_md(struct spdk_ftl_dev *dev) 71 { 72 uint64_t i; 73 int rc = 0; 74 75 for (i = 0; i < ftl_get_num_bands(dev); ++i) { 76 rc = ftl_band_init_md(&dev->bands[i]); 77 if (rc) { 78 FTL_ERRLOG(dev, "Failed to initialize metadata structures for band [%lu]\n", i); 79 break; 80 } 81 } 82 83 return rc; 84 } 85 86 static void 87 ftl_dev_deinit_bands(struct spdk_ftl_dev *dev) 88 { 89 free(dev->bands); 90 } 91 92 static void 93 ftl_dev_deinit_bands_md(struct spdk_ftl_dev *dev) 94 { 95 if (dev->bands) { 96 uint64_t i; 97 for (i = 0; i < dev->num_bands; ++i) { 98 struct ftl_band *band = &dev->bands[i]; 99 100 ftl_bitmap_destroy(band->p2l_map.valid); 101 band->p2l_map.valid = NULL; 102 103 band->md = NULL; 104 } 105 } 106 } 107 108 void 109 ftl_mngt_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 110 { 111 if (ftl_dev_init_bands(dev)) { 112 ftl_mngt_fail_step(mngt); 113 } else { 114 ftl_mngt_next_step(mngt); 115 } 116 } 117 118 void 119 ftl_mngt_init_bands_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 120 { 121 if (ftl_dev_init_bands_md(dev)) { 122 ftl_mngt_fail_step(mngt); 123 } else { 124 ftl_mngt_next_step(mngt); 125 } 126 } 127 128 void 129 ftl_mngt_deinit_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 130 { 131 ftl_dev_deinit_bands(dev); 132 ftl_mngt_next_step(mngt); 133 } 134 135 void 136 ftl_mngt_deinit_bands_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 137 { 138 ftl_dev_deinit_bands_md(dev); 139 ftl_mngt_next_step(mngt); 140 } 141 142 /* 143 * For grouping multiple logical bands (1GiB) to make any IOs more sequential from the drive's 144 * perspective. Improves WAF. 145 */ 146 #define BASE_BDEV_RECLAIM_UNIT_SIZE (72 * GiB) 147 148 static void 149 decorate_bands(struct spdk_ftl_dev *dev) 150 { 151 struct ftl_band *band; 152 uint64_t i, num_to_drop, phys_id = 0; 153 uint64_t num_blocks, num_bands; 154 uint64_t num_blocks_in_band = ftl_get_num_blocks_in_band(dev); 155 uint64_t reclaim_unit_num_blocks = BASE_BDEV_RECLAIM_UNIT_SIZE / FTL_BLOCK_SIZE; 156 uint32_t num_logical_in_phys = 2; 157 158 assert(reclaim_unit_num_blocks % num_blocks_in_band == 0); 159 160 num_blocks = spdk_bdev_get_num_blocks(spdk_bdev_desc_get_bdev(dev->base_bdev_desc)); 161 162 /* For base bdev bigger than 1TB take reclaim uint size for grouping GC bands */ 163 if (num_blocks > (TiB / FTL_BLOCK_SIZE)) { 164 assert(reclaim_unit_num_blocks < num_blocks); 165 num_logical_in_phys = reclaim_unit_num_blocks / num_blocks_in_band; 166 } 167 168 num_to_drop = ftl_get_num_bands(dev) % num_logical_in_phys; 169 170 i = 0; 171 while (i < ftl_get_num_bands(dev) - num_to_drop) { 172 band = &dev->bands[i]; 173 174 band->phys_id = phys_id; 175 i++; 176 if (i % num_logical_in_phys == 0) { 177 phys_id++; 178 } 179 } 180 181 /* Mark not aligned logical bands as broken */ 182 num_bands = ftl_get_num_bands(dev); 183 while (i < num_bands) { 184 band = &dev->bands[i]; 185 dev->num_bands--; 186 TAILQ_REMOVE(&dev->shut_bands, band, queue_entry); 187 i++; 188 } 189 190 dev->num_logical_bands_in_physical = num_logical_in_phys; 191 } 192 193 void 194 ftl_mngt_decorate_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 195 { 196 decorate_bands(dev); 197 ftl_mngt_next_step(mngt); 198 } 199 200 void 201 ftl_mngt_initialize_band_address(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 202 { 203 struct ftl_band *band; 204 struct ftl_md *data_md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_DATA_BASE]; 205 uint64_t i; 206 207 for (i = 0; i < ftl_get_num_bands(dev); i++) { 208 band = &dev->bands[i]; 209 band->start_addr = data_md->region->current.offset + i * dev->num_blocks_in_band; 210 band->tail_md_addr = ftl_band_tail_md_addr(band); 211 } 212 213 ftl_mngt_next_step(mngt); 214 } 215 216 void 217 ftl_recover_max_seq(struct spdk_ftl_dev *dev) 218 { 219 struct ftl_band *band; 220 size_t band_close_seq_id = 0, band_open_seq_id = 0; 221 size_t chunk_close_seq_id = 0, chunk_open_seq_id = 0; 222 size_t max = 0; 223 224 TAILQ_FOREACH(band, &dev->shut_bands, queue_entry) { 225 band_open_seq_id = spdk_max(band_open_seq_id, band->md->seq); 226 band_close_seq_id = spdk_max(band_close_seq_id, band->md->close_seq_id); 227 } 228 ftl_nv_cache_get_max_seq_id(&dev->nv_cache, &chunk_open_seq_id, &chunk_close_seq_id); 229 230 231 dev->nv_cache.last_seq_id = chunk_close_seq_id; 232 dev->writer_gc.last_seq_id = band_close_seq_id; 233 dev->writer_user.last_seq_id = band_close_seq_id; 234 235 max = spdk_max(max, band_open_seq_id); 236 max = spdk_max(max, band_close_seq_id); 237 max = spdk_max(max, chunk_open_seq_id); 238 max = spdk_max(max, chunk_close_seq_id); 239 240 dev->sb->seq_id = max; 241 } 242 243 static int 244 _band_cmp(const void *_a, const void *_b) 245 { 246 struct ftl_band *a, *b; 247 248 a = *((struct ftl_band **)_a); 249 b = *((struct ftl_band **)_b); 250 251 return a->md->seq - b->md->seq; 252 } 253 254 static struct ftl_band * 255 next_high_prio_band(struct spdk_ftl_dev *dev) 256 { 257 struct ftl_band *result = NULL, *band; 258 uint64_t validity = UINT64_MAX; 259 260 TAILQ_FOREACH(band, &dev->shut_bands, queue_entry) { 261 if (band->p2l_map.num_valid < validity) { 262 result = band; 263 validity = result->p2l_map.num_valid; 264 } 265 } 266 267 return result; 268 } 269 270 static int 271 finalize_init_gc(struct spdk_ftl_dev *dev) 272 { 273 struct ftl_band *band; 274 uint64_t free_blocks, blocks_to_move; 275 276 ftl_band_init_gc_iter(dev); 277 dev->sb_shm->gc_info.band_id_high_prio = FTL_BAND_ID_INVALID; 278 279 if (0 == dev->num_free) { 280 /* Get number of available blocks in writer */ 281 free_blocks = ftl_writer_get_free_blocks(&dev->writer_gc); 282 283 /* 284 * First, check a band candidate to GC 285 */ 286 band = ftl_band_search_next_to_reloc(dev); 287 ftl_bug(NULL == band); 288 blocks_to_move = band->p2l_map.num_valid; 289 if (blocks_to_move <= free_blocks) { 290 /* This GC band can be moved */ 291 return 0; 292 } 293 294 /* 295 * The GC candidate cannot be moved because no enough space. We need to find 296 * another band. 297 */ 298 band = next_high_prio_band(dev); 299 ftl_bug(NULL == band); 300 301 if (band->p2l_map.num_valid > free_blocks) { 302 FTL_ERRLOG(dev, "CRITICAL ERROR, no more free bands and cannot start\n"); 303 return -1; 304 } else { 305 /* GC needs to start using this band */ 306 dev->sb_shm->gc_info.band_id_high_prio = band->id; 307 } 308 } 309 310 return 0; 311 } 312 313 void 314 ftl_mngt_finalize_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 315 { 316 struct ftl_band *band, *temp_band, *open_bands[FTL_MAX_OPEN_BANDS]; 317 struct ftl_writer *writer = NULL; 318 uint64_t i, num_open = 0, num_shut = 0; 319 uint64_t offset; 320 bool fast_startup = ftl_fast_startup(dev); 321 322 ftl_recover_max_seq(dev); 323 324 TAILQ_FOREACH_SAFE(band, &dev->free_bands, queue_entry, temp_band) { 325 band->md->df_p2l_map = FTL_DF_OBJ_ID_INVALID; 326 } 327 328 TAILQ_FOREACH_SAFE(band, &dev->shut_bands, queue_entry, temp_band) { 329 if (band->md->state == FTL_BAND_STATE_OPEN || 330 band->md->state == FTL_BAND_STATE_FULL) { 331 TAILQ_REMOVE(&dev->shut_bands, band, queue_entry); 332 open_bands[num_open++] = band; 333 assert(num_open <= FTL_MAX_OPEN_BANDS); 334 continue; 335 } 336 337 if (dev->conf.mode & SPDK_FTL_MODE_CREATE) { 338 TAILQ_REMOVE(&dev->shut_bands, band, queue_entry); 339 assert(band->md->state == FTL_BAND_STATE_FREE); 340 band->md->state = FTL_BAND_STATE_CLOSED; 341 ftl_band_set_state(band, FTL_BAND_STATE_FREE); 342 } else { 343 num_shut++; 344 } 345 346 band->md->df_p2l_map = FTL_DF_OBJ_ID_INVALID; 347 } 348 349 /* Assign open bands to writers and alloc necessary resources */ 350 qsort(open_bands, num_open, sizeof(open_bands[0]), _band_cmp); 351 352 for (i = 0; i < num_open; ++i) { 353 band = open_bands[i]; 354 355 if (band->md->type == FTL_BAND_TYPE_COMPACTION) { 356 writer = &dev->writer_user; 357 } else if (band->md->type == FTL_BAND_TYPE_GC) { 358 writer = &dev->writer_gc; 359 } else { 360 assert(false); 361 } 362 363 if (band->md->state == FTL_BAND_STATE_FULL) { 364 TAILQ_INSERT_TAIL(&writer->full_bands, band, queue_entry); 365 } else { 366 if (writer->band == NULL) { 367 writer->band = band; 368 } else { 369 writer->next_band = band; 370 } 371 } 372 373 writer->num_bands++; 374 ftl_band_set_owner(band, ftl_writer_band_state_change, writer); 375 376 if (fast_startup) { 377 FTL_NOTICELOG(dev, "SHM: band open P2L map df_id 0x%"PRIx64"\n", band->md->df_p2l_map); 378 if (ftl_band_open_p2l_map(band)) { 379 ftl_mngt_fail_step(mngt); 380 return; 381 } 382 383 offset = band->md->iter.offset; 384 ftl_band_iter_init(band); 385 ftl_band_iter_set(band, offset); 386 ftl_mngt_p2l_ckpt_restore_shm_clean(band); 387 } else if (dev->sb->clean) { 388 band->md->df_p2l_map = FTL_DF_OBJ_ID_INVALID; 389 if (ftl_band_alloc_p2l_map(band)) { 390 ftl_mngt_fail_step(mngt); 391 return; 392 } 393 394 offset = band->md->iter.offset; 395 ftl_band_iter_init(band); 396 ftl_band_iter_set(band, offset); 397 398 if (ftl_mngt_p2l_ckpt_restore_clean(band)) { 399 ftl_mngt_fail_step(mngt); 400 return; 401 } 402 } 403 } 404 405 if (fast_startup) { 406 ftl_mempool_initialize_ext(dev->p2l_pool); 407 } 408 409 410 /* Recalculate number of free bands */ 411 dev->num_free = 0; 412 TAILQ_FOREACH(band, &dev->free_bands, queue_entry) { 413 assert(band->md->state == FTL_BAND_STATE_FREE); 414 dev->num_free++; 415 } 416 ftl_apply_limits(dev); 417 418 if ((num_shut + num_open + dev->num_free) != ftl_get_num_bands(dev)) { 419 FTL_ERRLOG(dev, "ERROR, band list inconsistent state\n"); 420 ftl_mngt_fail_step(mngt); 421 return; 422 } 423 424 if (finalize_init_gc(dev)) { 425 ftl_mngt_fail_step(mngt); 426 } else { 427 ftl_mngt_next_step(mngt); 428 } 429 } 430