1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "ftl_core.h" 7 #include "ftl_mngt_steps.h" 8 #include "ftl_band.h" 9 #include "ftl_internal.h" 10 11 static int 12 ftl_band_init_md(struct ftl_band *band) 13 { 14 struct spdk_ftl_dev *dev = band->dev; 15 struct ftl_p2l_map *p2l_map = &band->p2l_map; 16 struct ftl_md *band_info_md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_BAND_MD]; 17 struct ftl_md *valid_map_md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_VALID_MAP]; 18 uint64_t band_num_blocks = ftl_get_num_blocks_in_band(band->dev); 19 size_t band_valid_map_bytes; 20 struct ftl_band_md *band_md = ftl_md_get_buffer(band_info_md); 21 22 if (band_num_blocks % (ftl_bitmap_buffer_alignment * 8)) { 23 FTL_ERRLOG(dev, "The number of blocks in band is not divisible by bitmap word bits\n"); 24 return -EINVAL; 25 } 26 band_valid_map_bytes = band_num_blocks / 8; 27 28 p2l_map->valid = ftl_bitmap_create(ftl_md_get_buffer(valid_map_md) + 29 band_valid_map_bytes * band->id, band_valid_map_bytes); 30 if (!p2l_map->valid) { 31 return -ENOMEM; 32 } 33 34 band->md = &band_md[band->id]; 35 if (!ftl_fast_startup(dev)) { 36 band->md->df_p2l_map = FTL_DF_OBJ_ID_INVALID; 37 } 38 39 return 0; 40 } 41 42 static int 43 ftl_dev_init_bands(struct spdk_ftl_dev *dev) 44 { 45 struct ftl_band *band; 46 uint64_t i; 47 48 TAILQ_INIT(&dev->free_bands); 49 TAILQ_INIT(&dev->shut_bands); 50 51 dev->num_free = 0; 52 dev->bands = calloc(ftl_get_num_bands(dev), sizeof(*dev->bands)); 53 if (!dev->bands) { 54 return -ENOMEM; 55 } 56 57 for (i = 0; i < ftl_get_num_bands(dev); ++i) { 58 band = &dev->bands[i]; 59 band->id = i; 60 band->dev = dev; 61 62 /* Adding to shut_bands is necessary - see ftl_restore_band_close_cb() */ 63 TAILQ_INSERT_TAIL(&dev->shut_bands, band, queue_entry); 64 } 65 66 return 0; 67 } 68 69 static int 70 ftl_dev_init_bands_md(struct spdk_ftl_dev *dev) 71 { 72 uint64_t i; 73 int rc = 0; 74 75 for (i = 0; i < ftl_get_num_bands(dev); ++i) { 76 rc = ftl_band_init_md(&dev->bands[i]); 77 if (rc) { 78 FTL_ERRLOG(dev, "Failed to initialize metadata structures for band [%lu]\n", i); 79 break; 80 } 81 } 82 83 return rc; 84 } 85 86 static void 87 ftl_dev_deinit_bands(struct spdk_ftl_dev *dev) 88 { 89 free(dev->bands); 90 } 91 92 static void 93 ftl_dev_deinit_bands_md(struct spdk_ftl_dev *dev) 94 { 95 if (dev->bands) { 96 uint64_t i; 97 for (i = 0; i < dev->num_bands; ++i) { 98 struct ftl_band *band = &dev->bands[i]; 99 100 ftl_bitmap_destroy(band->p2l_map.valid); 101 band->p2l_map.valid = NULL; 102 103 band->md = NULL; 104 } 105 } 106 } 107 108 void 109 ftl_mngt_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 110 { 111 if (ftl_dev_init_bands(dev)) { 112 ftl_mngt_fail_step(mngt); 113 } else { 114 ftl_mngt_next_step(mngt); 115 } 116 } 117 118 void 119 ftl_mngt_init_bands_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 120 { 121 if (ftl_dev_init_bands_md(dev)) { 122 ftl_mngt_fail_step(mngt); 123 } else { 124 ftl_mngt_next_step(mngt); 125 } 126 } 127 128 void 129 ftl_mngt_deinit_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 130 { 131 ftl_dev_deinit_bands(dev); 132 ftl_mngt_next_step(mngt); 133 } 134 135 void 136 ftl_mngt_deinit_bands_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 137 { 138 ftl_dev_deinit_bands_md(dev); 139 ftl_mngt_next_step(mngt); 140 } 141 142 /* 143 * For grouping multiple logical bands (1GiB) to make any IOs more sequential from the drive's 144 * perspective. Improves WAF. 145 */ 146 #define BASE_BDEV_RECLAIM_UNIT_SIZE (72 * GiB) 147 148 static void 149 decorate_bands(struct spdk_ftl_dev *dev) 150 { 151 struct ftl_band *band; 152 uint64_t i, num_to_drop, phys_id = 0; 153 uint64_t num_blocks, num_bands; 154 uint64_t num_blocks_in_band = ftl_get_num_blocks_in_band(dev); 155 uint64_t reclaim_unit_num_blocks = BASE_BDEV_RECLAIM_UNIT_SIZE / FTL_BLOCK_SIZE; 156 uint32_t num_logical_in_phys = 2; 157 158 assert(reclaim_unit_num_blocks % num_blocks_in_band == 0); 159 160 num_blocks = spdk_bdev_get_num_blocks(spdk_bdev_desc_get_bdev(dev->base_bdev_desc)); 161 162 /* For base bdev bigger than 1TB take reclaim uint size for grouping GC bands */ 163 if (num_blocks > (TiB / FTL_BLOCK_SIZE)) { 164 assert(reclaim_unit_num_blocks < num_blocks); 165 num_logical_in_phys = reclaim_unit_num_blocks / num_blocks_in_band; 166 } 167 168 num_to_drop = ftl_get_num_bands(dev) % num_logical_in_phys; 169 170 i = 0; 171 while (i < ftl_get_num_bands(dev) - num_to_drop) { 172 band = &dev->bands[i]; 173 band->start_addr = i * dev->num_blocks_in_band; 174 band->tail_md_addr = ftl_band_tail_md_addr(band); 175 176 band->phys_id = phys_id; 177 i++; 178 if (i % num_logical_in_phys == 0) { 179 phys_id++; 180 } 181 } 182 183 /* Mark not aligned logical bands as broken */ 184 num_bands = ftl_get_num_bands(dev); 185 while (i < num_bands) { 186 band = &dev->bands[i]; 187 dev->num_bands--; 188 TAILQ_REMOVE(&dev->shut_bands, band, queue_entry); 189 i++; 190 } 191 192 dev->num_logical_bands_in_physical = num_logical_in_phys; 193 } 194 195 void 196 ftl_mngt_decorate_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 197 { 198 decorate_bands(dev); 199 ftl_mngt_next_step(mngt); 200 } 201 202 void 203 ftl_recover_max_seq(struct spdk_ftl_dev *dev) 204 { 205 struct ftl_band *band; 206 size_t band_close_seq_id = 0, band_open_seq_id = 0; 207 size_t chunk_close_seq_id = 0, chunk_open_seq_id = 0; 208 size_t max = 0; 209 210 TAILQ_FOREACH(band, &dev->shut_bands, queue_entry) { 211 band_open_seq_id = spdk_max(band_open_seq_id, band->md->seq); 212 band_close_seq_id = spdk_max(band_close_seq_id, band->md->close_seq_id); 213 } 214 ftl_nv_cache_get_max_seq_id(&dev->nv_cache, &chunk_open_seq_id, &chunk_close_seq_id); 215 216 217 dev->nv_cache.last_seq_id = chunk_close_seq_id; 218 dev->writer_gc.last_seq_id = band_close_seq_id; 219 dev->writer_user.last_seq_id = band_close_seq_id; 220 221 max = spdk_max(max, band_open_seq_id); 222 max = spdk_max(max, band_close_seq_id); 223 max = spdk_max(max, chunk_open_seq_id); 224 max = spdk_max(max, chunk_close_seq_id); 225 226 dev->sb->seq_id = max; 227 } 228 229 static int 230 _band_cmp(const void *_a, const void *_b) 231 { 232 struct ftl_band *a, *b; 233 234 a = *((struct ftl_band **)_a); 235 b = *((struct ftl_band **)_b); 236 237 return a->md->seq - b->md->seq; 238 } 239 240 static struct ftl_band * 241 next_high_prio_band(struct spdk_ftl_dev *dev) 242 { 243 struct ftl_band *result = NULL, *band; 244 uint64_t validity = UINT64_MAX; 245 246 TAILQ_FOREACH(band, &dev->shut_bands, queue_entry) { 247 if (band->p2l_map.num_valid < validity) { 248 result = band; 249 validity = result->p2l_map.num_valid; 250 } 251 } 252 253 return result; 254 } 255 256 static int 257 finalize_init_gc(struct spdk_ftl_dev *dev) 258 { 259 struct ftl_band *band; 260 uint64_t free_blocks, blocks_to_move; 261 262 ftl_band_init_gc_iter(dev); 263 dev->sb_shm->gc_info.band_id_high_prio = FTL_BAND_ID_INVALID; 264 265 if (0 == dev->num_free) { 266 /* Get number of available blocks in writer */ 267 free_blocks = ftl_writer_get_free_blocks(&dev->writer_gc); 268 269 /* 270 * First, check a band candidate to GC 271 */ 272 band = ftl_band_search_next_to_reloc(dev); 273 ftl_bug(NULL == band); 274 blocks_to_move = band->p2l_map.num_valid; 275 if (blocks_to_move <= free_blocks) { 276 /* This GC band can be moved */ 277 return 0; 278 } 279 280 /* 281 * The GC candidate cannot be moved because no enough space. We need to find 282 * another band. 283 */ 284 band = next_high_prio_band(dev); 285 ftl_bug(NULL == band); 286 287 if (band->p2l_map.num_valid > free_blocks) { 288 FTL_ERRLOG(dev, "CRITICAL ERROR, no more free bands and cannot start\n"); 289 return -1; 290 } else { 291 /* GC needs to start using this band */ 292 dev->sb_shm->gc_info.band_id_high_prio = band->id; 293 } 294 } 295 296 return 0; 297 } 298 299 void 300 ftl_mngt_finalize_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 301 { 302 struct ftl_band *band, *temp_band, *open_bands[FTL_MAX_OPEN_BANDS]; 303 struct ftl_writer *writer; 304 uint64_t i, num_open = 0, num_shut = 0; 305 uint64_t offset; 306 bool fast_startup = ftl_fast_startup(dev); 307 308 ftl_recover_max_seq(dev); 309 310 TAILQ_FOREACH_SAFE(band, &dev->free_bands, queue_entry, temp_band) { 311 band->md->df_p2l_map = FTL_DF_OBJ_ID_INVALID; 312 } 313 314 TAILQ_FOREACH_SAFE(band, &dev->shut_bands, queue_entry, temp_band) { 315 if (band->md->state == FTL_BAND_STATE_OPEN || 316 band->md->state == FTL_BAND_STATE_FULL) { 317 TAILQ_REMOVE(&dev->shut_bands, band, queue_entry); 318 open_bands[num_open++] = band; 319 assert(num_open <= FTL_MAX_OPEN_BANDS); 320 continue; 321 } 322 323 if (dev->conf.mode & SPDK_FTL_MODE_CREATE) { 324 TAILQ_REMOVE(&dev->shut_bands, band, queue_entry); 325 assert(band->md->state == FTL_BAND_STATE_FREE); 326 band->md->state = FTL_BAND_STATE_CLOSED; 327 ftl_band_set_state(band, FTL_BAND_STATE_FREE); 328 } else { 329 num_shut++; 330 } 331 332 band->md->df_p2l_map = FTL_DF_OBJ_ID_INVALID; 333 } 334 335 /* Assign open bands to writers and alloc necessary resources */ 336 qsort(open_bands, num_open, sizeof(open_bands[0]), _band_cmp); 337 338 for (i = 0; i < num_open; ++i) { 339 band = open_bands[i]; 340 341 if (band->md->type == FTL_BAND_TYPE_COMPACTION) { 342 writer = &dev->writer_user; 343 } else if (band->md->type == FTL_BAND_TYPE_GC) { 344 writer = &dev->writer_gc; 345 } else { 346 assert(false); 347 } 348 349 if (band->md->state == FTL_BAND_STATE_FULL) { 350 TAILQ_INSERT_TAIL(&writer->full_bands, band, queue_entry); 351 } else { 352 if (writer->band == NULL) { 353 writer->band = band; 354 } else { 355 writer->next_band = band; 356 } 357 } 358 359 writer->num_bands++; 360 ftl_band_set_owner(band, ftl_writer_band_state_change, writer); 361 362 if (fast_startup) { 363 FTL_NOTICELOG(dev, "SHM: band open P2L map df_id 0x%"PRIx64"\n", band->md->df_p2l_map); 364 if (ftl_band_open_p2l_map(band)) { 365 ftl_mngt_fail_step(mngt); 366 return; 367 } 368 369 offset = band->md->iter.offset; 370 ftl_band_iter_init(band); 371 ftl_band_iter_set(band, offset); 372 ftl_mngt_p2l_ckpt_restore_shm_clean(band); 373 } else if (dev->sb->clean) { 374 band->md->df_p2l_map = FTL_DF_OBJ_ID_INVALID; 375 if (ftl_band_alloc_p2l_map(band)) { 376 ftl_mngt_fail_step(mngt); 377 return; 378 } 379 380 offset = band->md->iter.offset; 381 ftl_band_iter_init(band); 382 ftl_band_iter_set(band, offset); 383 384 if (ftl_mngt_p2l_ckpt_restore_clean(band)) { 385 ftl_mngt_fail_step(mngt); 386 return; 387 } 388 } 389 } 390 391 if (fast_startup) { 392 ftl_mempool_initialize_ext(dev->p2l_pool); 393 } 394 395 396 /* Recalculate number of free bands */ 397 dev->num_free = 0; 398 TAILQ_FOREACH(band, &dev->free_bands, queue_entry) { 399 assert(band->md->state == FTL_BAND_STATE_FREE); 400 dev->num_free++; 401 } 402 ftl_apply_limits(dev); 403 404 if ((num_shut + num_open + dev->num_free) != ftl_get_num_bands(dev)) { 405 FTL_ERRLOG(dev, "ERROR, band list inconsistent state\n"); 406 ftl_mngt_fail_step(mngt); 407 return; 408 } 409 410 if (finalize_init_gc(dev)) { 411 ftl_mngt_fail_step(mngt); 412 } else { 413 ftl_mngt_next_step(mngt); 414 } 415 } 416