1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "ftl_core.h" 7 #include "ftl_mngt_steps.h" 8 #include "ftl_band.h" 9 #include "ftl_internal.h" 10 11 static int 12 ftl_band_init_md(struct ftl_band *band) 13 { 14 struct spdk_ftl_dev *dev = band->dev; 15 struct ftl_p2l_map *p2l_map = &band->p2l_map; 16 struct ftl_md *band_info_md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_BAND_MD]; 17 struct ftl_md *valid_map_md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_VALID_MAP]; 18 uint64_t band_num_blocks = ftl_get_num_blocks_in_band(band->dev); 19 size_t band_valid_map_bytes; 20 struct ftl_band_md *band_md = ftl_md_get_buffer(band_info_md); 21 22 if (band_num_blocks % (ftl_bitmap_buffer_alignment * 8)) { 23 FTL_ERRLOG(dev, "The number of blocks in band is not divisible by bitmap word bits\n"); 24 return -EINVAL; 25 } 26 band_valid_map_bytes = band_num_blocks / 8; 27 28 p2l_map->valid = ftl_bitmap_create(ftl_md_get_buffer(valid_map_md) + 29 band_valid_map_bytes * band->id, band_valid_map_bytes); 30 if (!p2l_map->valid) { 31 return -ENOMEM; 32 } 33 34 band->md = &band_md[band->id]; 35 if (!ftl_fast_startup(dev)) { 36 band->md->df_p2l_map = FTL_DF_OBJ_ID_INVALID; 37 } 38 39 return 0; 40 } 41 42 static int 43 ftl_dev_init_bands(struct spdk_ftl_dev *dev) 44 { 45 struct ftl_band *band; 46 uint64_t i; 47 48 TAILQ_INIT(&dev->free_bands); 49 TAILQ_INIT(&dev->shut_bands); 50 51 dev->num_free = 0; 52 dev->bands = calloc(ftl_get_num_bands(dev), sizeof(*dev->bands)); 53 if (!dev->bands) { 54 return -ENOMEM; 55 } 56 57 for (i = 0; i < ftl_get_num_bands(dev); ++i) { 58 band = &dev->bands[i]; 59 band->id = i; 60 band->dev = dev; 61 62 /* Adding to shut_bands is necessary - see ftl_restore_band_close_cb() */ 63 TAILQ_INSERT_TAIL(&dev->shut_bands, band, queue_entry); 64 } 65 66 return 0; 67 } 68 69 static int 70 ftl_dev_init_bands_md(struct spdk_ftl_dev *dev) 71 { 72 uint64_t i; 73 int rc = 0; 74 75 for (i = 0; i < ftl_get_num_bands(dev); ++i) { 76 rc = ftl_band_init_md(&dev->bands[i]); 77 if (rc) { 78 FTL_ERRLOG(dev, "Failed to initialize metadata structures for band [%lu]\n", i); 79 break; 80 } 81 } 82 83 return rc; 84 } 85 86 static void 87 ftl_dev_deinit_bands(struct spdk_ftl_dev *dev) 88 { 89 free(dev->bands); 90 } 91 92 static void 93 ftl_dev_deinit_bands_md(struct spdk_ftl_dev *dev) 94 { 95 if (dev->bands) { 96 uint64_t i; 97 for (i = 0; i < dev->num_bands; ++i) { 98 struct ftl_band *band = &dev->bands[i]; 99 100 ftl_bitmap_destroy(band->p2l_map.valid); 101 band->p2l_map.valid = NULL; 102 103 band->md = NULL; 104 } 105 } 106 } 107 108 void 109 ftl_mngt_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 110 { 111 if (ftl_dev_init_bands(dev)) { 112 ftl_mngt_fail_step(mngt); 113 } else { 114 ftl_mngt_next_step(mngt); 115 } 116 } 117 118 void 119 ftl_mngt_init_bands_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 120 { 121 if (ftl_dev_init_bands_md(dev)) { 122 ftl_mngt_fail_step(mngt); 123 } else { 124 ftl_mngt_next_step(mngt); 125 } 126 } 127 128 void 129 ftl_mngt_deinit_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 130 { 131 ftl_dev_deinit_bands(dev); 132 ftl_mngt_next_step(mngt); 133 } 134 135 void 136 ftl_mngt_deinit_bands_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 137 { 138 ftl_dev_deinit_bands_md(dev); 139 ftl_mngt_next_step(mngt); 140 } 141 142 /* 143 * For grouping multiple logical bands (1GiB) to make any IOs more sequential from the drive's 144 * perspective. Improves WAF. 145 */ 146 #define BASE_BDEV_RECLAIM_UNIT_SIZE (72 * GiB) 147 148 static void 149 decorate_bands(struct spdk_ftl_dev *dev) 150 { 151 struct ftl_band *band; 152 uint64_t i, num_to_drop, phys_id = 0; 153 uint64_t num_blocks, num_bands; 154 uint64_t num_blocks_in_band = ftl_get_num_blocks_in_band(dev); 155 uint64_t reclaim_unit_num_blocks = BASE_BDEV_RECLAIM_UNIT_SIZE / FTL_BLOCK_SIZE; 156 uint32_t num_logical_in_phys = 2; 157 158 assert(reclaim_unit_num_blocks % num_blocks_in_band == 0); 159 160 num_blocks = spdk_bdev_get_num_blocks(spdk_bdev_desc_get_bdev(dev->base_bdev_desc)); 161 162 /* For base bdev bigger than 1TB take reclaim uint size for grouping GC bands */ 163 if (num_blocks > (TiB / FTL_BLOCK_SIZE)) { 164 assert(reclaim_unit_num_blocks < num_blocks); 165 num_logical_in_phys = reclaim_unit_num_blocks / num_blocks_in_band; 166 } 167 168 num_to_drop = ftl_get_num_bands(dev) % num_logical_in_phys; 169 170 i = 0; 171 while (i < ftl_get_num_bands(dev) - num_to_drop) { 172 band = &dev->bands[i]; 173 band->start_addr = i * dev->num_blocks_in_band; 174 band->tail_md_addr = ftl_band_tail_md_addr(band); 175 176 band->phys_id = phys_id; 177 i++; 178 if (i % num_logical_in_phys == 0) { 179 phys_id++; 180 } 181 } 182 183 /* Mark not aligned logical bands as broken */ 184 num_bands = ftl_get_num_bands(dev); 185 while (i < num_bands) { 186 band = &dev->bands[i]; 187 dev->num_bands--; 188 TAILQ_REMOVE(&dev->shut_bands, band, queue_entry); 189 i++; 190 } 191 192 dev->num_logical_bands_in_physical = num_logical_in_phys; 193 } 194 195 void 196 ftl_mngt_decorate_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 197 { 198 decorate_bands(dev); 199 ftl_mngt_next_step(mngt); 200 } 201 202 static struct ftl_band * 203 next_high_prio_band(struct spdk_ftl_dev *dev) 204 { 205 struct ftl_band *result = NULL, *band; 206 uint64_t validity = UINT64_MAX; 207 208 TAILQ_FOREACH(band, &dev->shut_bands, queue_entry) { 209 if (band->p2l_map.num_valid < validity) { 210 result = band; 211 validity = result->p2l_map.num_valid; 212 } 213 } 214 215 return result; 216 } 217 218 static int 219 finalize_init_gc(struct spdk_ftl_dev *dev) 220 { 221 struct ftl_band *band; 222 uint64_t free_blocks, blocks_to_move; 223 224 ftl_band_init_gc_iter(dev); 225 dev->sb_shm->gc_info.band_id_high_prio = FTL_BAND_ID_INVALID; 226 227 if (0 == dev->num_free) { 228 /* Get number of available blocks in writer */ 229 free_blocks = ftl_writer_get_free_blocks(&dev->writer_gc); 230 231 /* 232 * First, check a band candidate to GC 233 */ 234 band = ftl_band_search_next_to_reloc(dev); 235 ftl_bug(NULL == band); 236 blocks_to_move = band->p2l_map.num_valid; 237 if (blocks_to_move <= free_blocks) { 238 /* This GC band can be moved */ 239 return 0; 240 } 241 242 /* 243 * The GC candidate cannot be moved because no enough space. We need to find 244 * another band. 245 */ 246 band = next_high_prio_band(dev); 247 ftl_bug(NULL == band); 248 249 if (band->p2l_map.num_valid > free_blocks) { 250 FTL_ERRLOG(dev, "CRITICAL ERROR, no more free bands and cannot start\n"); 251 return -1; 252 } else { 253 /* GC needs to start using this band */ 254 dev->sb_shm->gc_info.band_id_high_prio = band->id; 255 } 256 } 257 258 return 0; 259 } 260 261 void 262 ftl_mngt_finalize_init_bands(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 263 { 264 struct ftl_band *band, *temp_band, *open_bands[FTL_MAX_OPEN_BANDS]; 265 struct ftl_writer *writer; 266 uint64_t i, num_open = 0, num_shut = 0; 267 uint64_t offset; 268 bool fast_startup = ftl_fast_startup(dev); 269 270 TAILQ_FOREACH_SAFE(band, &dev->free_bands, queue_entry, temp_band) { 271 band->md->df_p2l_map = FTL_DF_OBJ_ID_INVALID; 272 } 273 274 TAILQ_FOREACH_SAFE(band, &dev->shut_bands, queue_entry, temp_band) { 275 if (band->md->state == FTL_BAND_STATE_OPEN || 276 band->md->state == FTL_BAND_STATE_FULL) { 277 TAILQ_REMOVE(&dev->shut_bands, band, queue_entry); 278 open_bands[num_open++] = band; 279 assert(num_open <= FTL_MAX_OPEN_BANDS); 280 continue; 281 } 282 283 if (dev->conf.mode & SPDK_FTL_MODE_CREATE) { 284 TAILQ_REMOVE(&dev->shut_bands, band, queue_entry); 285 assert(band->md->state == FTL_BAND_STATE_FREE); 286 band->md->state = FTL_BAND_STATE_CLOSED; 287 ftl_band_set_state(band, FTL_BAND_STATE_FREE); 288 } else { 289 num_shut++; 290 } 291 292 band->md->df_p2l_map = FTL_DF_OBJ_ID_INVALID; 293 } 294 295 /* Assign open bands to writers and alloc necessary resources */ 296 for (i = 0; i < num_open; ++i) { 297 band = open_bands[i]; 298 299 if (band->md->type == FTL_BAND_TYPE_COMPACTION) { 300 writer = &dev->writer_user; 301 } else if (band->md->type == FTL_BAND_TYPE_GC) { 302 writer = &dev->writer_gc; 303 } else { 304 assert(false); 305 } 306 307 if (band->md->state == FTL_BAND_STATE_FULL) { 308 TAILQ_INSERT_TAIL(&writer->full_bands, band, queue_entry); 309 } else { 310 if (writer->band == NULL) { 311 writer->band = band; 312 } else { 313 writer->next_band = band; 314 } 315 } 316 317 writer->num_bands++; 318 ftl_band_set_owner(band, ftl_writer_band_state_change, writer); 319 320 if (fast_startup) { 321 FTL_NOTICELOG(dev, "SHM: band open P2L map df_id 0x%"PRIx64"\n", band->md->df_p2l_map); 322 if (ftl_band_open_p2l_map(band)) { 323 ftl_mngt_fail_step(mngt); 324 return; 325 } 326 327 offset = band->md->iter.offset; 328 ftl_band_iter_init(band); 329 ftl_band_iter_set(band, offset); 330 } else if (dev->sb->clean) { 331 band->md->df_p2l_map = FTL_DF_OBJ_ID_INVALID; 332 if (ftl_band_alloc_p2l_map(band)) { 333 ftl_mngt_fail_step(mngt); 334 return; 335 } 336 337 offset = band->md->iter.offset; 338 ftl_band_iter_init(band); 339 ftl_band_iter_set(band, offset); 340 } 341 } 342 343 if (fast_startup) { 344 ftl_mempool_initialize_ext(dev->p2l_pool); 345 } 346 347 348 /* Recalculate number of free bands */ 349 dev->num_free = 0; 350 TAILQ_FOREACH(band, &dev->free_bands, queue_entry) { 351 assert(band->md->state == FTL_BAND_STATE_FREE); 352 dev->num_free++; 353 } 354 ftl_apply_limits(dev); 355 356 if ((num_shut + num_open + dev->num_free) != ftl_get_num_bands(dev)) { 357 FTL_ERRLOG(dev, "ERROR, band list inconsistent state\n"); 358 ftl_mngt_fail_step(mngt); 359 return; 360 } 361 362 if (finalize_init_gc(dev)) { 363 ftl_mngt_fail_step(mngt); 364 } else { 365 ftl_mngt_next_step(mngt); 366 } 367 } 368