1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "ftl_core.h" 7 #include "ftl_utils.h" 8 #include "ftl_mngt.h" 9 #include "ftl_mngt_steps.h" 10 #include "ftl_band.h" 11 #include "ftl_internal.h" 12 #include "ftl_nv_cache.h" 13 #include "ftl_debug.h" 14 #include "ftl_utils.h" 15 16 void 17 ftl_mngt_check_conf(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 18 { 19 if (ftl_conf_is_valid(&dev->conf)) { 20 ftl_mngt_next_step(mngt); 21 } else { 22 ftl_mngt_fail_step(mngt); 23 } 24 } 25 26 static int 27 init_p2l_map_pool(struct spdk_ftl_dev *dev) 28 { 29 size_t p2l_pool_el_blks = spdk_divide_round_up(ftl_p2l_map_pool_elem_size(dev), FTL_BLOCK_SIZE); 30 size_t p2l_pool_buf_blks = P2L_MEMPOOL_SIZE * p2l_pool_el_blks; 31 void *p2l_pool_buf; 32 33 dev->p2l_pool_md = ftl_md_create(dev, p2l_pool_buf_blks, 0, "p2l_pool", 34 ftl_md_create_shm_flags(dev), NULL); 35 if (!dev->p2l_pool_md) { 36 return -ENOMEM; 37 } 38 39 p2l_pool_buf = ftl_md_get_buffer(dev->p2l_pool_md); 40 dev->p2l_pool = ftl_mempool_create_ext(p2l_pool_buf, P2L_MEMPOOL_SIZE, 41 p2l_pool_el_blks * FTL_BLOCK_SIZE, 42 FTL_BLOCK_SIZE); 43 if (!dev->p2l_pool) { 44 return -ENOMEM; 45 } 46 47 if (!ftl_fast_startup(dev)) { 48 ftl_mempool_initialize_ext(dev->p2l_pool); 49 } 50 51 return 0; 52 } 53 54 static int 55 init_band_md_pool(struct spdk_ftl_dev *dev) 56 { 57 dev->band_md_pool = ftl_mempool_create(P2L_MEMPOOL_SIZE, 58 sizeof(struct ftl_band_md), 59 FTL_BLOCK_SIZE, 60 SPDK_ENV_SOCKET_ID_ANY); 61 if (!dev->band_md_pool) { 62 return -ENOMEM; 63 } 64 65 return 0; 66 } 67 68 void 69 ftl_mngt_init_mem_pools(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 70 { 71 if (init_p2l_map_pool(dev)) { 72 ftl_mngt_fail_step(mngt); 73 return; 74 } 75 76 if (init_band_md_pool(dev)) { 77 ftl_mngt_fail_step(mngt); 78 return; 79 } 80 81 ftl_mngt_next_step(mngt); 82 } 83 84 void 85 ftl_mngt_deinit_mem_pools(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 86 { 87 if (dev->p2l_pool) { 88 ftl_mempool_destroy_ext(dev->p2l_pool); 89 dev->p2l_pool = NULL; 90 } 91 92 if (dev->p2l_pool_md) { 93 ftl_md_destroy(dev->p2l_pool_md, ftl_md_destroy_shm_flags(dev)); 94 dev->p2l_pool_md = NULL; 95 } 96 97 if (dev->band_md_pool) { 98 ftl_mempool_destroy(dev->band_md_pool); 99 dev->band_md_pool = NULL; 100 } 101 102 ftl_mngt_next_step(mngt); 103 } 104 105 void 106 ftl_mngt_init_reloc(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 107 { 108 dev->reloc = ftl_reloc_init(dev); 109 if (!dev->reloc) { 110 FTL_ERRLOG(dev, "Unable to initialize reloc structures\n"); 111 ftl_mngt_fail_step(mngt); 112 return; 113 } 114 115 ftl_mngt_next_step(mngt); 116 } 117 118 void 119 ftl_mngt_deinit_reloc(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 120 { 121 ftl_reloc_free(dev->reloc); 122 ftl_mngt_next_step(mngt); 123 } 124 125 void 126 ftl_mngt_init_nv_cache(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 127 { 128 if (ftl_nv_cache_init(dev)) { 129 FTL_ERRLOG(dev, "Unable to initialize persistent cache\n"); 130 ftl_mngt_fail_step(mngt); 131 return; 132 } 133 134 ftl_mngt_next_step(mngt); 135 } 136 137 void 138 ftl_mngt_deinit_nv_cache(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 139 { 140 ftl_nv_cache_deinit(dev); 141 ftl_mngt_next_step(mngt); 142 } 143 144 static void 145 user_clear_cb(struct spdk_ftl_dev *dev, struct ftl_md *md, int status) 146 { 147 struct ftl_mngt_process *mngt = md->owner.cb_ctx; 148 149 if (status) { 150 FTL_ERRLOG(ftl_mngt_get_dev(mngt), "FTL NV Cache: ERROR of clearing user cache data\n"); 151 ftl_mngt_fail_step(mngt); 152 } else { 153 ftl_mngt_next_step(mngt); 154 } 155 } 156 157 void 158 ftl_mngt_scrub_nv_cache(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 159 { 160 struct ftl_layout_region *region = &dev->layout.region[FTL_LAYOUT_REGION_TYPE_DATA_NVC]; 161 struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_DATA_NVC]; 162 union ftl_md_vss vss; 163 164 FTL_NOTICELOG(dev, "First startup needs to scrub nv cache data region, this may take some time.\n"); 165 FTL_NOTICELOG(dev, "Scrubbing %lluGiB\n", region->current.blocks * FTL_BLOCK_SIZE / GiB); 166 167 /* Need to scrub user data, so in case of dirty shutdown the recovery won't 168 * pull in data during open chunks recovery from any previous instance (since during short 169 * tests it's very likely that chunks seq_id will be in line between new head md and old VSS) 170 */ 171 md->cb = user_clear_cb; 172 md->owner.cb_ctx = mngt; 173 174 vss.version.md_version = region->current.version; 175 vss.nv_cache.lba = FTL_ADDR_INVALID; 176 ftl_md_clear(md, 0, &vss); 177 } 178 179 void 180 ftl_mngt_finalize_startup(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 181 { 182 if (ftl_bitmap_find_first_set(dev->unmap_map, 0, UINT64_MAX) != UINT64_MAX) { 183 dev->unmap_in_progress = true; 184 } 185 186 ftl_property_register(dev, "superblock_version", &dev->sb->header.version, 187 sizeof(dev->sb->header.version), ftl_property_dump_uint64); 188 189 /* Clear the limit applications as they're incremented incorrectly by 190 * the initialization code. 191 */ 192 memset(dev->stats.limits, 0, sizeof(dev->stats.limits)); 193 dev->initialized = 1; 194 dev->sb_shm->shm_ready = true; 195 196 ftl_l2p_resume(dev); 197 ftl_reloc_resume(dev->reloc); 198 ftl_writer_resume(&dev->writer_user); 199 ftl_writer_resume(&dev->writer_gc); 200 ftl_nv_cache_resume(&dev->nv_cache); 201 202 ftl_mngt_next_step(mngt); 203 } 204 205 void 206 ftl_mngt_start_core_poller(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 207 { 208 dev->core_poller = SPDK_POLLER_REGISTER(ftl_core_poller, dev, 0); 209 if (!dev->core_poller) { 210 FTL_ERRLOG(dev, "Unable to register core poller\n"); 211 ftl_mngt_fail_step(mngt); 212 return; 213 } 214 215 ftl_mngt_next_step(mngt); 216 } 217 218 void 219 ftl_mngt_stop_core_poller(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 220 { 221 dev->halt = true; 222 223 if (dev->core_poller) { 224 ftl_mngt_continue_step(mngt); 225 } else { 226 ftl_mngt_next_step(mngt); 227 } 228 } 229 230 void 231 ftl_mngt_dump_stats(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 232 { 233 ftl_dev_dump_bands(dev); 234 ftl_dev_dump_stats(dev); 235 ftl_mngt_next_step(mngt); 236 } 237 238 void 239 ftl_mngt_init_vld_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 240 { 241 struct ftl_md *valid_map_md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_VALID_MAP]; 242 243 dev->valid_map = ftl_bitmap_create(ftl_md_get_buffer(valid_map_md), 244 ftl_md_get_buffer_size(valid_map_md)); 245 if (!dev->valid_map) { 246 FTL_ERRLOG(dev, "Failed to create valid map\n"); 247 ftl_mngt_fail_step(mngt); 248 return; 249 } 250 251 ftl_mngt_next_step(mngt); 252 } 253 254 void 255 ftl_mngt_deinit_vld_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 256 { 257 if (dev->valid_map) { 258 ftl_bitmap_destroy(dev->valid_map); 259 dev->valid_map = NULL; 260 } 261 262 ftl_mngt_next_step(mngt); 263 } 264 void 265 ftl_mngt_init_unmap_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 266 { 267 uint64_t num_l2p_pages = spdk_divide_round_up(dev->num_lbas, dev->layout.l2p.lbas_in_page); 268 uint64_t map_blocks = ftl_bitmap_bits_to_blocks(num_l2p_pages); 269 270 dev->unmap_map_md = ftl_md_create(dev, 271 map_blocks, 272 0, 273 "trim_bitmap", 274 ftl_md_create_shm_flags(dev), NULL); 275 276 if (!dev->unmap_map_md) { 277 FTL_ERRLOG(dev, "Failed to create trim bitmap md\n"); 278 ftl_mngt_fail_step(mngt); 279 return; 280 } 281 282 dev->unmap_map = ftl_bitmap_create(ftl_md_get_buffer(dev->unmap_map_md), 283 ftl_md_get_buffer_size(dev->unmap_map_md)); 284 285 if (!dev->unmap_map) { 286 FTL_ERRLOG(dev, "Failed to create unmap map\n"); 287 ftl_mngt_fail_step(mngt); 288 return; 289 } 290 291 ftl_mngt_next_step(mngt); 292 } 293 294 static void 295 unmap_clear_cb(struct spdk_ftl_dev *dev, struct ftl_md *md, int status) 296 { 297 struct ftl_mngt_process *mngt = md->owner.cb_ctx; 298 299 if (status) { 300 FTL_ERRLOG(dev, "ERROR of clearing trim unmap\n"); 301 ftl_mngt_fail_step(mngt); 302 } else { 303 ftl_mngt_next_step(mngt); 304 } 305 } 306 307 void 308 ftl_mngt_unmap_clear(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 309 { 310 struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_TRIM_MD]; 311 312 md->cb = unmap_clear_cb; 313 md->owner.cb_ctx = mngt; 314 315 ftl_md_clear(md, 0, NULL); 316 } 317 318 void 319 ftl_mngt_deinit_unmap_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 320 { 321 ftl_bitmap_destroy(dev->unmap_map); 322 dev->unmap_map = NULL; 323 324 ftl_md_destroy(dev->unmap_map_md, ftl_md_destroy_shm_flags(dev)); 325 dev->unmap_map_md = NULL; 326 327 ftl_mngt_next_step(mngt); 328 } 329 330 struct ftl_get_properties_ctx { 331 struct spdk_ftl_dev *dev; 332 struct spdk_jsonrpc_request *request; 333 spdk_ftl_fn cb_fn; 334 void *cb_arg; 335 struct spdk_thread *cb_thread; 336 }; 337 338 static void 339 ftl_get_properties_cb(void *arg) 340 { 341 struct ftl_get_properties_ctx *ctx = arg; 342 343 ctx->cb_fn(ctx->cb_arg, 0); 344 free(ctx); 345 } 346 347 static void 348 ftl_get_properties_msg(void *arg) 349 { 350 struct ftl_get_properties_ctx *ctx = arg; 351 int rc; 352 353 ftl_property_dump(ctx->dev, ctx->request); 354 rc = spdk_thread_send_msg(ctx->cb_thread, ftl_get_properties_cb, ctx); 355 ftl_bug(rc); 356 } 357 358 int 359 spdk_ftl_get_properties(struct spdk_ftl_dev *dev, struct spdk_jsonrpc_request *request, 360 spdk_ftl_fn cb_fn, void *cb_arg) 361 { 362 int rc; 363 struct ftl_get_properties_ctx *ctx = calloc(1, sizeof(*ctx)); 364 365 if (ctx == NULL) { 366 return -ENOMEM; 367 } 368 ctx->dev = dev; 369 ctx->request = request; 370 ctx->cb_fn = cb_fn; 371 ctx->cb_arg = cb_arg; 372 ctx->cb_thread = spdk_get_thread(); 373 374 rc = spdk_thread_send_msg(dev->core_thread, ftl_get_properties_msg, ctx); 375 if (rc) { 376 free(ctx); 377 return rc; 378 } 379 380 return rc; 381 } 382