1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "ftl_core.h" 7 #include "ftl_utils.h" 8 #include "ftl_mngt.h" 9 #include "ftl_mngt_steps.h" 10 #include "ftl_band.h" 11 #include "ftl_internal.h" 12 #include "ftl_nv_cache.h" 13 #include "ftl_debug.h" 14 #include "ftl_utils.h" 15 16 void 17 ftl_mngt_check_conf(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 18 { 19 if (ftl_conf_is_valid(&dev->conf)) { 20 ftl_mngt_next_step(mngt); 21 } else { 22 ftl_mngt_fail_step(mngt); 23 } 24 } 25 26 static int 27 init_p2l_map_pool(struct spdk_ftl_dev *dev) 28 { 29 size_t p2l_pool_el_blks = spdk_divide_round_up(ftl_p2l_map_pool_elem_size(dev), FTL_BLOCK_SIZE); 30 size_t p2l_pool_buf_blks = P2L_MEMPOOL_SIZE * p2l_pool_el_blks; 31 void *p2l_pool_buf; 32 33 dev->p2l_pool_md = ftl_md_create(dev, p2l_pool_buf_blks, 0, "p2l_pool", 34 ftl_md_create_shm_flags(dev), NULL); 35 if (!dev->p2l_pool_md) { 36 return -ENOMEM; 37 } 38 39 p2l_pool_buf = ftl_md_get_buffer(dev->p2l_pool_md); 40 dev->p2l_pool = ftl_mempool_create_ext(p2l_pool_buf, P2L_MEMPOOL_SIZE, 41 p2l_pool_el_blks * FTL_BLOCK_SIZE, 42 FTL_BLOCK_SIZE); 43 if (!dev->p2l_pool) { 44 return -ENOMEM; 45 } 46 47 if (!ftl_fast_startup(dev)) { 48 ftl_mempool_initialize_ext(dev->p2l_pool); 49 } 50 51 return 0; 52 } 53 54 static int 55 init_band_md_pool(struct spdk_ftl_dev *dev) 56 { 57 dev->band_md_pool = ftl_mempool_create(P2L_MEMPOOL_SIZE, 58 sizeof(struct ftl_band_md), 59 FTL_BLOCK_SIZE, 60 SPDK_ENV_SOCKET_ID_ANY); 61 if (!dev->band_md_pool) { 62 return -ENOMEM; 63 } 64 65 return 0; 66 } 67 68 void 69 ftl_mngt_init_mem_pools(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 70 { 71 if (init_p2l_map_pool(dev)) { 72 ftl_mngt_fail_step(mngt); 73 return; 74 } 75 76 if (init_band_md_pool(dev)) { 77 ftl_mngt_fail_step(mngt); 78 return; 79 } 80 81 ftl_mngt_next_step(mngt); 82 } 83 84 void 85 ftl_mngt_deinit_mem_pools(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 86 { 87 if (dev->p2l_pool) { 88 ftl_mempool_destroy_ext(dev->p2l_pool); 89 dev->p2l_pool = NULL; 90 } 91 92 if (dev->p2l_pool_md) { 93 ftl_md_destroy(dev->p2l_pool_md, ftl_md_destroy_shm_flags(dev)); 94 dev->p2l_pool_md = NULL; 95 } 96 97 if (dev->band_md_pool) { 98 ftl_mempool_destroy(dev->band_md_pool); 99 dev->band_md_pool = NULL; 100 } 101 102 ftl_mngt_next_step(mngt); 103 } 104 105 void 106 ftl_mngt_init_reloc(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 107 { 108 dev->reloc = ftl_reloc_init(dev); 109 if (!dev->reloc) { 110 FTL_ERRLOG(dev, "Unable to initialize reloc structures\n"); 111 ftl_mngt_fail_step(mngt); 112 return; 113 } 114 115 ftl_mngt_next_step(mngt); 116 } 117 118 void 119 ftl_mngt_deinit_reloc(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 120 { 121 ftl_reloc_free(dev->reloc); 122 ftl_mngt_next_step(mngt); 123 } 124 125 void 126 ftl_mngt_init_nv_cache(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 127 { 128 if (ftl_nv_cache_init(dev)) { 129 FTL_ERRLOG(dev, "Unable to initialize persistent cache\n"); 130 ftl_mngt_fail_step(mngt); 131 return; 132 } 133 134 ftl_mngt_next_step(mngt); 135 } 136 137 void 138 ftl_mngt_deinit_nv_cache(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 139 { 140 ftl_nv_cache_deinit(dev); 141 ftl_mngt_next_step(mngt); 142 } 143 144 static void 145 user_clear_cb(struct spdk_ftl_dev *dev, void *cb_ctx, int status) 146 { 147 struct ftl_mngt_process *mngt = cb_ctx; 148 149 if (status) { 150 FTL_ERRLOG(ftl_mngt_get_dev(mngt), "FTL NV Cache: ERROR of clearing user cache data\n"); 151 ftl_mngt_fail_step(mngt); 152 } else { 153 ftl_mngt_next_step(mngt); 154 } 155 } 156 157 void 158 ftl_mngt_scrub_nv_cache(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 159 { 160 bool is_first_start = (dev->conf.mode & SPDK_FTL_MODE_CREATE) != 0; 161 bool is_major_upgrade = dev->sb->clean == 1 && dev->sb_shm->shm_clean == 0 && 162 dev->sb->upgrade_ready == 1; 163 164 if (is_first_start || is_major_upgrade) { 165 FTL_NOTICELOG(dev, "NV cache data region needs scrubbing, this may take a while.\n"); 166 FTL_NOTICELOG(dev, "Scrubbing %"PRIu64" chunks\n", dev->layout.nvc.chunk_count); 167 168 /* Need to scrub user data, so in case of dirty shutdown the recovery won't 169 * pull in data during open chunks recovery from any previous instance (since during short 170 * tests it's very likely that chunks seq_id will be in line between new head md and old VSS) 171 */ 172 ftl_nv_cache_scrub(dev, user_clear_cb, mngt); 173 } else { 174 ftl_mngt_skip_step(mngt); 175 } 176 } 177 178 void 179 ftl_mngt_finalize_startup(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 180 { 181 if (ftl_bitmap_find_first_set(dev->unmap_map, 0, UINT64_MAX) != UINT64_MAX) { 182 dev->unmap_in_progress = true; 183 } 184 185 ftl_property_register(dev, "superblock_version", &dev->sb->header.version, 186 sizeof(dev->sb->header.version), NULL, NULL, 187 ftl_property_dump_uint64, NULL, NULL, false); 188 189 /* Clear the limit applications as they're incremented incorrectly by 190 * the initialization code. 191 */ 192 memset(dev->stats.limits, 0, sizeof(dev->stats.limits)); 193 dev->initialized = 1; 194 dev->sb_shm->shm_ready = true; 195 196 ftl_l2p_resume(dev); 197 ftl_reloc_resume(dev->reloc); 198 ftl_writer_resume(&dev->writer_user); 199 ftl_writer_resume(&dev->writer_gc); 200 ftl_nv_cache_resume(&dev->nv_cache); 201 202 ftl_mngt_next_step(mngt); 203 } 204 205 void 206 ftl_mngt_start_core_poller(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 207 { 208 dev->core_poller = SPDK_POLLER_REGISTER(ftl_core_poller, dev, 0); 209 if (!dev->core_poller) { 210 FTL_ERRLOG(dev, "Unable to register core poller\n"); 211 ftl_mngt_fail_step(mngt); 212 return; 213 } 214 215 ftl_mngt_next_step(mngt); 216 } 217 218 void 219 ftl_mngt_stop_core_poller(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 220 { 221 dev->halt = true; 222 223 if (dev->core_poller) { 224 ftl_mngt_continue_step(mngt); 225 } else { 226 ftl_mngt_next_step(mngt); 227 } 228 } 229 230 void 231 ftl_mngt_dump_stats(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 232 { 233 ftl_dev_dump_bands(dev); 234 ftl_dev_dump_stats(dev); 235 ftl_mngt_next_step(mngt); 236 } 237 238 void 239 ftl_mngt_init_vld_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 240 { 241 struct ftl_md *valid_map_md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_VALID_MAP]; 242 243 dev->valid_map = ftl_bitmap_create(ftl_md_get_buffer(valid_map_md), 244 ftl_md_get_buffer_size(valid_map_md)); 245 if (!dev->valid_map) { 246 FTL_ERRLOG(dev, "Failed to create valid map\n"); 247 ftl_mngt_fail_step(mngt); 248 return; 249 } 250 251 ftl_mngt_next_step(mngt); 252 } 253 254 void 255 ftl_mngt_deinit_vld_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 256 { 257 if (dev->valid_map) { 258 ftl_bitmap_destroy(dev->valid_map); 259 dev->valid_map = NULL; 260 } 261 262 ftl_mngt_next_step(mngt); 263 } 264 void 265 ftl_mngt_init_unmap_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 266 { 267 uint64_t num_l2p_pages = spdk_divide_round_up(dev->num_lbas, dev->layout.l2p.lbas_in_page); 268 uint64_t map_blocks = ftl_bitmap_bits_to_blocks(num_l2p_pages); 269 270 dev->unmap_map_md = ftl_md_create(dev, 271 map_blocks, 272 0, 273 "trim_bitmap", 274 ftl_md_create_shm_flags(dev), NULL); 275 276 if (!dev->unmap_map_md) { 277 FTL_ERRLOG(dev, "Failed to create trim bitmap md\n"); 278 ftl_mngt_fail_step(mngt); 279 return; 280 } 281 282 dev->unmap_map = ftl_bitmap_create(ftl_md_get_buffer(dev->unmap_map_md), 283 ftl_md_get_buffer_size(dev->unmap_map_md)); 284 285 if (!dev->unmap_map) { 286 FTL_ERRLOG(dev, "Failed to create unmap map\n"); 287 ftl_mngt_fail_step(mngt); 288 return; 289 } 290 291 ftl_mngt_next_step(mngt); 292 } 293 294 static void 295 unmap_clear_cb(struct spdk_ftl_dev *dev, struct ftl_md *md, int status) 296 { 297 struct ftl_mngt_process *mngt = md->owner.cb_ctx; 298 299 if (status) { 300 FTL_ERRLOG(dev, "ERROR of clearing trim unmap\n"); 301 ftl_mngt_fail_step(mngt); 302 } else { 303 ftl_mngt_next_step(mngt); 304 } 305 } 306 307 void 308 ftl_mngt_unmap_clear(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 309 { 310 struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_TRIM_MD]; 311 312 md->cb = unmap_clear_cb; 313 md->owner.cb_ctx = mngt; 314 315 ftl_md_clear(md, 0, NULL); 316 } 317 318 void 319 ftl_mngt_deinit_unmap_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 320 { 321 ftl_bitmap_destroy(dev->unmap_map); 322 dev->unmap_map = NULL; 323 324 ftl_md_destroy(dev->unmap_map_md, ftl_md_destroy_shm_flags(dev)); 325 dev->unmap_map_md = NULL; 326 327 ftl_mngt_next_step(mngt); 328 } 329 330 struct ftl_mngt_property_caller_ctx { 331 struct spdk_ftl_dev *dev; 332 struct spdk_jsonrpc_request *request; 333 spdk_ftl_fn cb_fn; 334 void *cb_arg; 335 struct spdk_thread *cb_thread; 336 const char *property; 337 const char *value; 338 size_t value_size; 339 }; 340 341 static void 342 ftl_get_properties_cb(void *arg) 343 { 344 struct ftl_mngt_property_caller_ctx *cctx = arg; 345 346 cctx->cb_fn(cctx->cb_arg, 0); 347 free(cctx); 348 } 349 350 static void 351 ftl_get_properties_msg(void *arg) 352 { 353 struct ftl_mngt_property_caller_ctx *cctx = arg; 354 int rc; 355 356 ftl_property_dump(cctx->dev, cctx->request); 357 rc = spdk_thread_send_msg(cctx->cb_thread, ftl_get_properties_cb, cctx); 358 ftl_bug(rc); 359 } 360 361 int 362 spdk_ftl_get_properties(struct spdk_ftl_dev *dev, struct spdk_jsonrpc_request *request, 363 spdk_ftl_fn cb_fn, void *cb_arg) 364 { 365 int rc; 366 struct ftl_mngt_property_caller_ctx *ctx = calloc(1, sizeof(*ctx)); 367 368 if (ctx == NULL) { 369 return -ENOMEM; 370 } 371 ctx->dev = dev; 372 ctx->request = request; 373 ctx->cb_fn = cb_fn; 374 ctx->cb_arg = cb_arg; 375 ctx->cb_thread = spdk_get_thread(); 376 377 rc = spdk_thread_send_msg(dev->core_thread, ftl_get_properties_msg, ctx); 378 if (rc) { 379 free(ctx); 380 return rc; 381 } 382 383 return 0; 384 } 385 386 struct ftl_set_property_process_ctx { 387 void *value; 388 size_t value_size; 389 }; 390 391 static void 392 ftl_mngt_set_property_decode(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 393 { 394 struct ftl_set_property_process_ctx *pctx = ftl_mngt_get_process_ctx(mngt); 395 struct ftl_mngt_property_caller_ctx *cctx = ftl_mngt_get_caller_ctx(mngt); 396 397 if (ftl_property_decode(dev, cctx->property, cctx->value, cctx->value_size, 398 &pctx->value, &pctx->value_size)) { 399 ftl_mngt_fail_step(mngt); 400 } else { 401 ftl_mngt_next_step(mngt); 402 } 403 } 404 405 static void 406 ftl_mngt_set_property(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 407 { 408 struct ftl_set_property_process_ctx *pctx = ftl_mngt_get_process_ctx(mngt); 409 struct ftl_mngt_property_caller_ctx *cctx = ftl_mngt_get_caller_ctx(mngt); 410 411 if (ftl_property_set(dev, mngt, cctx->property, pctx->value, pctx->value_size)) { 412 ftl_mngt_fail_step(mngt); 413 } 414 } 415 416 static void 417 ftl_mngt_set_property_cleanup(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 418 { 419 struct ftl_set_property_process_ctx *pctx = ftl_mngt_get_process_ctx(mngt); 420 free(pctx->value); 421 pctx->value = NULL; 422 pctx->value_size = 0; 423 ftl_mngt_next_step(mngt); 424 } 425 426 static const struct ftl_mngt_process_desc desc_set_property = { 427 .name = "Set FTL property", 428 .ctx_size = sizeof(struct ftl_set_property_process_ctx), 429 .steps = { 430 { 431 .name = "Decode property", 432 .action = ftl_mngt_set_property_decode, 433 .cleanup = ftl_mngt_set_property_cleanup 434 }, 435 { 436 .name = "Set property", 437 .action = ftl_mngt_set_property, 438 .cleanup = ftl_mngt_set_property_cleanup 439 }, 440 { 441 .name = "Property setting cleanup", 442 .action = ftl_mngt_set_property_cleanup, 443 }, 444 {} 445 } 446 }; 447 448 static void 449 ftl_mngt_property_caller_cb(struct spdk_ftl_dev *dev, void *ctx, int status) 450 { 451 struct ftl_mngt_property_caller_ctx *cctx = ctx; 452 453 cctx->cb_fn(cctx->cb_arg, status); 454 free(cctx); 455 } 456 457 int 458 spdk_ftl_set_property(struct spdk_ftl_dev *dev, 459 const char *property, const char *value, size_t value_size, 460 spdk_ftl_fn cb_fn, void *cb_arg) 461 { 462 int rc; 463 struct ftl_mngt_property_caller_ctx *cctx = calloc(1, sizeof(*cctx)); 464 465 if (cctx == NULL) { 466 return -EAGAIN; 467 } 468 cctx->cb_fn = cb_fn; 469 cctx->cb_arg = cb_arg; 470 cctx->property = property; 471 cctx->value = value; 472 cctx->value_size = value_size; 473 474 rc = ftl_mngt_process_execute(dev, &desc_set_property, ftl_mngt_property_caller_cb, cctx); 475 if (rc) { 476 free(cctx); 477 } 478 479 return rc; 480 } 481