1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "ftl_core.h" 7 #include "ftl_utils.h" 8 #include "ftl_mngt.h" 9 #include "ftl_mngt_steps.h" 10 #include "ftl_band.h" 11 #include "ftl_internal.h" 12 #include "ftl_nv_cache.h" 13 #include "ftl_debug.h" 14 #include "ftl_utils.h" 15 16 void 17 ftl_mngt_check_conf(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 18 { 19 if (ftl_conf_is_valid(&dev->conf)) { 20 ftl_mngt_next_step(mngt); 21 } else { 22 ftl_mngt_fail_step(mngt); 23 } 24 } 25 26 static int 27 init_p2l_map_pool(struct spdk_ftl_dev *dev) 28 { 29 size_t p2l_pool_el_blks = spdk_divide_round_up(ftl_p2l_map_pool_elem_size(dev), FTL_BLOCK_SIZE); 30 size_t p2l_pool_buf_blks = P2L_MEMPOOL_SIZE * p2l_pool_el_blks; 31 void *p2l_pool_buf; 32 33 dev->p2l_pool_md = ftl_md_create(dev, p2l_pool_buf_blks, 0, "p2l_pool", 34 ftl_md_create_shm_flags(dev), NULL); 35 if (!dev->p2l_pool_md) { 36 return -ENOMEM; 37 } 38 39 p2l_pool_buf = ftl_md_get_buffer(dev->p2l_pool_md); 40 dev->p2l_pool = ftl_mempool_create_ext(p2l_pool_buf, P2L_MEMPOOL_SIZE, 41 p2l_pool_el_blks * FTL_BLOCK_SIZE, 42 FTL_BLOCK_SIZE); 43 if (!dev->p2l_pool) { 44 return -ENOMEM; 45 } 46 47 if (!ftl_fast_startup(dev)) { 48 ftl_mempool_initialize_ext(dev->p2l_pool); 49 } 50 51 return 0; 52 } 53 54 static int 55 init_band_md_pool(struct spdk_ftl_dev *dev) 56 { 57 dev->band_md_pool = ftl_mempool_create(P2L_MEMPOOL_SIZE, 58 sizeof(struct ftl_band_md), 59 FTL_BLOCK_SIZE, 60 SPDK_ENV_NUMA_ID_ANY); 61 if (!dev->band_md_pool) { 62 return -ENOMEM; 63 } 64 65 return 0; 66 } 67 68 void 69 ftl_mngt_init_mem_pools(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 70 { 71 if (init_p2l_map_pool(dev)) { 72 ftl_mngt_fail_step(mngt); 73 return; 74 } 75 76 if (init_band_md_pool(dev)) { 77 ftl_mngt_fail_step(mngt); 78 return; 79 } 80 81 ftl_mngt_next_step(mngt); 82 } 83 84 void 85 ftl_mngt_deinit_mem_pools(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 86 { 87 if (dev->p2l_pool) { 88 ftl_mempool_destroy_ext(dev->p2l_pool); 89 dev->p2l_pool = NULL; 90 } 91 92 if (dev->p2l_pool_md) { 93 ftl_md_destroy(dev->p2l_pool_md, ftl_md_destroy_shm_flags(dev)); 94 dev->p2l_pool_md = NULL; 95 } 96 97 if (dev->band_md_pool) { 98 ftl_mempool_destroy(dev->band_md_pool); 99 dev->band_md_pool = NULL; 100 } 101 102 ftl_mngt_next_step(mngt); 103 } 104 105 void 106 ftl_mngt_init_reloc(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 107 { 108 dev->reloc = ftl_reloc_init(dev); 109 if (!dev->reloc) { 110 FTL_ERRLOG(dev, "Unable to initialize reloc structures\n"); 111 ftl_mngt_fail_step(mngt); 112 return; 113 } 114 115 ftl_mngt_next_step(mngt); 116 } 117 118 void 119 ftl_mngt_deinit_reloc(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 120 { 121 ftl_reloc_free(dev->reloc); 122 ftl_mngt_next_step(mngt); 123 } 124 125 void 126 ftl_mngt_init_nv_cache(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 127 { 128 if (ftl_nv_cache_init(dev)) { 129 FTL_ERRLOG(dev, "Unable to initialize persistent cache\n"); 130 ftl_mngt_fail_step(mngt); 131 return; 132 } 133 134 ftl_mngt_next_step(mngt); 135 } 136 137 void 138 ftl_mngt_deinit_nv_cache(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 139 { 140 ftl_nv_cache_deinit(dev); 141 ftl_mngt_next_step(mngt); 142 } 143 144 static void 145 user_clear_cb(struct spdk_ftl_dev *dev, void *cb_ctx, int status) 146 { 147 struct ftl_mngt_process *mngt = cb_ctx; 148 149 if (status) { 150 FTL_ERRLOG(ftl_mngt_get_dev(mngt), "FTL NV Cache: ERROR of clearing user cache data\n"); 151 ftl_mngt_fail_step(mngt); 152 } else { 153 ftl_mngt_next_step(mngt); 154 } 155 } 156 157 void 158 ftl_mngt_scrub_nv_cache(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 159 { 160 bool is_first_start = (dev->conf.mode & SPDK_FTL_MODE_CREATE) != 0; 161 bool is_major_upgrade = dev->sb->clean == 1 && dev->sb_shm->shm_clean == 0 && 162 dev->sb->upgrade_ready == 1; 163 164 if (is_first_start || is_major_upgrade) { 165 FTL_NOTICELOG(dev, "NV cache data region needs scrubbing, this may take a while.\n"); 166 FTL_NOTICELOG(dev, "Scrubbing %"PRIu64" chunks\n", dev->layout.nvc.chunk_count); 167 168 /* Need to scrub user data, so in case of dirty shutdown the recovery won't 169 * pull in data during open chunks recovery from any previous instance (since during short 170 * tests it's very likely that chunks seq_id will be in line between new head md and old VSS) 171 */ 172 ftl_nv_cache_scrub(dev, user_clear_cb, mngt); 173 } else { 174 ftl_mngt_skip_step(mngt); 175 } 176 } 177 178 void 179 ftl_mngt_finalize_startup(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 180 { 181 if (ftl_bitmap_find_first_set(dev->trim_map, 0, UINT64_MAX) != UINT64_MAX) { 182 dev->trim_in_progress = true; 183 } 184 185 ftl_property_register(dev, "superblock_version", &dev->sb->header.version, 186 sizeof(dev->sb->header.version), NULL, NULL, 187 ftl_property_dump_uint64, NULL, NULL, false); 188 189 /* Clear the limit applications as they're incremented incorrectly by 190 * the initialization code. 191 */ 192 memset(dev->stats.limits, 0, sizeof(dev->stats.limits)); 193 dev->initialized = 1; 194 dev->sb_shm->shm_ready = true; 195 196 ftl_l2p_resume(dev); 197 ftl_reloc_resume(dev->reloc); 198 ftl_writer_resume(&dev->writer_user); 199 ftl_writer_resume(&dev->writer_gc); 200 ftl_nv_cache_resume(&dev->nv_cache); 201 202 ftl_mngt_next_step(mngt); 203 } 204 205 void 206 ftl_mngt_start_core_poller(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 207 { 208 dev->core_poller = SPDK_POLLER_REGISTER(ftl_core_poller, dev, 0); 209 if (!dev->core_poller) { 210 FTL_ERRLOG(dev, "Unable to register core poller\n"); 211 ftl_mngt_fail_step(mngt); 212 return; 213 } 214 215 ftl_mngt_next_step(mngt); 216 } 217 218 void 219 ftl_mngt_stop_core_poller(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 220 { 221 dev->halt = true; 222 223 if (dev->core_poller) { 224 ftl_mngt_continue_step(mngt); 225 } else { 226 ftl_mngt_next_step(mngt); 227 } 228 } 229 230 void 231 ftl_mngt_dump_stats(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 232 { 233 ftl_dev_dump_bands(dev); 234 ftl_dev_dump_stats(dev); 235 ftl_mngt_next_step(mngt); 236 } 237 238 void 239 ftl_mngt_init_vld_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 240 { 241 struct ftl_md *valid_map_md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_VALID_MAP]; 242 243 dev->valid_map = ftl_bitmap_create(ftl_md_get_buffer(valid_map_md), 244 ftl_md_get_buffer_size(valid_map_md)); 245 if (!dev->valid_map) { 246 FTL_ERRLOG(dev, "Failed to create valid map\n"); 247 ftl_mngt_fail_step(mngt); 248 return; 249 } 250 251 ftl_mngt_next_step(mngt); 252 } 253 254 void 255 ftl_mngt_deinit_vld_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 256 { 257 if (dev->valid_map) { 258 ftl_bitmap_destroy(dev->valid_map); 259 dev->valid_map = NULL; 260 } 261 262 ftl_mngt_next_step(mngt); 263 } 264 void 265 ftl_mngt_init_trim_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 266 { 267 uint64_t num_l2p_pages = spdk_divide_round_up(dev->num_lbas, dev->layout.l2p.lbas_in_page); 268 uint64_t map_blocks = ftl_bitmap_bits_to_blocks(num_l2p_pages); 269 270 dev->trim_map_md = ftl_md_create(dev, 271 map_blocks, 272 0, 273 "trim_bitmap", 274 ftl_md_create_shm_flags(dev), NULL); 275 276 if (!dev->trim_map_md) { 277 FTL_ERRLOG(dev, "Failed to create trim bitmap md\n"); 278 ftl_mngt_fail_step(mngt); 279 return; 280 } 281 282 dev->trim_map = ftl_bitmap_create(ftl_md_get_buffer(dev->trim_map_md), 283 ftl_md_get_buffer_size(dev->trim_map_md)); 284 285 if (!dev->trim_map) { 286 FTL_ERRLOG(dev, "Failed to create trim map\n"); 287 ftl_mngt_fail_step(mngt); 288 return; 289 } 290 291 ftl_mngt_next_step(mngt); 292 } 293 294 static void 295 trim_clear_cb(struct spdk_ftl_dev *dev, struct ftl_md *md, int status) 296 { 297 struct ftl_mngt_process *mngt = md->owner.cb_ctx; 298 299 if (status) { 300 ftl_mngt_fail_step(mngt); 301 } else { 302 ftl_mngt_next_step(mngt); 303 } 304 } 305 306 void 307 ftl_mngt_trim_metadata_clear(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 308 { 309 struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_TRIM_MD]; 310 311 md->cb = trim_clear_cb; 312 md->owner.cb_ctx = mngt; 313 ftl_md_clear(md, 0, NULL); 314 } 315 316 void 317 ftl_mngt_trim_log_clear(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 318 { 319 struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_TRIM_LOG]; 320 321 md->cb = trim_clear_cb; 322 md->owner.cb_ctx = mngt; 323 ftl_md_clear(md, 0, NULL); 324 } 325 326 void 327 ftl_mngt_deinit_trim_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 328 { 329 ftl_bitmap_destroy(dev->trim_map); 330 dev->trim_map = NULL; 331 332 ftl_md_destroy(dev->trim_map_md, ftl_md_destroy_shm_flags(dev)); 333 dev->trim_map_md = NULL; 334 335 ftl_mngt_next_step(mngt); 336 } 337 338 struct ftl_mngt_property_caller_ctx { 339 struct spdk_ftl_dev *dev; 340 struct spdk_jsonrpc_request *request; 341 spdk_ftl_fn cb_fn; 342 void *cb_arg; 343 struct spdk_thread *cb_thread; 344 const char *property; 345 const char *value; 346 size_t value_size; 347 }; 348 349 static void 350 ftl_get_properties_cb(void *arg) 351 { 352 struct ftl_mngt_property_caller_ctx *cctx = arg; 353 354 cctx->cb_fn(cctx->cb_arg, 0); 355 free(cctx); 356 } 357 358 static void 359 ftl_get_properties_msg(void *arg) 360 { 361 struct ftl_mngt_property_caller_ctx *cctx = arg; 362 int rc; 363 364 ftl_property_dump(cctx->dev, cctx->request); 365 rc = spdk_thread_send_msg(cctx->cb_thread, ftl_get_properties_cb, cctx); 366 ftl_bug(rc); 367 } 368 369 int 370 spdk_ftl_get_properties(struct spdk_ftl_dev *dev, struct spdk_jsonrpc_request *request, 371 spdk_ftl_fn cb_fn, void *cb_arg) 372 { 373 int rc; 374 struct ftl_mngt_property_caller_ctx *ctx = calloc(1, sizeof(*ctx)); 375 376 if (ctx == NULL) { 377 return -ENOMEM; 378 } 379 ctx->dev = dev; 380 ctx->request = request; 381 ctx->cb_fn = cb_fn; 382 ctx->cb_arg = cb_arg; 383 ctx->cb_thread = spdk_get_thread(); 384 385 rc = spdk_thread_send_msg(dev->core_thread, ftl_get_properties_msg, ctx); 386 if (rc) { 387 free(ctx); 388 return rc; 389 } 390 391 return 0; 392 } 393 394 struct ftl_set_property_process_ctx { 395 void *value; 396 size_t value_size; 397 }; 398 399 static void 400 ftl_mngt_set_property_decode(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 401 { 402 struct ftl_set_property_process_ctx *pctx = ftl_mngt_get_process_ctx(mngt); 403 struct ftl_mngt_property_caller_ctx *cctx = ftl_mngt_get_caller_ctx(mngt); 404 405 if (ftl_property_decode(dev, cctx->property, cctx->value, cctx->value_size, 406 &pctx->value, &pctx->value_size)) { 407 ftl_mngt_fail_step(mngt); 408 } else { 409 ftl_mngt_next_step(mngt); 410 } 411 } 412 413 static void 414 ftl_mngt_set_property(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 415 { 416 struct ftl_set_property_process_ctx *pctx = ftl_mngt_get_process_ctx(mngt); 417 struct ftl_mngt_property_caller_ctx *cctx = ftl_mngt_get_caller_ctx(mngt); 418 419 if (ftl_property_set(dev, mngt, cctx->property, pctx->value, pctx->value_size)) { 420 ftl_mngt_fail_step(mngt); 421 } 422 } 423 424 static void 425 ftl_mngt_set_property_cleanup(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 426 { 427 struct ftl_set_property_process_ctx *pctx = ftl_mngt_get_process_ctx(mngt); 428 free(pctx->value); 429 pctx->value = NULL; 430 pctx->value_size = 0; 431 ftl_mngt_next_step(mngt); 432 } 433 434 static const struct ftl_mngt_process_desc desc_set_property = { 435 .name = "Set FTL property", 436 .ctx_size = sizeof(struct ftl_set_property_process_ctx), 437 .steps = { 438 { 439 .name = "Decode property", 440 .action = ftl_mngt_set_property_decode, 441 .cleanup = ftl_mngt_set_property_cleanup 442 }, 443 { 444 .name = "Set property", 445 .action = ftl_mngt_set_property, 446 .cleanup = ftl_mngt_set_property_cleanup 447 }, 448 { 449 .name = "Property setting cleanup", 450 .action = ftl_mngt_set_property_cleanup, 451 }, 452 {} 453 } 454 }; 455 456 static void 457 ftl_mngt_property_caller_cb(struct spdk_ftl_dev *dev, void *ctx, int status) 458 { 459 struct ftl_mngt_property_caller_ctx *cctx = ctx; 460 461 cctx->cb_fn(cctx->cb_arg, status); 462 free(cctx); 463 } 464 465 int 466 spdk_ftl_set_property(struct spdk_ftl_dev *dev, 467 const char *property, const char *value, size_t value_size, 468 spdk_ftl_fn cb_fn, void *cb_arg) 469 { 470 int rc; 471 struct ftl_mngt_property_caller_ctx *cctx = calloc(1, sizeof(*cctx)); 472 473 if (cctx == NULL) { 474 return -EAGAIN; 475 } 476 cctx->cb_fn = cb_fn; 477 cctx->cb_arg = cb_arg; 478 cctx->property = property; 479 cctx->value = value; 480 cctx->value_size = value_size; 481 482 rc = ftl_mngt_process_execute(dev, &desc_set_property, ftl_mngt_property_caller_cb, cctx); 483 if (rc) { 484 free(cctx); 485 } 486 487 return rc; 488 } 489