1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "ftl_core.h" 7 #include "ftl_utils.h" 8 #include "ftl_mngt.h" 9 #include "ftl_mngt_steps.h" 10 #include "ftl_band.h" 11 #include "ftl_internal.h" 12 #include "ftl_nv_cache.h" 13 #include "ftl_debug.h" 14 #include "ftl_utils.h" 15 16 void 17 ftl_mngt_check_conf(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 18 { 19 if (ftl_conf_is_valid(&dev->conf)) { 20 ftl_mngt_next_step(mngt); 21 } else { 22 ftl_mngt_fail_step(mngt); 23 } 24 } 25 26 static int 27 init_p2l_map_pool(struct spdk_ftl_dev *dev) 28 { 29 size_t p2l_pool_el_blks = spdk_divide_round_up(ftl_p2l_map_pool_elem_size(dev), FTL_BLOCK_SIZE); 30 size_t p2l_pool_buf_blks = P2L_MEMPOOL_SIZE * p2l_pool_el_blks; 31 void *p2l_pool_buf; 32 33 dev->p2l_pool_md = ftl_md_create(dev, p2l_pool_buf_blks, 0, "p2l_pool", 34 ftl_md_create_shm_flags(dev), NULL); 35 if (!dev->p2l_pool_md) { 36 return -ENOMEM; 37 } 38 39 p2l_pool_buf = ftl_md_get_buffer(dev->p2l_pool_md); 40 dev->p2l_pool = ftl_mempool_create_ext(p2l_pool_buf, P2L_MEMPOOL_SIZE, 41 p2l_pool_el_blks * FTL_BLOCK_SIZE, 42 FTL_BLOCK_SIZE); 43 if (!dev->p2l_pool) { 44 return -ENOMEM; 45 } 46 47 if (!ftl_fast_startup(dev)) { 48 ftl_mempool_initialize_ext(dev->p2l_pool); 49 } 50 51 return 0; 52 } 53 54 static int 55 init_band_md_pool(struct spdk_ftl_dev *dev) 56 { 57 dev->band_md_pool = ftl_mempool_create(P2L_MEMPOOL_SIZE, 58 sizeof(struct ftl_band_md), 59 FTL_BLOCK_SIZE, 60 SPDK_ENV_SOCKET_ID_ANY); 61 if (!dev->band_md_pool) { 62 return -ENOMEM; 63 } 64 65 return 0; 66 } 67 68 void 69 ftl_mngt_init_mem_pools(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 70 { 71 if (init_p2l_map_pool(dev)) { 72 ftl_mngt_fail_step(mngt); 73 return; 74 } 75 76 if (init_band_md_pool(dev)) { 77 ftl_mngt_fail_step(mngt); 78 return; 79 } 80 81 ftl_mngt_next_step(mngt); 82 } 83 84 void 85 ftl_mngt_deinit_mem_pools(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 86 { 87 if (dev->p2l_pool) { 88 ftl_mempool_destroy_ext(dev->p2l_pool); 89 dev->p2l_pool = NULL; 90 } 91 92 if (dev->p2l_pool_md) { 93 ftl_md_destroy(dev->p2l_pool_md, ftl_md_destroy_shm_flags(dev)); 94 dev->p2l_pool_md = NULL; 95 } 96 97 if (dev->band_md_pool) { 98 ftl_mempool_destroy(dev->band_md_pool); 99 dev->band_md_pool = NULL; 100 } 101 102 ftl_mngt_next_step(mngt); 103 } 104 105 void 106 ftl_mngt_init_reloc(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 107 { 108 dev->reloc = ftl_reloc_init(dev); 109 if (!dev->reloc) { 110 FTL_ERRLOG(dev, "Unable to initialize reloc structures\n"); 111 ftl_mngt_fail_step(mngt); 112 return; 113 } 114 115 ftl_mngt_next_step(mngt); 116 } 117 118 void 119 ftl_mngt_deinit_reloc(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 120 { 121 ftl_reloc_free(dev->reloc); 122 ftl_mngt_next_step(mngt); 123 } 124 125 void 126 ftl_mngt_init_nv_cache(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 127 { 128 if (ftl_nv_cache_init(dev)) { 129 FTL_ERRLOG(dev, "Unable to initialize persistent cache\n"); 130 ftl_mngt_fail_step(mngt); 131 return; 132 } 133 134 ftl_mngt_next_step(mngt); 135 } 136 137 void 138 ftl_mngt_deinit_nv_cache(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 139 { 140 ftl_nv_cache_deinit(dev); 141 ftl_mngt_next_step(mngt); 142 } 143 144 static void 145 user_clear_cb(struct spdk_ftl_dev *dev, struct ftl_md *md, int status) 146 { 147 struct ftl_mngt_process *mngt = md->owner.cb_ctx; 148 149 if (status) { 150 FTL_ERRLOG(ftl_mngt_get_dev(mngt), "FTL NV Cache: ERROR of clearing user cache data\n"); 151 ftl_mngt_fail_step(mngt); 152 } else { 153 ftl_mngt_next_step(mngt); 154 } 155 } 156 157 void 158 ftl_mngt_scrub_nv_cache(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 159 { 160 struct ftl_layout_region *region = &dev->layout.region[FTL_LAYOUT_REGION_TYPE_DATA_NVC]; 161 struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_DATA_NVC]; 162 union ftl_md_vss vss; 163 164 FTL_NOTICELOG(dev, "First startup needs to scrub nv cache data region, this may take some time.\n"); 165 FTL_NOTICELOG(dev, "Scrubbing %lluGiB\n", region->current.blocks * FTL_BLOCK_SIZE / GiB); 166 167 /* Need to scrub user data, so in case of dirty shutdown the recovery won't 168 * pull in data during open chunks recovery from any previous instance (since during short 169 * tests it's very likely that chunks seq_id will be in line between new head md and old VSS) 170 */ 171 md->cb = user_clear_cb; 172 md->owner.cb_ctx = mngt; 173 174 vss.version.md_version = region->current.version; 175 vss.nv_cache.lba = FTL_ADDR_INVALID; 176 ftl_md_clear(md, 0, &vss); 177 } 178 179 void 180 ftl_mngt_finalize_startup(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 181 { 182 if (ftl_bitmap_find_first_set(dev->unmap_map, 0, UINT64_MAX) != UINT64_MAX) { 183 dev->unmap_in_progress = true; 184 } 185 186 ftl_property_register(dev, "superblock_version", &dev->sb->header.version, 187 sizeof(dev->sb->header.version), NULL, NULL, 188 ftl_property_dump_uint64, NULL, NULL, false); 189 190 /* Clear the limit applications as they're incremented incorrectly by 191 * the initialization code. 192 */ 193 memset(dev->stats.limits, 0, sizeof(dev->stats.limits)); 194 dev->initialized = 1; 195 dev->sb_shm->shm_ready = true; 196 197 ftl_l2p_resume(dev); 198 ftl_reloc_resume(dev->reloc); 199 ftl_writer_resume(&dev->writer_user); 200 ftl_writer_resume(&dev->writer_gc); 201 ftl_nv_cache_resume(&dev->nv_cache); 202 203 ftl_mngt_next_step(mngt); 204 } 205 206 void 207 ftl_mngt_start_core_poller(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 208 { 209 dev->core_poller = SPDK_POLLER_REGISTER(ftl_core_poller, dev, 0); 210 if (!dev->core_poller) { 211 FTL_ERRLOG(dev, "Unable to register core poller\n"); 212 ftl_mngt_fail_step(mngt); 213 return; 214 } 215 216 ftl_mngt_next_step(mngt); 217 } 218 219 void 220 ftl_mngt_stop_core_poller(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 221 { 222 dev->halt = true; 223 224 if (dev->core_poller) { 225 ftl_mngt_continue_step(mngt); 226 } else { 227 ftl_mngt_next_step(mngt); 228 } 229 } 230 231 void 232 ftl_mngt_dump_stats(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 233 { 234 ftl_dev_dump_bands(dev); 235 ftl_dev_dump_stats(dev); 236 ftl_mngt_next_step(mngt); 237 } 238 239 void 240 ftl_mngt_init_vld_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 241 { 242 struct ftl_md *valid_map_md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_VALID_MAP]; 243 244 dev->valid_map = ftl_bitmap_create(ftl_md_get_buffer(valid_map_md), 245 ftl_md_get_buffer_size(valid_map_md)); 246 if (!dev->valid_map) { 247 FTL_ERRLOG(dev, "Failed to create valid map\n"); 248 ftl_mngt_fail_step(mngt); 249 return; 250 } 251 252 ftl_mngt_next_step(mngt); 253 } 254 255 void 256 ftl_mngt_deinit_vld_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 257 { 258 if (dev->valid_map) { 259 ftl_bitmap_destroy(dev->valid_map); 260 dev->valid_map = NULL; 261 } 262 263 ftl_mngt_next_step(mngt); 264 } 265 void 266 ftl_mngt_init_unmap_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 267 { 268 uint64_t num_l2p_pages = spdk_divide_round_up(dev->num_lbas, dev->layout.l2p.lbas_in_page); 269 uint64_t map_blocks = ftl_bitmap_bits_to_blocks(num_l2p_pages); 270 271 dev->unmap_map_md = ftl_md_create(dev, 272 map_blocks, 273 0, 274 "trim_bitmap", 275 ftl_md_create_shm_flags(dev), NULL); 276 277 if (!dev->unmap_map_md) { 278 FTL_ERRLOG(dev, "Failed to create trim bitmap md\n"); 279 ftl_mngt_fail_step(mngt); 280 return; 281 } 282 283 dev->unmap_map = ftl_bitmap_create(ftl_md_get_buffer(dev->unmap_map_md), 284 ftl_md_get_buffer_size(dev->unmap_map_md)); 285 286 if (!dev->unmap_map) { 287 FTL_ERRLOG(dev, "Failed to create unmap map\n"); 288 ftl_mngt_fail_step(mngt); 289 return; 290 } 291 292 ftl_mngt_next_step(mngt); 293 } 294 295 static void 296 unmap_clear_cb(struct spdk_ftl_dev *dev, struct ftl_md *md, int status) 297 { 298 struct ftl_mngt_process *mngt = md->owner.cb_ctx; 299 300 if (status) { 301 FTL_ERRLOG(dev, "ERROR of clearing trim unmap\n"); 302 ftl_mngt_fail_step(mngt); 303 } else { 304 ftl_mngt_next_step(mngt); 305 } 306 } 307 308 void 309 ftl_mngt_unmap_clear(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 310 { 311 struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_TRIM_MD]; 312 313 md->cb = unmap_clear_cb; 314 md->owner.cb_ctx = mngt; 315 316 ftl_md_clear(md, 0, NULL); 317 } 318 319 void 320 ftl_mngt_deinit_unmap_map(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 321 { 322 ftl_bitmap_destroy(dev->unmap_map); 323 dev->unmap_map = NULL; 324 325 ftl_md_destroy(dev->unmap_map_md, ftl_md_destroy_shm_flags(dev)); 326 dev->unmap_map_md = NULL; 327 328 ftl_mngt_next_step(mngt); 329 } 330 331 struct ftl_mngt_property_caller_ctx { 332 struct spdk_ftl_dev *dev; 333 struct spdk_jsonrpc_request *request; 334 spdk_ftl_fn cb_fn; 335 void *cb_arg; 336 struct spdk_thread *cb_thread; 337 const char *property; 338 const char *value; 339 size_t value_size; 340 }; 341 342 static void 343 ftl_get_properties_cb(void *arg) 344 { 345 struct ftl_mngt_property_caller_ctx *cctx = arg; 346 347 cctx->cb_fn(cctx->cb_arg, 0); 348 free(cctx); 349 } 350 351 static void 352 ftl_get_properties_msg(void *arg) 353 { 354 struct ftl_mngt_property_caller_ctx *cctx = arg; 355 int rc; 356 357 ftl_property_dump(cctx->dev, cctx->request); 358 rc = spdk_thread_send_msg(cctx->cb_thread, ftl_get_properties_cb, cctx); 359 ftl_bug(rc); 360 } 361 362 int 363 spdk_ftl_get_properties(struct spdk_ftl_dev *dev, struct spdk_jsonrpc_request *request, 364 spdk_ftl_fn cb_fn, void *cb_arg) 365 { 366 int rc; 367 struct ftl_mngt_property_caller_ctx *ctx = calloc(1, sizeof(*ctx)); 368 369 if (ctx == NULL) { 370 return -ENOMEM; 371 } 372 ctx->dev = dev; 373 ctx->request = request; 374 ctx->cb_fn = cb_fn; 375 ctx->cb_arg = cb_arg; 376 ctx->cb_thread = spdk_get_thread(); 377 378 rc = spdk_thread_send_msg(dev->core_thread, ftl_get_properties_msg, ctx); 379 if (rc) { 380 free(ctx); 381 return rc; 382 } 383 384 return 0; 385 } 386 387 struct ftl_set_property_process_ctx { 388 void *value; 389 size_t value_size; 390 }; 391 392 static void 393 ftl_mngt_set_property_decode(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 394 { 395 struct ftl_set_property_process_ctx *pctx = ftl_mngt_get_process_ctx(mngt); 396 struct ftl_mngt_property_caller_ctx *cctx = ftl_mngt_get_caller_ctx(mngt); 397 398 if (ftl_property_decode(dev, cctx->property, cctx->value, cctx->value_size, 399 &pctx->value, &pctx->value_size)) { 400 ftl_mngt_fail_step(mngt); 401 } else { 402 ftl_mngt_next_step(mngt); 403 } 404 } 405 406 static void 407 ftl_mngt_set_property(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 408 { 409 struct ftl_set_property_process_ctx *pctx = ftl_mngt_get_process_ctx(mngt); 410 struct ftl_mngt_property_caller_ctx *cctx = ftl_mngt_get_caller_ctx(mngt); 411 412 if (ftl_property_set(dev, mngt, cctx->property, pctx->value, pctx->value_size)) { 413 ftl_mngt_fail_step(mngt); 414 } 415 } 416 417 static void 418 ftl_mngt_set_property_cleanup(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 419 { 420 struct ftl_set_property_process_ctx *pctx = ftl_mngt_get_process_ctx(mngt); 421 free(pctx->value); 422 pctx->value = NULL; 423 pctx->value_size = 0; 424 ftl_mngt_next_step(mngt); 425 } 426 427 static const struct ftl_mngt_process_desc desc_set_property = { 428 .name = "Set FTL property", 429 .ctx_size = sizeof(struct ftl_set_property_process_ctx), 430 .steps = { 431 { 432 .name = "Decode property", 433 .action = ftl_mngt_set_property_decode, 434 .cleanup = ftl_mngt_set_property_cleanup 435 }, 436 { 437 .name = "Set property", 438 .action = ftl_mngt_set_property, 439 .cleanup = ftl_mngt_set_property_cleanup 440 }, 441 { 442 .name = "Property setting cleanup", 443 .action = ftl_mngt_set_property_cleanup, 444 }, 445 {} 446 } 447 }; 448 449 static void 450 ftl_mngt_property_caller_cb(struct spdk_ftl_dev *dev, void *ctx, int status) 451 { 452 struct ftl_mngt_property_caller_ctx *cctx = ctx; 453 454 cctx->cb_fn(cctx->cb_arg, status); 455 free(cctx); 456 } 457 458 int 459 spdk_ftl_set_property(struct spdk_ftl_dev *dev, 460 const char *property, const char *value, size_t value_size, 461 spdk_ftl_fn cb_fn, void *cb_arg) 462 { 463 int rc; 464 struct ftl_mngt_property_caller_ctx *cctx = calloc(1, sizeof(*cctx)); 465 466 if (cctx == NULL) { 467 return -EAGAIN; 468 } 469 cctx->cb_fn = cb_fn; 470 cctx->cb_arg = cb_arg; 471 cctx->property = property; 472 cctx->value = value; 473 cctx->value_size = value_size; 474 475 rc = ftl_mngt_process_execute(dev, &desc_set_property, ftl_mngt_property_caller_cb, cctx); 476 if (rc) { 477 free(cctx); 478 } 479 480 return rc; 481 } 482