45 recovery_iter_done(struct spdk_ftl_dev *dev, struct ftl_mngt_recovery_ctx *ctx)  in recovery_iter_done()  argument
51 recovery_iter_advance(struct spdk_ftl_dev *dev, struct ftl_mngt_recovery_ctx *ctx)  in recovery_iter_advance()  argument
57 	region = ftl_layout_region_get(dev, FTL_LAYOUT_REGION_TYPE_L2P);  in recovery_iter_advance()
66 	ctx->iter.lba_first = first_block * (FTL_BLOCK_SIZE / dev->layout.l2p.addr_size);  in recovery_iter_advance()
69 	ctx->iter.lba_last = last_blocks * (FTL_BLOCK_SIZE / dev->layout.l2p.addr_size);  in recovery_iter_advance()
71 	if (ctx->iter.lba_last > dev->num_lbas) {  in recovery_iter_advance()
72 		ctx->iter.lba_last = dev->num_lbas;  in recovery_iter_advance()
77 ftl_mngt_recovery_init(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)  in ftl_mngt_recovery_init()  argument
80 	const uint64_t lbas_in_block = FTL_BLOCK_SIZE / dev->layout.l2p.addr_size;  in ftl_mngt_recovery_init()
87 	if (ftl_fast_recovery(dev)) {  in ftl_mngt_recovery_init()
97 	ftl_md_unlink(dev, FTL_L2P_CACHE_MD_NAME_L1, ftl_md_create_shm_flags(dev));  in ftl_mngt_recovery_init()
98 	ftl_md_unlink(dev, FTL_L2P_CACHE_MD_NAME_L2, ftl_md_create_shm_flags(dev));  in ftl_mngt_recovery_init()
99 	ftl_md_unlink(dev, FTL_L2P_CACHE_MD_NAME_L2_CTX, ftl_md_create_shm_flags(dev));  in ftl_mngt_recovery_init()
102 	mem_limit = dev->conf.l2p_dram_limit * MiB;  in ftl_mngt_recovery_init()
103 	mem_limit = spdk_min(mem_limit, spdk_divide_round_up(dev->num_lbas * dev->layout.l2p.addr_size,  in ftl_mngt_recovery_init()
106 	lba_limit = mem_limit / (sizeof(uint64_t) + dev->layout.l2p.addr_size);  in ftl_mngt_recovery_init()
107 	l2p_limit = lba_limit * dev->layout.l2p.addr_size;  in ftl_mngt_recovery_init()
108 	iterations = spdk_divide_round_up(dev->num_lbas, lba_limit);  in ftl_mngt_recovery_init()
117 	FTL_NOTICELOG(dev, "Recovery memory limit: %"PRIu64"MiB\n", (uint64_t)(mem_limit / MiB));  in ftl_mngt_recovery_init()
118 	FTL_NOTICELOG(dev, "L2P resident size: %"PRIu64"MiB\n", (uint64_t)(l2p_limit / MiB));  in ftl_mngt_recovery_init()
119 	FTL_NOTICELOG(dev, "Seq ID resident size: %"PRIu64"MiB\n", (uint64_t)(seq_limit / MiB));  in ftl_mngt_recovery_init()
120 	FTL_NOTICELOG(dev, "Recovery iterations: %"PRIu64"\n", iterations);  in ftl_mngt_recovery_init()
121 	dev->sb->ckpt_seq_id = 0;  in ftl_mngt_recovery_init()
124 	ctx->l2p_snippet.region = *ftl_layout_region_get(dev, FTL_LAYOUT_REGION_TYPE_L2P);  in ftl_mngt_recovery_init()
135 	ctx->l2p_snippet.md = ftl_md_create(dev, md_blocks, 0, "l2p_recovery", md_flags,  in ftl_mngt_recovery_init()
148 	recovery_iter_advance(dev, ctx);  in ftl_mngt_recovery_init()
159 ftl_mngt_recovery_deinit(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)  in ftl_mngt_recovery_deinit()  argument
171 recovery_iteration_cb(struct spdk_ftl_dev *dev, void *_ctx, int status)  in recovery_iteration_cb()  argument
175 	recovery_iter_advance(dev, ctx);  in recovery_iteration_cb()
185 ftl_mngt_recovery_run_iteration(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)  in ftl_mngt_recovery_run_iteration()  argument
189 	if (ftl_fast_recovery(dev)) {  in ftl_mngt_recovery_run_iteration()
194 	if (recovery_iter_done(dev, ctx)) {  in ftl_mngt_recovery_run_iteration()
197 		ftl_mngt_process_execute(dev, &g_desc_recovery_iteration, recovery_iteration_cb, ctx);  in ftl_mngt_recovery_run_iteration()
202 restore_band_state_cb(struct spdk_ftl_dev *dev, struct ftl_md *md, int status)  in restore_band_state_cb()  argument
207 	uint64_t num_bands = ftl_get_num_bands(dev);  in restore_band_state_cb()
217 		band = &dev->bands[i];  in restore_band_state_cb()
224 			TAILQ_REMOVE(&band->dev->shut_bands, band, queue_entry);  in restore_band_state_cb()
242 ftl_mngt_recovery_restore_band_state(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)  in ftl_mngt_recovery_restore_band_state()  argument
244 	struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_BAND_MD];  in ftl_mngt_recovery_restore_band_state()
258 ftl_mngt_recovery_walk_band_tail_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt,  in ftl_mngt_recovery_walk_band_tail_md()  argument
262 	uint64_t num_bands = ftl_get_num_bands(dev);  in ftl_mngt_recovery_walk_band_tail_md()
278 		struct ftl_band *band = &dev->bands[sctx->id];  in ftl_mngt_recovery_walk_band_tail_md()
293 			if (dev->sb->ckpt_seq_id && (band->md->close_seq_id <= dev->sb->ckpt_seq_id)) {  in ftl_mngt_recovery_walk_band_tail_md()
321 ftl_mngt_recovery_iteration_init_seq_ids(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)  in ftl_mngt_recovery_iteration_init_seq_ids()  argument
324 	struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_TRIM_MD];  in ftl_mngt_recovery_iteration_init_seq_ids()
327 	uint32_t lbas_in_page = FTL_BLOCK_SIZE / dev->layout.l2p.addr_size;  in ftl_mngt_recovery_iteration_init_seq_ids()
330 	if (dev->sb->ckpt_seq_id) {  in ftl_mngt_recovery_iteration_init_seq_ids()
331 		FTL_ERRLOG(dev, "Checkpoint recovery not supported!\n");  in ftl_mngt_recovery_iteration_init_seq_ids()
341 		assert(page_id < ftl_layout_region_get(dev, FTL_LAYOUT_REGION_TYPE_L2P)->current.blocks);  in ftl_mngt_recovery_iteration_init_seq_ids()
347 		ftl_addr_store(dev, ctx->l2p_snippet.l2p, lba_off, FTL_ADDR_INVALID);  in ftl_mngt_recovery_iteration_init_seq_ids()
354 l2p_cb(struct spdk_ftl_dev *dev, struct ftl_md *md, int status)  in l2p_cb()  argument
366 ftl_mngt_recovery_iteration_load_l2p(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)  in ftl_mngt_recovery_iteration_load_l2p()  argument
372 	FTL_NOTICELOG(dev, "L2P recovery, iteration %u\n", ctx->iter.i);  in ftl_mngt_recovery_iteration_load_l2p()
373 	FTL_NOTICELOG(dev, "Load L2P, blocks [%"PRIu64", %"PRIu64"), LBAs [%"PRIu64", %"PRIu64")\n",  in ftl_mngt_recovery_iteration_load_l2p()
385 ftl_mngt_recovery_iteration_save_l2p(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)  in ftl_mngt_recovery_iteration_save_l2p()  argument
401 	struct spdk_ftl_dev *dev = band->dev;  in restore_band_l2p_cb()  local
408 		FTL_ERRLOG(dev, "L2P band restore error, failed to read P2L map\n");  in restore_band_l2p_cb()
414 					  ftl_tail_md_num_blocks(band->dev) * FTL_BLOCK_SIZE, 0);  in restore_band_l2p_cb()
418 		FTL_ERRLOG(dev, "L2P band restore error, inconsistent P2L map CRC\n");  in restore_band_l2p_cb()
419 		ftl_stats_crc_error(dev, FTL_STATS_TYPE_MD_BASE);  in restore_band_l2p_cb()
424 	num_blks_in_band = ftl_get_num_blocks_in_band(dev);  in restore_band_l2p_cb()
433 		if (lba >= dev->num_lbas) {  in restore_band_l2p_cb()
434 			FTL_ERRLOG(dev, "L2P band restore ERROR, LBA out of range\n");  in restore_band_l2p_cb()
457 		curr_addr = ftl_addr_load(dev, pctx->l2p_snippet.l2p, lba_off);  in restore_band_l2p_cb()
460 		if (curr_addr != FTL_ADDR_INVALID && !ftl_addr_in_nvc(dev, curr_addr) && curr_addr != addr) {  in restore_band_l2p_cb()
461 			struct ftl_band *curr_band = ftl_band_from_addr(dev, curr_addr);  in restore_band_l2p_cb()
472 		ftl_addr_store(dev, pctx->l2p_snippet.l2p, lba_off, addr);  in restore_band_l2p_cb()
489 ftl_mngt_recovery_iteration_restore_band_l2p(struct spdk_ftl_dev *dev,  in ftl_mngt_recovery_iteration_restore_band_l2p()  argument
492 	ftl_mngt_recovery_walk_band_tail_md(dev, mngt, restore_band_l2p_cb);  in ftl_mngt_recovery_iteration_restore_band_l2p()
499 	struct spdk_ftl_dev *dev;  in restore_chunk_l2p_cb()  local
506 	dev = SPDK_CONTAINEROF(chunk->nv_cache, struct spdk_ftl_dev, nv_cache);  in restore_chunk_l2p_cb()
511 		ftl_stats_crc_error(dev, FTL_STATS_TYPE_MD_NV_CACHE);  in restore_chunk_l2p_cb()
523 		if (lba >= dev->num_lbas) {  in restore_chunk_l2p_cb()
524 			FTL_ERRLOG(dev, "L2P Chunk restore ERROR, LBA out of range\n");  in restore_chunk_l2p_cb()
537 		addr = ftl_addr_from_nvc_offset(dev, chunk->offset + i);  in restore_chunk_l2p_cb()
538 		ftl_addr_store(dev, pctx->l2p_snippet.l2p, lba_off, addr);  in restore_chunk_l2p_cb()
546 ftl_mngt_recovery_iteration_restore_chunk_l2p(struct spdk_ftl_dev *dev,  in ftl_mngt_recovery_iteration_restore_chunk_l2p()  argument
549 	ftl_mngt_nv_cache_restore_l2p(dev, mngt, restore_chunk_l2p_cb, ftl_mngt_get_caller_ctx(mngt));  in ftl_mngt_recovery_iteration_restore_chunk_l2p()
553 ftl_mngt_recovery_iteration_restore_valid_map(struct spdk_ftl_dev *dev,  in ftl_mngt_recovery_iteration_restore_valid_map()  argument
562 		addr = ftl_addr_load(dev, pctx->l2p_snippet.l2p, lba_off);  in ftl_mngt_recovery_iteration_restore_valid_map()
568 		if (!ftl_addr_in_nvc(dev, addr)) {  in ftl_mngt_recovery_iteration_restore_valid_map()
569 			struct ftl_band *band = ftl_band_from_addr(dev, addr);  in ftl_mngt_recovery_iteration_restore_valid_map()
573 		if (ftl_bitmap_get(dev->valid_map, addr)) {  in ftl_mngt_recovery_iteration_restore_valid_map()
578 			ftl_bitmap_set(dev->valid_map, addr);  in ftl_mngt_recovery_iteration_restore_valid_map()
586 p2l_ckpt_preprocess(struct spdk_ftl_dev *dev, struct ftl_mngt_recovery_ctx *pctx)  in p2l_ckpt_preprocess()  argument
594 		seq_id = ftl_mngt_p2l_ckpt_get_seq_id(dev, md_region);  in p2l_ckpt_preprocess()
596 		FTL_NOTICELOG(dev, "P2L ckpt_id=%d found seq_id=%"PRIu64"\n", ckpt_id, seq_id);  in p2l_ckpt_preprocess()
607 	       FTL_BLOCK_SIZE * ftl_p2l_map_num_blocks(band->dev));  in p2l_ckpt_restore_p2l()
614 			FTL_NOTICELOG(band->dev, "Restore band P2L band_id=%u ckpt_id=%d seq_id=%"  in p2l_ckpt_restore_p2l()
622 	FTL_NOTICELOG(band->dev, "Restore band P2L band_id=%u, band_seq_id=%"PRIu64" does not"  in p2l_ckpt_restore_p2l()
628 ftl_mngt_recovery_pre_process_p2l(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)  in ftl_mngt_recovery_pre_process_p2l()  argument
632 	p2l_ckpt_preprocess(dev, pctx);  in ftl_mngt_recovery_pre_process_p2l()
637 ftl_mngt_recover_seq_id(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)  in ftl_mngt_recover_seq_id()  argument
639 	ftl_recover_max_seq(dev);  in ftl_mngt_recover_seq_id()
644 ftl_mngt_recovery_open_bands_p2l(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)  in ftl_mngt_recovery_open_bands_p2l()  argument
650 		FTL_NOTICELOG(dev, "No more open bands to recover from P2L\n");  in ftl_mngt_recovery_open_bands_p2l()
666 				FTL_ERRLOG(dev, "Open band recovery ERROR, Cannot allocate P2L map\n");  in ftl_mngt_recovery_open_bands_p2l()
672 				FTL_ERRLOG(dev, "Open band recovery ERROR, Cannot restore P2L\n");  in ftl_mngt_recovery_open_bands_p2l()
678 				band->p2l_map.p2l_ckpt = ftl_p2l_ckpt_acquire_region_type(dev, band->md->p2l_md_region);  in ftl_mngt_recovery_open_bands_p2l()
680 					FTL_ERRLOG(dev, "Open band recovery ERROR, Cannot acquire P2L\n");  in ftl_mngt_recovery_open_bands_p2l()
700 	TAILQ_INSERT_TAIL(&dev->shut_bands, band, queue_entry);  in ftl_mngt_recovery_open_bands_p2l()
702 	FTL_NOTICELOG(dev, "Open band recovered, id = %u, seq id %"PRIu64", write offset %"PRIu64"\n",  in ftl_mngt_recovery_open_bands_p2l()
709 ftl_mngt_restore_valid_counters(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)  in ftl_mngt_restore_valid_counters()  argument
711 	ftl_valid_map_load_state(dev);  in ftl_mngt_restore_valid_counters()
716 trim_pending(struct spdk_ftl_dev *dev)  in trim_pending()  argument
718 	struct ftl_trim_log *log = ftl_md_get_buffer(dev->layout.md[FTL_LAYOUT_REGION_TYPE_TRIM_LOG]);  in trim_pending()
728 ftl_mngt_recover_trim_cb(struct spdk_ftl_dev *dev, struct ftl_md *md, int status)  in ftl_mngt_recover_trim_cb()  argument
739 ftl_mngt_complete_trim(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)  in ftl_mngt_complete_trim()  argument
743 	if (dev->sb_shm->trim.in_progress) {  in ftl_mngt_complete_trim()
744 		start_lba = dev->sb_shm->trim.start_lba;  in ftl_mngt_complete_trim()
745 		num_blocks = dev->sb_shm->trim.num_blocks;  in ftl_mngt_complete_trim()
746 		seq_id = dev->sb_shm->trim.seq_id;  in ftl_mngt_complete_trim()
747 		assert(seq_id <= dev->sb->seq_id);  in ftl_mngt_complete_trim()
748 		ftl_set_trim_map(dev, start_lba, num_blocks, seq_id);  in ftl_mngt_complete_trim()
751 	if (trim_pending(dev)) {  in ftl_mngt_complete_trim()
752 		struct ftl_trim_log *log = ftl_md_get_buffer(dev->layout.md[FTL_LAYOUT_REGION_TYPE_TRIM_LOG]);  in ftl_mngt_complete_trim()
753 		FTL_NOTICELOG(dev, "Incomplete trim detected lba: %"PRIu64" num_blocks: %"PRIu64"\n",  in ftl_mngt_complete_trim()
761 ftl_mngt_recover_trim_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)  in ftl_mngt_recover_trim_md()  argument
763 	struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_TRIM_MD];  in ftl_mngt_recover_trim_md()
771 ftl_mngt_recover_trim_md_persist(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)  in ftl_mngt_recover_trim_md_persist()  argument
773 	struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_TRIM_MD];  in ftl_mngt_recover_trim_md_persist()
775 	if (!trim_pending(dev)) {  in ftl_mngt_recover_trim_md_persist()
787 ftl_mngt_recover_trim_log_cb(struct spdk_ftl_dev *dev, struct ftl_md *md, int status)  in ftl_mngt_recover_trim_log_cb()  argument
798 	if (!trim_pending(dev)) {  in ftl_mngt_recover_trim_log_cb()
808 	const uint64_t lbas_in_page = FTL_BLOCK_SIZE / dev->layout.l2p.addr_size;  in ftl_mngt_recover_trim_log_cb()
812 	page = ftl_md_get_buffer(dev->layout.md[FTL_LAYOUT_REGION_TYPE_TRIM_MD]);  in ftl_mngt_recover_trim_log_cb()
815 		FTL_ERRLOG(dev, "Invalid trim log content\n");  in ftl_mngt_recover_trim_log_cb()
822 			FTL_ERRLOG(dev, "Invalid trim metadata content\n");  in ftl_mngt_recover_trim_log_cb()
833 ftl_mngt_recover_trim_log(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)  in ftl_mngt_recover_trim_log()  argument
835 	struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_TRIM_LOG];  in ftl_mngt_recover_trim_log()
843 ftl_mngt_recover_trim_persist(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)  in ftl_mngt_recover_trim_persist()  argument
845 	struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_TRIM_LOG];  in ftl_mngt_recover_trim_persist()
847 	if (!trim_pending(dev)) {  in ftl_mngt_recover_trim_persist()
859 ftl_mngt_recover_trim_log_clear(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)  in ftl_mngt_recover_trim_log_clear()  argument
861 	struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_TRIM_LOG];  in ftl_mngt_recover_trim_log_clear()
864 	if (!trim_pending(dev)) {  in ftl_mngt_recover_trim_log_clear()
923 ftl_mngt_recover_trim(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)  in ftl_mngt_recover_trim()  argument
925 	if (ftl_fast_recovery(dev)) {  in ftl_mngt_recover_trim()
934 ftl_mngt_recover_trim_shm(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)  in ftl_mngt_recover_trim_shm()  argument
940 ftl_mngt_recovery_shm_l2p(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)  in ftl_mngt_recovery_shm_l2p()  argument
942 	if (ftl_fast_recovery(dev)) {  in ftl_mngt_recovery_shm_l2p()
1099 ftl_mngt_recover(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt)  in ftl_mngt_recover()  argument