1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2023 Solidigm All Rights Reserved 3 * Copyright (C) 2022 Intel Corporation. 4 * All rights reserved. 5 */ 6 7 #ifndef FTL_NV_CACHE_H 8 #define FTL_NV_CACHE_H 9 10 #include "spdk/stdinc.h" 11 #include "spdk/crc32.h" 12 13 #include "ftl_io.h" 14 #include "ftl_utils.h" 15 #include "nvc/ftl_nvc_dev.h" 16 17 /* 18 * FTL non volatile cache is divided into groups of blocks called chunks. 19 * Size of each chunk is multiple of xfer size plus additional metadata. 20 * For each block associated lba is stored in metadata. Cache space is 21 * written chunk by chunk sequentially. When number of free chunks reaches 22 * some threshold oldest chunks are moved from cache to backend storage to 23 * create space for new user data. 24 */ 25 26 #define FTL_NVC_VERSION_0 0 27 #define FTL_NVC_VERSION_1 1 28 29 #define FTL_NVC_VERSION_CURRENT FTL_NVC_VERSION_1 30 31 #define FTL_NV_CACHE_NUM_COMPACTORS 8 32 33 /* 34 * Parameters controlling nv cache write throttling. 35 * 36 * The write throttle limit value is calculated as follows: 37 * limit = compaction_average_bw * (1.0 + modifier) 38 * 39 * The modifier depends on the number of free chunks vs the configured threshold. Its value is 40 * zero if the number of free chunks is at the threshold, negative if below and positive if above. 41 */ 42 43 /* Interval in milliseconds between write throttle updates. */ 44 #define FTL_NV_CACHE_THROTTLE_INTERVAL_MS 20 45 /* Throttle modifier proportional gain */ 46 #define FTL_NV_CACHE_THROTTLE_MODIFIER_KP 20 47 /* Min and max modifier values */ 48 #define FTL_NV_CACHE_THROTTLE_MODIFIER_MIN -0.8 49 #define FTL_NV_CACHE_THROTTLE_MODIFIER_MAX 0.5 50 51 struct ftl_nvcache_restore; 52 typedef void (*ftl_nv_cache_restore_fn)(struct ftl_nvcache_restore *, int, void *cb_arg); 53 54 enum ftl_chunk_state { 55 FTL_CHUNK_STATE_FREE, 56 FTL_CHUNK_STATE_OPEN, 57 FTL_CHUNK_STATE_CLOSED, 58 FTL_CHUNK_STATE_MAX 59 }; 60 61 struct ftl_nv_cache_chunk_md { 62 /* Sequence id of writing */ 63 uint64_t seq_id; 64 65 /* Sequence ID when chunk was closed */ 66 uint64_t close_seq_id; 67 68 /* Current lba to write */ 69 uint32_t write_pointer; 70 71 /* Number of blocks written */ 72 uint32_t blocks_written; 73 74 /* Number of skipped block (case when IO size is greater than blocks left in chunk) */ 75 uint32_t blocks_skipped; 76 77 /* Next block to be compacted */ 78 uint32_t read_pointer; 79 80 /* Number of compacted (both valid and invalid) blocks */ 81 uint32_t blocks_compacted; 82 83 /* Chunk state */ 84 enum ftl_chunk_state state; 85 86 /* CRC32 checksum of the associated P2L map when chunk is in closed state */ 87 uint32_t p2l_map_checksum; 88 89 /* Reserved */ 90 uint8_t reserved[4052]; 91 } __attribute__((packed)); 92 93 #define FTL_NV_CACHE_CHUNK_MD_SIZE sizeof(struct ftl_nv_cache_chunk_md) 94 SPDK_STATIC_ASSERT(FTL_NV_CACHE_CHUNK_MD_SIZE == FTL_BLOCK_SIZE, 95 "FTL NV Chunk metadata size is invalid"); 96 97 struct ftl_nv_cache_chunk { 98 struct ftl_nv_cache *nv_cache; 99 100 struct ftl_nv_cache_chunk_md *md; 101 102 /* Offset from start lba of the cache */ 103 uint64_t offset; 104 105 /* P2L map */ 106 struct ftl_p2l_map p2l_map; 107 108 /* Metadata request */ 109 struct ftl_basic_rq metadata_rq; 110 111 TAILQ_ENTRY(ftl_nv_cache_chunk) entry; 112 113 /* This flag is used to indicate chunk is used in recovery */ 114 bool recovery; 115 116 /* Compaction start time */ 117 uint64_t compaction_start_tsc; 118 119 /* Compaction duration */ 120 uint64_t compaction_length_tsc; 121 122 /* For writing metadata */ 123 struct ftl_md_io_entry_ctx md_persist_entry_ctx; 124 }; 125 126 struct ftl_nv_cache_compactor { 127 struct ftl_nv_cache *nv_cache; 128 struct ftl_rq *rq; 129 TAILQ_ENTRY(ftl_nv_cache_compactor) entry; 130 struct spdk_bdev_io_wait_entry bdev_io_wait; 131 }; 132 133 struct ftl_nv_cache { 134 /* Flag indicating halt request */ 135 bool halt; 136 137 /* NV cache device descriptor */ 138 const struct ftl_nv_cache_device_desc *nvc_desc; 139 140 /* Write buffer cache bdev */ 141 struct spdk_bdev_desc *bdev_desc; 142 143 /* Persistent cache IO channel */ 144 struct spdk_io_channel *cache_ioch; 145 146 /* Metadata pool */ 147 struct ftl_mempool *md_pool; 148 149 /* P2L map memory pool */ 150 struct ftl_mempool *p2l_pool; 151 152 /* Chunk md memory pool */ 153 struct ftl_mempool *chunk_md_pool; 154 155 /* Chunk md memory pool for freeing chunks */ 156 struct ftl_mempool *free_chunk_md_pool; 157 158 /* Block Metadata size */ 159 uint64_t md_size; 160 161 /* NV cache metadata object handle */ 162 struct ftl_md *md; 163 164 /* Number of blocks in chunk */ 165 uint64_t chunk_blocks; 166 167 /* Number of blocks in tail md per chunk */ 168 uint64_t tail_md_chunk_blocks; 169 170 /* Number of chunks */ 171 uint64_t chunk_count; 172 173 /* Current processed chunk */ 174 struct ftl_nv_cache_chunk *chunk_current; 175 176 /* Free chunks list */ 177 TAILQ_HEAD(, ftl_nv_cache_chunk) chunk_free_list; 178 uint64_t chunk_free_count; 179 180 /* Open chunks list */ 181 TAILQ_HEAD(, ftl_nv_cache_chunk) chunk_open_list; 182 uint64_t chunk_open_count; 183 184 /* Full chunks list */ 185 TAILQ_HEAD(, ftl_nv_cache_chunk) chunk_full_list; 186 uint64_t chunk_full_count; 187 188 /* Chunks being compacted */ 189 TAILQ_HEAD(, ftl_nv_cache_chunk) chunk_comp_list; 190 uint64_t chunk_comp_count; 191 192 /* Chunks being freed */ 193 TAILQ_HEAD(, ftl_nv_cache_chunk) needs_free_persist_list; 194 uint64_t chunk_free_persist_count; 195 196 TAILQ_HEAD(, ftl_nv_cache_compactor) compactor_list; 197 uint64_t compaction_active_count; 198 uint64_t chunk_compaction_threshold; 199 200 struct ftl_nv_cache_chunk *chunks; 201 202 uint64_t last_seq_id; 203 204 uint64_t chunk_free_target; 205 206 /* Simple moving average of recent compaction velocity values */ 207 double compaction_sma; 208 209 #define FTL_NV_CACHE_COMPACTION_SMA_N (FTL_NV_CACHE_NUM_COMPACTORS * 2) 210 /* Circular buffer holding values for calculating compaction SMA */ 211 struct compaction_bw_stats { 212 double buf[FTL_NV_CACHE_COMPACTION_SMA_N]; 213 ptrdiff_t first; 214 size_t count; 215 double sum; 216 } compaction_recent_bw; 217 218 struct { 219 uint64_t interval_tsc; 220 uint64_t start_tsc; 221 uint64_t blocks_submitted; 222 uint64_t blocks_submitted_limit; 223 } throttle; 224 }; 225 226 int ftl_nv_cache_init(struct spdk_ftl_dev *dev); 227 void ftl_nv_cache_deinit(struct spdk_ftl_dev *dev); 228 bool ftl_nv_cache_write(struct ftl_io *io); 229 void ftl_nv_cache_fill_md(struct ftl_io *io); 230 int ftl_nv_cache_read(struct ftl_io *io, ftl_addr addr, uint32_t num_blocks, 231 spdk_bdev_io_completion_cb cb, void *cb_arg); 232 bool ftl_nv_cache_throttle(struct spdk_ftl_dev *dev); 233 void ftl_nv_cache_process(struct spdk_ftl_dev *dev); 234 235 void ftl_chunk_map_set_lba(struct ftl_nv_cache_chunk *chunk, 236 uint64_t offset, uint64_t lba); 237 uint64_t ftl_chunk_map_get_lba(struct ftl_nv_cache_chunk *chunk, uint64_t offset); 238 239 void ftl_nv_cache_set_addr(struct spdk_ftl_dev *dev, uint64_t lba, ftl_addr addr); 240 241 int ftl_nv_cache_save_state(struct ftl_nv_cache *nv_cache); 242 243 int ftl_nv_cache_load_state(struct ftl_nv_cache *nv_cache); 244 245 void ftl_nv_cache_halt(struct ftl_nv_cache *nv_cache); 246 247 int ftl_nv_cache_chunks_busy(struct ftl_nv_cache *nv_cache); 248 249 static inline void 250 ftl_nv_cache_resume(struct ftl_nv_cache *nv_cache) 251 { 252 nv_cache->halt = false; 253 } 254 255 bool ftl_nv_cache_is_halted(struct ftl_nv_cache *nv_cache); 256 257 size_t ftl_nv_cache_chunk_tail_md_num_blocks(const struct ftl_nv_cache *nv_cache); 258 259 uint64_t chunk_tail_md_offset(struct ftl_nv_cache *nv_cache); 260 /** 261 * @brief Iterates over NV caches chunks and returns the max open and closed sequence id 262 * 263 * @param nv_cache FLT NV cache 264 * @param[out] open_seq_id Max detected open sequence id 265 * @param[out] close_seq_id Max detected close sequence id 266 */ 267 void ftl_nv_cache_get_max_seq_id(struct ftl_nv_cache *nv_cache, uint64_t *open_seq_id, 268 uint64_t *close_seq_id); 269 270 void ftl_mngt_nv_cache_restore_chunk_state(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); 271 272 void ftl_mngt_nv_cache_recover_open_chunk(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt); 273 274 typedef int (*ftl_chunk_md_cb)(struct ftl_nv_cache_chunk *chunk, void *cntx); 275 276 void ftl_mngt_nv_cache_restore_l2p(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt, 277 ftl_chunk_md_cb cb, void *cb_ctx); 278 279 struct ftl_nv_cache_chunk *ftl_nv_cache_get_chunk_from_addr(struct spdk_ftl_dev *dev, 280 ftl_addr addr); 281 282 uint64_t ftl_nv_cache_acquire_trim_seq_id(struct ftl_nv_cache *nv_cache); 283 284 #endif /* FTL_NV_CACHE_H */ 285