1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #ifndef FTL_CORE_H 7 #define FTL_CORE_H 8 9 #include "spdk/stdinc.h" 10 #include "spdk/uuid.h" 11 #include "spdk/thread.h" 12 #include "spdk/util.h" 13 #include "spdk/log.h" 14 #include "spdk/likely.h" 15 #include "spdk/queue.h" 16 #include "spdk/ftl.h" 17 #include "spdk/bdev.h" 18 #include "spdk/bdev_zone.h" 19 20 #include "ftl_addr.h" 21 #include "ftl_io.h" 22 #include "ftl_trace.h" 23 24 #ifdef SPDK_CONFIG_PMDK 25 #include "libpmem.h" 26 #endif /* SPDK_CONFIG_PMDK */ 27 28 struct spdk_ftl_dev; 29 struct ftl_band; 30 struct ftl_zone; 31 struct ftl_io; 32 struct ftl_restore; 33 struct ftl_wptr; 34 struct ftl_flush; 35 struct ftl_reloc; 36 struct ftl_anm_event; 37 struct ftl_band_flush; 38 39 struct ftl_stats { 40 /* Number of writes scheduled directly by the user */ 41 uint64_t write_user; 42 43 /* Total number of writes */ 44 uint64_t write_total; 45 46 /* Traces */ 47 struct ftl_trace trace; 48 49 /* Number of limits applied */ 50 uint64_t limits[SPDK_FTL_LIMIT_MAX]; 51 }; 52 53 struct ftl_global_md { 54 /* Device instance */ 55 struct spdk_uuid uuid; 56 /* Size of the l2p table */ 57 uint64_t num_lbas; 58 }; 59 60 struct ftl_nv_cache { 61 /* Write buffer cache bdev */ 62 struct spdk_bdev_desc *bdev_desc; 63 /* Write pointer */ 64 uint64_t current_addr; 65 /* Number of available blocks left */ 66 uint64_t num_available; 67 /* Maximum number of blocks */ 68 uint64_t num_data_blocks; 69 /* 70 * Phase of the current cycle of writes. Each time whole cache area is filled, the phase is 71 * advanced. Current phase is saved in every IO's metadata, as well as in the header saved 72 * in the first sector. By looking at the phase of each block, it's possible to find the 73 * oldest block and replay the order of the writes when recovering the data from the cache. 74 */ 75 unsigned int phase; 76 /* Indicates that the data can be written to the cache */ 77 bool ready; 78 /* Metadata pool */ 79 struct spdk_mempool *md_pool; 80 /* DMA buffer for writing the header */ 81 void *dma_buf; 82 /* Cache lock */ 83 pthread_spinlock_t lock; 84 }; 85 86 struct ftl_batch { 87 /* Queue of write buffer entries, can reach up to xfer_size entries */ 88 TAILQ_HEAD(, ftl_wbuf_entry) entries; 89 /* Number of entries in the queue above */ 90 uint32_t num_entries; 91 /* Index within spdk_ftl_dev.batch_array */ 92 uint32_t index; 93 struct iovec *iov; 94 void *metadata; 95 TAILQ_ENTRY(ftl_batch) tailq; 96 }; 97 98 struct spdk_ftl_dev { 99 /* Device instance */ 100 struct spdk_uuid uuid; 101 /* Device name */ 102 char *name; 103 /* Configuration */ 104 struct spdk_ftl_conf conf; 105 106 /* Indicates the device is fully initialized */ 107 int initialized; 108 /* Indicates the device is about to be stopped */ 109 int halt; 110 /* Indicates the device is about to start stopping - use to handle multiple stop request */ 111 bool halt_started; 112 113 /* Underlying device */ 114 struct spdk_bdev_desc *base_bdev_desc; 115 116 /* Non-volatile write buffer cache */ 117 struct ftl_nv_cache nv_cache; 118 119 /* LBA map memory pool */ 120 struct spdk_mempool *lba_pool; 121 122 /* LBA map requests pool */ 123 struct spdk_mempool *lba_request_pool; 124 125 /* Media management events pool */ 126 struct spdk_mempool *media_events_pool; 127 128 /* Statistics */ 129 struct ftl_stats stats; 130 131 /* Current sequence number */ 132 uint64_t seq; 133 134 /* Array of bands */ 135 struct ftl_band *bands; 136 /* Number of operational bands */ 137 size_t num_bands; 138 /* Next write band */ 139 struct ftl_band *next_band; 140 /* Free band list */ 141 LIST_HEAD(, ftl_band) free_bands; 142 /* Closed bands list */ 143 LIST_HEAD(, ftl_band) shut_bands; 144 /* Number of free bands */ 145 size_t num_free; 146 147 /* List of write pointers */ 148 LIST_HEAD(, ftl_wptr) wptr_list; 149 150 /* Logical -> physical table */ 151 void *l2p; 152 /* Size of the l2p table */ 153 uint64_t num_lbas; 154 /* Size of pages mmapped for l2p, valid only for mapping on persistent memory */ 155 size_t l2p_pmem_len; 156 157 /* Address size */ 158 size_t addr_len; 159 160 /* Flush list */ 161 LIST_HEAD(, ftl_flush) flush_list; 162 /* List of band flush requests */ 163 LIST_HEAD(, ftl_band_flush) band_flush_list; 164 165 /* Device specific md buffer */ 166 struct ftl_global_md global_md; 167 168 /* Metadata size */ 169 size_t md_size; 170 void *md_buf; 171 172 /* Transfer unit size */ 173 size_t xfer_size; 174 175 /* Current user write limit */ 176 int limit; 177 178 /* Inflight IO operations */ 179 uint32_t num_inflight; 180 181 /* Manages data relocation */ 182 struct ftl_reloc *reloc; 183 184 /* Thread on which the poller is running */ 185 struct spdk_thread *core_thread; 186 /* IO channel */ 187 struct spdk_io_channel *ioch; 188 /* Poller */ 189 struct spdk_poller *core_poller; 190 191 /* IO channel array provides means for retrieving write buffer entries 192 * from their address stored in L2P. The address is divided into two 193 * parts - IO channel offset poining at specific IO channel (within this 194 * array) and entry offset pointing at specific entry within that IO 195 * channel. 196 */ 197 struct ftl_io_channel **ioch_array; 198 TAILQ_HEAD(, ftl_io_channel) ioch_queue; 199 uint64_t num_io_channels; 200 /* Value required to shift address of a write buffer entry to retrieve 201 * the IO channel it's part of. The other part of the address describes 202 * the offset of an entry within the IO channel's entry array. 203 */ 204 uint64_t ioch_shift; 205 206 /* Write buffer batches */ 207 #define FTL_BATCH_COUNT 4096 208 struct ftl_batch batch_array[FTL_BATCH_COUNT]; 209 /* Iovec buffer used by batches */ 210 struct iovec *iov_buf; 211 /* Batch currently being filled */ 212 struct ftl_batch *current_batch; 213 /* Full and ready to be sent batches. A batch is put on this queue in 214 * case it's already filled, but cannot be sent. 215 */ 216 TAILQ_HEAD(, ftl_batch) pending_batches; 217 TAILQ_HEAD(, ftl_batch) free_batches; 218 219 /* Devices' list */ 220 STAILQ_ENTRY(spdk_ftl_dev) stailq; 221 }; 222 223 struct ftl_nv_cache_header { 224 /* Version of the header */ 225 uint32_t version; 226 /* UUID of the FTL device */ 227 struct spdk_uuid uuid; 228 /* Size of the non-volatile cache (in blocks) */ 229 uint64_t size; 230 /* Contains the next address to be written after clean shutdown, invalid LBA otherwise */ 231 uint64_t current_addr; 232 /* Current phase */ 233 uint8_t phase; 234 /* Checksum of the header, needs to be last element */ 235 uint32_t checksum; 236 } __attribute__((packed)); 237 238 struct ftl_media_event { 239 /* Owner */ 240 struct spdk_ftl_dev *dev; 241 /* Media event */ 242 struct spdk_bdev_media_event event; 243 }; 244 245 typedef void (*ftl_restore_fn)(struct ftl_restore *, int, void *cb_arg); 246 247 void ftl_apply_limits(struct spdk_ftl_dev *dev); 248 void ftl_io_read(struct ftl_io *io); 249 void ftl_io_write(struct ftl_io *io); 250 int ftl_flush_wbuf(struct spdk_ftl_dev *dev, spdk_ftl_fn cb_fn, void *cb_arg); 251 int ftl_current_limit(const struct spdk_ftl_dev *dev); 252 int ftl_invalidate_addr(struct spdk_ftl_dev *dev, struct ftl_addr addr); 253 int ftl_task_core(void *ctx); 254 int ftl_task_read(void *ctx); 255 void ftl_process_anm_event(struct ftl_anm_event *event); 256 size_t ftl_tail_md_num_blocks(const struct spdk_ftl_dev *dev); 257 size_t ftl_tail_md_hdr_num_blocks(void); 258 size_t ftl_vld_map_num_blocks(const struct spdk_ftl_dev *dev); 259 size_t ftl_lba_map_num_blocks(const struct spdk_ftl_dev *dev); 260 size_t ftl_head_md_num_blocks(const struct spdk_ftl_dev *dev); 261 int ftl_restore_md(struct spdk_ftl_dev *dev, ftl_restore_fn cb, void *cb_arg); 262 int ftl_restore_device(struct ftl_restore *restore, ftl_restore_fn cb, void *cb_arg); 263 void ftl_restore_nv_cache(struct ftl_restore *restore, ftl_restore_fn cb, void *cb_arg); 264 int ftl_band_set_direct_access(struct ftl_band *band, bool access); 265 bool ftl_addr_is_written(struct ftl_band *band, struct ftl_addr addr); 266 int ftl_flush_active_bands(struct spdk_ftl_dev *dev, spdk_ftl_fn cb_fn, void *cb_arg); 267 int ftl_nv_cache_write_header(struct ftl_nv_cache *nv_cache, bool shutdown, 268 spdk_bdev_io_completion_cb cb_fn, void *cb_arg); 269 int ftl_nv_cache_scrub(struct ftl_nv_cache *nv_cache, spdk_bdev_io_completion_cb cb_fn, 270 void *cb_arg); 271 void ftl_get_media_events(struct spdk_ftl_dev *dev); 272 int ftl_io_channel_poll(void *arg); 273 void ftl_evict_cache_entry(struct spdk_ftl_dev *dev, struct ftl_wbuf_entry *entry); 274 struct spdk_io_channel *ftl_get_io_channel(const struct spdk_ftl_dev *dev); 275 struct ftl_io_channel *ftl_io_channel_get_ctx(struct spdk_io_channel *ioch); 276 277 278 #define ftl_to_addr(address) \ 279 (struct ftl_addr) { .offset = (uint64_t)(address) } 280 281 #define ftl_to_addr_packed(address) \ 282 (struct ftl_addr) { .pack.offset = (uint32_t)(address) } 283 284 static inline struct spdk_thread * 285 ftl_get_core_thread(const struct spdk_ftl_dev *dev) 286 { 287 return dev->core_thread; 288 } 289 290 static inline size_t 291 ftl_get_num_bands(const struct spdk_ftl_dev *dev) 292 { 293 return dev->num_bands; 294 } 295 296 static inline size_t 297 ftl_get_num_punits(const struct spdk_ftl_dev *dev) 298 { 299 return spdk_bdev_get_optimal_open_zones(spdk_bdev_desc_get_bdev(dev->base_bdev_desc)); 300 } 301 302 static inline size_t 303 ftl_get_num_zones(const struct spdk_ftl_dev *dev) 304 { 305 return ftl_get_num_bands(dev) * ftl_get_num_punits(dev); 306 } 307 308 static inline size_t 309 ftl_get_num_blocks_in_zone(const struct spdk_ftl_dev *dev) 310 { 311 return spdk_bdev_get_zone_size(spdk_bdev_desc_get_bdev(dev->base_bdev_desc)); 312 } 313 314 static inline uint64_t 315 ftl_get_num_blocks_in_band(const struct spdk_ftl_dev *dev) 316 { 317 return ftl_get_num_punits(dev) * ftl_get_num_blocks_in_zone(dev); 318 } 319 320 static inline uint64_t 321 ftl_addr_get_zone_slba(const struct spdk_ftl_dev *dev, struct ftl_addr addr) 322 { 323 return addr.offset -= (addr.offset % ftl_get_num_blocks_in_zone(dev)); 324 } 325 326 static inline uint64_t 327 ftl_addr_get_band(const struct spdk_ftl_dev *dev, struct ftl_addr addr) 328 { 329 return addr.offset / ftl_get_num_blocks_in_band(dev); 330 } 331 332 static inline uint64_t 333 ftl_addr_get_punit(const struct spdk_ftl_dev *dev, struct ftl_addr addr) 334 { 335 return (addr.offset / ftl_get_num_blocks_in_zone(dev)) % ftl_get_num_punits(dev); 336 } 337 338 static inline uint64_t 339 ftl_addr_get_zone_offset(const struct spdk_ftl_dev *dev, struct ftl_addr addr) 340 { 341 return addr.offset % ftl_get_num_blocks_in_zone(dev); 342 } 343 344 static inline size_t 345 ftl_vld_map_size(const struct spdk_ftl_dev *dev) 346 { 347 return (size_t)spdk_divide_round_up(ftl_get_num_blocks_in_band(dev), CHAR_BIT); 348 } 349 350 static inline int 351 ftl_addr_packed(const struct spdk_ftl_dev *dev) 352 { 353 return dev->addr_len < 32; 354 } 355 356 static inline void 357 ftl_l2p_lba_persist(const struct spdk_ftl_dev *dev, uint64_t lba) 358 { 359 #ifdef SPDK_CONFIG_PMDK 360 size_t ftl_addr_size = ftl_addr_packed(dev) ? 4 : 8; 361 pmem_persist((char *)dev->l2p + (lba * ftl_addr_size), ftl_addr_size); 362 #else /* SPDK_CONFIG_PMDK */ 363 SPDK_ERRLOG("Libpmem not available, cannot flush l2p to pmem\n"); 364 assert(0); 365 #endif /* SPDK_CONFIG_PMDK */ 366 } 367 368 static inline int 369 ftl_addr_invalid(struct ftl_addr addr) 370 { 371 return addr.offset == ftl_to_addr(FTL_ADDR_INVALID).offset; 372 } 373 374 static inline int 375 ftl_addr_cached(struct ftl_addr addr) 376 { 377 return !ftl_addr_invalid(addr) && addr.cached; 378 } 379 380 static inline struct ftl_addr 381 ftl_addr_to_packed(const struct spdk_ftl_dev *dev, struct ftl_addr addr) 382 { 383 struct ftl_addr p = {}; 384 385 if (ftl_addr_invalid(addr)) { 386 p = ftl_to_addr_packed(FTL_ADDR_INVALID); 387 } else if (ftl_addr_cached(addr)) { 388 p.pack.cached = 1; 389 p.pack.cache_offset = (uint32_t) addr.cache_offset; 390 } else { 391 p.pack.offset = (uint32_t) addr.offset; 392 } 393 394 return p; 395 } 396 397 static inline struct ftl_addr 398 ftl_addr_from_packed(const struct spdk_ftl_dev *dev, struct ftl_addr p) 399 { 400 struct ftl_addr addr = {}; 401 402 if (p.pack.offset == (uint32_t)FTL_ADDR_INVALID) { 403 addr = ftl_to_addr(FTL_ADDR_INVALID); 404 } else if (p.pack.cached) { 405 addr.cached = 1; 406 addr.cache_offset = p.pack.cache_offset; 407 } else { 408 addr = p; 409 } 410 411 return addr; 412 } 413 414 #define _ftl_l2p_set(l2p, off, val, bits) \ 415 __atomic_store_n(((uint##bits##_t *)(l2p)) + (off), val, __ATOMIC_SEQ_CST) 416 417 #define _ftl_l2p_set32(l2p, off, val) \ 418 _ftl_l2p_set(l2p, off, val, 32) 419 420 #define _ftl_l2p_set64(l2p, off, val) \ 421 _ftl_l2p_set(l2p, off, val, 64) 422 423 #define _ftl_l2p_get(l2p, off, bits) \ 424 __atomic_load_n(((uint##bits##_t *)(l2p)) + (off), __ATOMIC_SEQ_CST) 425 426 #define _ftl_l2p_get32(l2p, off) \ 427 _ftl_l2p_get(l2p, off, 32) 428 429 #define _ftl_l2p_get64(l2p, off) \ 430 _ftl_l2p_get(l2p, off, 64) 431 432 #define ftl_addr_cmp(p1, p2) \ 433 ((p1).offset == (p2).offset) 434 435 static inline void 436 ftl_l2p_set(struct spdk_ftl_dev *dev, uint64_t lba, struct ftl_addr addr) 437 { 438 assert(dev->num_lbas > lba); 439 440 if (ftl_addr_packed(dev)) { 441 _ftl_l2p_set32(dev->l2p, lba, ftl_addr_to_packed(dev, addr).offset); 442 } else { 443 _ftl_l2p_set64(dev->l2p, lba, addr.offset); 444 } 445 446 if (dev->l2p_pmem_len != 0) { 447 ftl_l2p_lba_persist(dev, lba); 448 } 449 } 450 451 static inline struct ftl_addr 452 ftl_l2p_get(struct spdk_ftl_dev *dev, uint64_t lba) 453 { 454 assert(dev->num_lbas > lba); 455 456 if (ftl_addr_packed(dev)) { 457 return ftl_addr_from_packed(dev, ftl_to_addr_packed( 458 _ftl_l2p_get32(dev->l2p, lba))); 459 } else { 460 return ftl_to_addr(_ftl_l2p_get64(dev->l2p, lba)); 461 } 462 } 463 464 static inline bool 465 ftl_dev_has_nv_cache(const struct spdk_ftl_dev *dev) 466 { 467 return dev->nv_cache.bdev_desc != NULL; 468 } 469 470 #define FTL_NV_CACHE_HEADER_VERSION (1) 471 #define FTL_NV_CACHE_DATA_OFFSET (1) 472 #define FTL_NV_CACHE_PHASE_OFFSET (62) 473 #define FTL_NV_CACHE_PHASE_COUNT (4) 474 #define FTL_NV_CACHE_PHASE_MASK (3ULL << FTL_NV_CACHE_PHASE_OFFSET) 475 #define FTL_NV_CACHE_LBA_INVALID (FTL_LBA_INVALID & ~FTL_NV_CACHE_PHASE_MASK) 476 477 static inline bool 478 ftl_nv_cache_phase_is_valid(unsigned int phase) 479 { 480 return phase > 0 && phase <= 3; 481 } 482 483 static inline unsigned int 484 ftl_nv_cache_next_phase(unsigned int current) 485 { 486 static const unsigned int phases[] = { 0, 2, 3, 1 }; 487 assert(ftl_nv_cache_phase_is_valid(current)); 488 return phases[current]; 489 } 490 491 static inline unsigned int 492 ftl_nv_cache_prev_phase(unsigned int current) 493 { 494 static const unsigned int phases[] = { 0, 3, 1, 2 }; 495 assert(ftl_nv_cache_phase_is_valid(current)); 496 return phases[current]; 497 } 498 499 static inline uint64_t 500 ftl_nv_cache_pack_lba(uint64_t lba, unsigned int phase) 501 { 502 assert(ftl_nv_cache_phase_is_valid(phase)); 503 return (lba & ~FTL_NV_CACHE_PHASE_MASK) | ((uint64_t)phase << FTL_NV_CACHE_PHASE_OFFSET); 504 } 505 506 static inline void 507 ftl_nv_cache_unpack_lba(uint64_t in_lba, uint64_t *out_lba, unsigned int *phase) 508 { 509 *out_lba = in_lba & ~FTL_NV_CACHE_PHASE_MASK; 510 *phase = (in_lba & FTL_NV_CACHE_PHASE_MASK) >> FTL_NV_CACHE_PHASE_OFFSET; 511 512 /* If the phase is invalid the block wasn't written yet, so treat the LBA as invalid too */ 513 if (!ftl_nv_cache_phase_is_valid(*phase) || *out_lba == FTL_NV_CACHE_LBA_INVALID) { 514 *out_lba = FTL_LBA_INVALID; 515 } 516 } 517 518 static inline bool 519 ftl_is_append_supported(const struct spdk_ftl_dev *dev) 520 { 521 return dev->conf.use_append; 522 } 523 524 #endif /* FTL_CORE_H */ 525