1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #ifndef FTL_CORE_H 35 #define FTL_CORE_H 36 37 #include "spdk/stdinc.h" 38 #include "spdk/nvme.h" 39 #include "spdk/nvme_ocssd.h" 40 #include "spdk/uuid.h" 41 #include "spdk/thread.h" 42 #include "spdk/util.h" 43 #include "spdk_internal/log.h" 44 #include "spdk/likely.h" 45 #include "spdk/queue.h" 46 #include "spdk/ftl.h" 47 #include "spdk/bdev.h" 48 49 #include "ftl_ppa.h" 50 #include "ftl_io.h" 51 #include "ftl_trace.h" 52 53 struct spdk_ftl_dev; 54 struct ftl_band; 55 struct ftl_chunk; 56 struct ftl_io; 57 struct ftl_restore; 58 struct ftl_wptr; 59 struct ftl_flush; 60 struct ftl_reloc; 61 struct ftl_anm_event; 62 struct ftl_band_flush; 63 64 struct ftl_stats { 65 /* Number of writes scheduled directly by the user */ 66 uint64_t write_user; 67 68 /* Total number of writes */ 69 uint64_t write_total; 70 71 /* Traces */ 72 struct ftl_trace trace; 73 74 /* Number of limits applied */ 75 uint64_t limits[SPDK_FTL_LIMIT_MAX]; 76 }; 77 78 struct ftl_punit { 79 struct spdk_ftl_dev *dev; 80 81 struct ftl_ppa start_ppa; 82 }; 83 84 struct ftl_thread { 85 /* Owner */ 86 struct spdk_ftl_dev *dev; 87 /* I/O queue pair */ 88 struct spdk_nvme_qpair *qpair; 89 90 /* Thread on which the poller is running */ 91 struct spdk_thread *thread; 92 93 /* IO channel */ 94 struct spdk_io_channel *ioch; 95 /* Poller */ 96 struct spdk_poller *poller; 97 /* Poller's function */ 98 spdk_poller_fn poller_fn; 99 /* Poller's frequency */ 100 uint64_t period_us; 101 }; 102 103 struct ftl_global_md { 104 /* Device instance */ 105 struct spdk_uuid uuid; 106 /* Size of the l2p table */ 107 uint64_t num_lbas; 108 }; 109 110 struct ftl_nv_cache { 111 /* Write buffer cache bdev */ 112 struct spdk_bdev_desc *bdev_desc; 113 /* Write pointer */ 114 uint64_t current_addr; 115 /* Number of available blocks left */ 116 uint64_t num_available; 117 /* Maximum number of blocks */ 118 uint64_t num_data_blocks; 119 /* 120 * Phase of the current cycle of writes. Each time whole cache area is filled, the phase is 121 * advanced. Current phase is saved in every IO's metadata, as well as in the header saved 122 * in the first sector. By looking at the phase of each block, it's possible to find the 123 * oldest block and replay the order of the writes when recovering the data from the cache. 124 */ 125 unsigned int phase; 126 /* Indicates that the data can be written to the cache */ 127 bool ready; 128 /* Metadata pool */ 129 struct spdk_mempool *md_pool; 130 /* DMA buffer for writing the header */ 131 void *dma_buf; 132 /* Cache lock */ 133 pthread_spinlock_t lock; 134 }; 135 136 struct ftl_init_context { 137 /* User's callback */ 138 spdk_ftl_init_fn cb_fn; 139 /* Callback's argument */ 140 void *cb_arg; 141 /* Thread to call the callback on */ 142 struct spdk_thread *thread; 143 /* Poller to check if the device has been destroyed/initialized */ 144 struct spdk_poller *poller; 145 }; 146 147 struct spdk_ftl_dev { 148 /* Device instance */ 149 struct spdk_uuid uuid; 150 /* Device name */ 151 char *name; 152 /* Configuration */ 153 struct spdk_ftl_conf conf; 154 155 /* Indicates the device is fully initialized */ 156 int initialized; 157 /* Indicates the device is about to be stopped */ 158 int halt; 159 160 /* Status to return for halt completion callback */ 161 int halt_complete_status; 162 /* Initializaton context */ 163 struct ftl_init_context init_ctx; 164 /* Destruction context */ 165 struct ftl_init_context fini_ctx; 166 167 /* NVMe controller */ 168 struct spdk_nvme_ctrlr *ctrlr; 169 /* NVMe namespace */ 170 struct spdk_nvme_ns *ns; 171 /* NVMe transport ID */ 172 struct spdk_nvme_transport_id trid; 173 174 /* Non-volatile write buffer cache */ 175 struct ftl_nv_cache nv_cache; 176 177 /* LBA map memory pool */ 178 struct spdk_mempool *lba_pool; 179 180 /* LBA map requests pool */ 181 struct spdk_mempool *lba_request_pool; 182 183 /* Statistics */ 184 struct ftl_stats stats; 185 186 /* Parallel unit range */ 187 struct spdk_ftl_punit_range range; 188 /* Array of parallel units */ 189 struct ftl_punit *punits; 190 191 /* Current sequence number */ 192 uint64_t seq; 193 194 /* Array of bands */ 195 struct ftl_band *bands; 196 /* Number of operational bands */ 197 size_t num_bands; 198 /* Next write band */ 199 struct ftl_band *next_band; 200 /* Free band list */ 201 LIST_HEAD(, ftl_band) free_bands; 202 /* Closed bands list */ 203 LIST_HEAD(, ftl_band) shut_bands; 204 /* Number of free bands */ 205 size_t num_free; 206 207 /* List of write pointers */ 208 LIST_HEAD(, ftl_wptr) wptr_list; 209 210 /* Logical -> physical table */ 211 void *l2p; 212 /* Size of the l2p table */ 213 uint64_t num_lbas; 214 215 /* PPA format */ 216 struct ftl_ppa_fmt ppaf; 217 /* PPA address size */ 218 size_t ppa_len; 219 /* Device's geometry */ 220 struct spdk_ocssd_geometry_data geo; 221 222 /* Flush list */ 223 LIST_HEAD(, ftl_flush) flush_list; 224 /* List of band flush requests */ 225 LIST_HEAD(, ftl_band_flush) band_flush_list; 226 227 /* Device specific md buffer */ 228 struct ftl_global_md global_md; 229 230 /* Metadata size */ 231 size_t md_size; 232 233 /* Transfer unit size */ 234 size_t xfer_size; 235 /* Ring write buffer */ 236 struct ftl_rwb *rwb; 237 238 /* Current user write limit */ 239 int limit; 240 241 /* Inflight IO operations */ 242 uint32_t num_inflight; 243 /* Queue of IO awaiting retry */ 244 TAILQ_HEAD(, ftl_io) retry_queue; 245 246 /* Manages data relocation */ 247 struct ftl_reloc *reloc; 248 249 /* Threads */ 250 struct ftl_thread core_thread; 251 struct ftl_thread read_thread; 252 253 /* Devices' list */ 254 STAILQ_ENTRY(spdk_ftl_dev) stailq; 255 }; 256 257 struct ftl_nv_cache_header { 258 /* Version of the header */ 259 uint32_t version; 260 /* UUID of the FTL device */ 261 struct spdk_uuid uuid; 262 /* Size of the non-volatile cache (in blocks) */ 263 uint64_t size; 264 /* Contains the next address to be written after clean shutdown, invalid LBA otherwise */ 265 uint64_t current_addr; 266 /* Current phase */ 267 uint8_t phase; 268 /* Checksum of the header, needs to be last element */ 269 uint32_t checksum; 270 } __attribute__((packed)); 271 272 typedef void (*ftl_restore_fn)(struct spdk_ftl_dev *, struct ftl_restore *, int); 273 274 void ftl_apply_limits(struct spdk_ftl_dev *dev); 275 void ftl_io_read(struct ftl_io *io); 276 void ftl_io_write(struct ftl_io *io); 277 int ftl_io_erase(struct ftl_io *io); 278 int ftl_flush_rwb(struct spdk_ftl_dev *dev, spdk_ftl_fn cb_fn, void *cb_arg); 279 int ftl_current_limit(const struct spdk_ftl_dev *dev); 280 int ftl_invalidate_addr(struct spdk_ftl_dev *dev, struct ftl_ppa ppa); 281 int ftl_task_core(void *ctx); 282 int ftl_task_read(void *ctx); 283 void ftl_process_anm_event(struct ftl_anm_event *event); 284 size_t ftl_tail_md_num_lbks(const struct spdk_ftl_dev *dev); 285 size_t ftl_tail_md_hdr_num_lbks(void); 286 size_t ftl_vld_map_num_lbks(const struct spdk_ftl_dev *dev); 287 size_t ftl_lba_map_num_lbks(const struct spdk_ftl_dev *dev); 288 size_t ftl_head_md_num_lbks(const struct spdk_ftl_dev *dev); 289 int ftl_restore_md(struct spdk_ftl_dev *dev, ftl_restore_fn cb); 290 int ftl_restore_device(struct ftl_restore *restore, ftl_restore_fn cb); 291 void ftl_restore_nv_cache(struct ftl_restore *restore, ftl_restore_fn cb); 292 int ftl_band_set_direct_access(struct ftl_band *band, bool access); 293 int ftl_retrieve_chunk_info(struct spdk_ftl_dev *dev, struct ftl_ppa ppa, 294 struct spdk_ocssd_chunk_information_entry *info, 295 unsigned int num_entries); 296 bool ftl_ppa_is_written(struct ftl_band *band, struct ftl_ppa ppa); 297 int ftl_flush_active_bands(struct spdk_ftl_dev *dev, spdk_ftl_fn cb_fn, void *cb_arg); 298 int ftl_nv_cache_write_header(struct ftl_nv_cache *nv_cache, bool shutdown, 299 spdk_bdev_io_completion_cb cb_fn, void *cb_arg); 300 int ftl_nv_cache_scrub(struct ftl_nv_cache *nv_cache, spdk_bdev_io_completion_cb cb_fn, 301 void *cb_arg); 302 303 struct spdk_io_channel * 304 ftl_get_io_channel(const struct spdk_ftl_dev *dev); 305 306 #define ftl_to_ppa(addr) \ 307 (struct ftl_ppa) { .ppa = (uint64_t)(addr) } 308 309 #define ftl_to_ppa_packed(addr) \ 310 (struct ftl_ppa) { .pack.ppa = (uint32_t)(addr) } 311 312 static inline struct spdk_thread * 313 ftl_get_core_thread(const struct spdk_ftl_dev *dev) 314 { 315 return dev->core_thread.thread; 316 } 317 318 static inline struct spdk_nvme_qpair * 319 ftl_get_write_qpair(const struct spdk_ftl_dev *dev) 320 { 321 return dev->core_thread.qpair; 322 } 323 324 static inline struct spdk_thread * 325 ftl_get_read_thread(const struct spdk_ftl_dev *dev) 326 { 327 return dev->read_thread.thread; 328 } 329 330 static inline struct spdk_nvme_qpair * 331 ftl_get_read_qpair(const struct spdk_ftl_dev *dev) 332 { 333 return dev->read_thread.qpair; 334 } 335 336 static inline int 337 ftl_ppa_packed(const struct spdk_ftl_dev *dev) 338 { 339 return dev->ppa_len < 32; 340 } 341 342 static inline int 343 ftl_ppa_invalid(struct ftl_ppa ppa) 344 { 345 return ppa.ppa == ftl_to_ppa(FTL_PPA_INVALID).ppa; 346 } 347 348 static inline int 349 ftl_ppa_cached(struct ftl_ppa ppa) 350 { 351 return !ftl_ppa_invalid(ppa) && ppa.cached; 352 } 353 354 static inline uint64_t 355 ftl_ppa_addr_pack(const struct spdk_ftl_dev *dev, struct ftl_ppa ppa) 356 { 357 uint64_t lbk, chk, pu, grp; 358 359 lbk = ppa.lbk; 360 chk = ppa.chk; 361 pu = ppa.pu; 362 grp = ppa.grp; 363 364 return (lbk << dev->ppaf.lbk_offset) | 365 (chk << dev->ppaf.chk_offset) | 366 (pu << dev->ppaf.pu_offset) | 367 (grp << dev->ppaf.grp_offset); 368 } 369 370 static inline struct ftl_ppa 371 ftl_ppa_addr_unpack(const struct spdk_ftl_dev *dev, uint64_t ppa) 372 { 373 struct ftl_ppa res = {}; 374 375 res.lbk = (ppa >> dev->ppaf.lbk_offset) & dev->ppaf.lbk_mask; 376 res.chk = (ppa >> dev->ppaf.chk_offset) & dev->ppaf.chk_mask; 377 res.pu = (ppa >> dev->ppaf.pu_offset) & dev->ppaf.pu_mask; 378 res.grp = (ppa >> dev->ppaf.grp_offset) & dev->ppaf.grp_mask; 379 380 return res; 381 } 382 383 static inline struct ftl_ppa 384 ftl_ppa_to_packed(const struct spdk_ftl_dev *dev, struct ftl_ppa ppa) 385 { 386 struct ftl_ppa p = {}; 387 388 if (ftl_ppa_invalid(ppa)) { 389 p = ftl_to_ppa_packed(FTL_PPA_INVALID); 390 } else if (ftl_ppa_cached(ppa)) { 391 p.pack.cached = 1; 392 p.pack.offset = (uint32_t) ppa.offset; 393 } else { 394 p.pack.ppa = (uint32_t) ftl_ppa_addr_pack(dev, ppa); 395 } 396 397 return p; 398 } 399 400 static inline struct ftl_ppa 401 ftl_ppa_from_packed(const struct spdk_ftl_dev *dev, struct ftl_ppa p) 402 { 403 struct ftl_ppa ppa = {}; 404 405 if (p.pack.ppa == (uint32_t)FTL_PPA_INVALID) { 406 ppa = ftl_to_ppa(FTL_PPA_INVALID); 407 } else if (p.pack.cached) { 408 ppa.cached = 1; 409 ppa.offset = p.pack.offset; 410 } else { 411 ppa = ftl_ppa_addr_unpack(dev, p.pack.ppa); 412 } 413 414 return ppa; 415 } 416 417 static inline unsigned int 418 ftl_ppa_flatten_punit(const struct spdk_ftl_dev *dev, struct ftl_ppa ppa) 419 { 420 return ppa.pu * dev->geo.num_grp + ppa.grp - dev->range.begin; 421 } 422 423 static inline int 424 ftl_ppa_in_range(const struct spdk_ftl_dev *dev, struct ftl_ppa ppa) 425 { 426 unsigned int punit = ftl_ppa_flatten_punit(dev, ppa) + dev->range.begin; 427 428 if (punit >= dev->range.begin && punit <= dev->range.end) { 429 return 1; 430 } 431 432 return 0; 433 } 434 435 #define _ftl_l2p_set(l2p, off, val, bits) \ 436 __atomic_store_n(((uint##bits##_t *)(l2p)) + (off), val, __ATOMIC_SEQ_CST) 437 438 #define _ftl_l2p_set32(l2p, off, val) \ 439 _ftl_l2p_set(l2p, off, val, 32) 440 441 #define _ftl_l2p_set64(l2p, off, val) \ 442 _ftl_l2p_set(l2p, off, val, 64) 443 444 #define _ftl_l2p_get(l2p, off, bits) \ 445 __atomic_load_n(((uint##bits##_t *)(l2p)) + (off), __ATOMIC_SEQ_CST) 446 447 #define _ftl_l2p_get32(l2p, off) \ 448 _ftl_l2p_get(l2p, off, 32) 449 450 #define _ftl_l2p_get64(l2p, off) \ 451 _ftl_l2p_get(l2p, off, 64) 452 453 #define ftl_ppa_cmp(p1, p2) \ 454 ((p1).ppa == (p2).ppa) 455 456 static inline void 457 ftl_l2p_set(struct spdk_ftl_dev *dev, uint64_t lba, struct ftl_ppa ppa) 458 { 459 assert(dev->num_lbas > lba); 460 461 if (ftl_ppa_packed(dev)) { 462 _ftl_l2p_set32(dev->l2p, lba, ftl_ppa_to_packed(dev, ppa).ppa); 463 } else { 464 _ftl_l2p_set64(dev->l2p, lba, ppa.ppa); 465 } 466 } 467 468 static inline struct ftl_ppa 469 ftl_l2p_get(struct spdk_ftl_dev *dev, uint64_t lba) 470 { 471 assert(dev->num_lbas > lba); 472 473 if (ftl_ppa_packed(dev)) { 474 return ftl_ppa_from_packed(dev, ftl_to_ppa_packed( 475 _ftl_l2p_get32(dev->l2p, lba))); 476 } else { 477 return ftl_to_ppa(_ftl_l2p_get64(dev->l2p, lba)); 478 } 479 } 480 static inline size_t 481 ftl_dev_num_bands(const struct spdk_ftl_dev *dev) 482 { 483 return dev->geo.num_chk; 484 } 485 486 static inline size_t 487 ftl_dev_lbks_in_chunk(const struct spdk_ftl_dev *dev) 488 { 489 return dev->geo.clba; 490 } 491 492 static inline size_t 493 ftl_dev_num_punits(const struct spdk_ftl_dev *dev) 494 { 495 return dev->range.end - dev->range.begin + 1; 496 } 497 498 static inline uint64_t 499 ftl_num_band_lbks(const struct spdk_ftl_dev *dev) 500 { 501 return ftl_dev_num_punits(dev) * ftl_dev_lbks_in_chunk(dev); 502 } 503 504 static inline size_t 505 ftl_vld_map_size(const struct spdk_ftl_dev *dev) 506 { 507 return (size_t)spdk_divide_round_up(ftl_num_band_lbks(dev), CHAR_BIT); 508 } 509 510 static inline bool 511 ftl_dev_has_nv_cache(const struct spdk_ftl_dev *dev) 512 { 513 return dev->nv_cache.bdev_desc != NULL; 514 } 515 516 #define FTL_NV_CACHE_HEADER_VERSION (1) 517 #define FTL_NV_CACHE_DATA_OFFSET (1) 518 #define FTL_NV_CACHE_PHASE_OFFSET (62) 519 #define FTL_NV_CACHE_PHASE_COUNT (4) 520 #define FTL_NV_CACHE_PHASE_MASK (3ULL << FTL_NV_CACHE_PHASE_OFFSET) 521 #define FTL_NV_CACHE_LBA_INVALID (FTL_LBA_INVALID & ~FTL_NV_CACHE_PHASE_MASK) 522 523 static inline bool 524 ftl_nv_cache_phase_is_valid(unsigned int phase) 525 { 526 return phase > 0 && phase <= 3; 527 } 528 529 static inline unsigned int 530 ftl_nv_cache_next_phase(unsigned int current) 531 { 532 static const unsigned int phases[] = { 0, 2, 3, 1 }; 533 assert(ftl_nv_cache_phase_is_valid(current)); 534 return phases[current]; 535 } 536 537 static inline unsigned int 538 ftl_nv_cache_prev_phase(unsigned int current) 539 { 540 static const unsigned int phases[] = { 0, 3, 1, 2 }; 541 assert(ftl_nv_cache_phase_is_valid(current)); 542 return phases[current]; 543 } 544 545 static inline uint64_t 546 ftl_nv_cache_pack_lba(uint64_t lba, unsigned int phase) 547 { 548 assert(ftl_nv_cache_phase_is_valid(phase)); 549 return (lba & ~FTL_NV_CACHE_PHASE_MASK) | ((uint64_t)phase << FTL_NV_CACHE_PHASE_OFFSET); 550 } 551 552 static inline void 553 ftl_nv_cache_unpack_lba(uint64_t in_lba, uint64_t *out_lba, unsigned int *phase) 554 { 555 *out_lba = in_lba & ~FTL_NV_CACHE_PHASE_MASK; 556 *phase = (in_lba & FTL_NV_CACHE_PHASE_MASK) >> FTL_NV_CACHE_PHASE_OFFSET; 557 558 /* If the phase is invalid the block wasn't written yet, so treat the LBA as invalid too */ 559 if (!ftl_nv_cache_phase_is_valid(*phase) || *out_lba == FTL_NV_CACHE_LBA_INVALID) { 560 *out_lba = FTL_LBA_INVALID; 561 } 562 } 563 564 #endif /* FTL_CORE_H */ 565