1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #ifndef FTL_CORE_H 35 #define FTL_CORE_H 36 37 #include "spdk/stdinc.h" 38 #include "spdk/nvme.h" 39 #include "spdk/nvme_ocssd.h" 40 #include "spdk/uuid.h" 41 #include "spdk/thread.h" 42 #include "spdk/util.h" 43 #include "spdk_internal/log.h" 44 #include "spdk/queue.h" 45 #include "spdk/ftl.h" 46 47 #include "ftl_ppa.h" 48 #include "ftl_io.h" 49 #include "ftl_trace.h" 50 51 struct spdk_ftl_dev; 52 struct ftl_band; 53 struct ftl_chunk; 54 struct ftl_io; 55 struct ftl_restore; 56 struct ftl_wptr; 57 struct ftl_flush; 58 struct ftl_reloc; 59 struct ftl_anm_event; 60 61 struct ftl_stats { 62 /* Number of writes scheduled directly by the user */ 63 uint64_t write_user; 64 65 /* Total number of writes */ 66 uint64_t write_total; 67 68 /* Traces */ 69 struct ftl_trace trace; 70 71 /* Number of limits applied */ 72 uint64_t limits[SPDK_FTL_LIMIT_MAX]; 73 }; 74 75 struct ftl_punit { 76 struct spdk_ftl_dev *dev; 77 78 struct ftl_ppa start_ppa; 79 }; 80 81 struct ftl_thread { 82 /* Owner */ 83 struct spdk_ftl_dev *dev; 84 /* I/O queue pair */ 85 struct spdk_nvme_qpair *qpair; 86 87 /* Thread on which the poller is running */ 88 struct spdk_thread *thread; 89 90 /* Poller */ 91 struct spdk_poller *poller; 92 /* Poller's function */ 93 spdk_poller_fn poller_fn; 94 /* Poller's frequency */ 95 uint64_t period_us; 96 }; 97 98 struct ftl_global_md { 99 /* Device instance */ 100 struct spdk_uuid uuid; 101 /* Size of the l2p table */ 102 uint64_t num_lbas; 103 }; 104 105 struct spdk_ftl_dev { 106 /* Device instance */ 107 struct spdk_uuid uuid; 108 /* Device name */ 109 char *name; 110 /* Configuration */ 111 struct spdk_ftl_conf conf; 112 113 /* Indicates the device is fully initialized */ 114 int initialized; 115 /* Indicates the device is about to be stopped */ 116 int halt; 117 118 /* Init callback */ 119 spdk_ftl_init_fn init_cb; 120 /* Init callback's context */ 121 void *init_arg; 122 123 /* Halt callback */ 124 spdk_ftl_fn halt_cb; 125 /* Halt callback's context */ 126 void *halt_arg; 127 /* Halt poller, checks if the device has been halted */ 128 struct spdk_poller *halt_poller; 129 130 /* IO channel */ 131 struct spdk_io_channel *ioch; 132 133 /* NVMe controller */ 134 struct spdk_nvme_ctrlr *ctrlr; 135 /* NVMe namespace */ 136 struct spdk_nvme_ns *ns; 137 /* NVMe transport ID */ 138 struct spdk_nvme_transport_id trid; 139 140 /* LBA map memory pool */ 141 struct spdk_mempool *lba_pool; 142 143 /* Statistics */ 144 struct ftl_stats stats; 145 146 /* Parallel unit range */ 147 struct spdk_ftl_punit_range range; 148 /* Array of parallel units */ 149 struct ftl_punit *punits; 150 151 /* Current sequence number */ 152 uint64_t seq; 153 154 /* Array of bands */ 155 struct ftl_band *bands; 156 /* Band being curently defraged */ 157 struct ftl_band *df_band; 158 /* Number of operational bands */ 159 size_t num_bands; 160 /* Next write band */ 161 struct ftl_band *next_band; 162 /* Free band list */ 163 LIST_HEAD(, ftl_band) free_bands; 164 /* Closed bands list */ 165 LIST_HEAD(, ftl_band) shut_bands; 166 /* Number of free bands */ 167 size_t num_free; 168 169 /* List of write pointers */ 170 LIST_HEAD(, ftl_wptr) wptr_list; 171 172 /* Logical -> physical table */ 173 void *l2p; 174 /* Size of the l2p table */ 175 uint64_t num_lbas; 176 177 /* PPA format */ 178 struct ftl_ppa_fmt ppaf; 179 /* PPA address size */ 180 size_t ppa_len; 181 /* Device's geometry */ 182 struct spdk_ocssd_geometry_data geo; 183 184 /* Flush list */ 185 LIST_HEAD(, ftl_flush) flush_list; 186 187 /* Device specific md buffer */ 188 struct ftl_global_md global_md; 189 190 /* Metadata size */ 191 size_t md_size; 192 193 /* Transfer unit size */ 194 size_t xfer_size; 195 /* Ring write buffer */ 196 struct ftl_rwb *rwb; 197 198 /* Current user write limit */ 199 int limit; 200 201 /* Inflight io operations */ 202 uint32_t num_inflight; 203 204 /* Manages data relocation */ 205 struct ftl_reloc *reloc; 206 207 /* Threads */ 208 struct ftl_thread core_thread; 209 struct ftl_thread read_thread; 210 211 /* Devices' list */ 212 STAILQ_ENTRY(spdk_ftl_dev) stailq; 213 }; 214 215 typedef void (*ftl_restore_fn)(struct spdk_ftl_dev *, struct ftl_restore *, int); 216 217 void ftl_apply_limits(struct spdk_ftl_dev *dev); 218 void ftl_io_read(struct ftl_io *io); 219 int ftl_io_write(struct ftl_io *io); 220 int ftl_io_erase(struct ftl_io *io); 221 int ftl_io_flush(struct ftl_io *io); 222 int ftl_current_limit(const struct spdk_ftl_dev *dev); 223 int ftl_invalidate_addr(struct spdk_ftl_dev *dev, struct ftl_ppa ppa); 224 int ftl_task_core(void *ctx); 225 int ftl_task_read(void *ctx); 226 void ftl_process_anm_event(struct ftl_anm_event *event); 227 size_t ftl_tail_md_num_lbks(const struct spdk_ftl_dev *dev); 228 size_t ftl_tail_md_hdr_num_lbks(void); 229 size_t ftl_vld_map_num_lbks(const struct spdk_ftl_dev *dev); 230 size_t ftl_lba_map_num_lbks(const struct spdk_ftl_dev *dev); 231 size_t ftl_head_md_num_lbks(const struct spdk_ftl_dev *dev); 232 int ftl_restore_md(struct spdk_ftl_dev *dev, ftl_restore_fn cb); 233 int ftl_restore_device(struct ftl_restore *restore, ftl_restore_fn cb); 234 235 #define ftl_to_ppa(addr) \ 236 (struct ftl_ppa) { .ppa = (uint64_t)(addr) } 237 238 #define ftl_to_ppa_packed(addr) \ 239 (struct ftl_ppa) { .pack.ppa = (uint32_t)(addr) } 240 241 static inline struct spdk_thread * 242 ftl_get_core_thread(const struct spdk_ftl_dev *dev) 243 { 244 return dev->core_thread.thread; 245 } 246 247 static inline struct spdk_nvme_qpair * 248 ftl_get_write_qpair(const struct spdk_ftl_dev *dev) 249 { 250 return dev->core_thread.qpair; 251 } 252 253 static inline struct spdk_thread * 254 ftl_get_read_thread(const struct spdk_ftl_dev *dev) 255 { 256 return dev->read_thread.thread; 257 } 258 259 static inline struct spdk_nvme_qpair * 260 ftl_get_read_qpair(const struct spdk_ftl_dev *dev) 261 { 262 return dev->read_thread.qpair; 263 } 264 265 static inline int 266 ftl_ppa_packed(const struct spdk_ftl_dev *dev) 267 { 268 return dev->ppa_len < 32; 269 } 270 271 static inline int 272 ftl_ppa_invalid(struct ftl_ppa ppa) 273 { 274 return ppa.ppa == ftl_to_ppa(FTL_PPA_INVALID).ppa; 275 } 276 277 static inline int 278 ftl_ppa_cached(struct ftl_ppa ppa) 279 { 280 return !ftl_ppa_invalid(ppa) && ppa.cached; 281 } 282 283 static inline uint64_t 284 ftl_ppa_addr_pack(const struct spdk_ftl_dev *dev, struct ftl_ppa ppa) 285 { 286 return (ppa.lbk << dev->ppaf.lbk_offset) | 287 (ppa.chk << dev->ppaf.chk_offset) | 288 (ppa.pu << dev->ppaf.pu_offset) | 289 (ppa.grp << dev->ppaf.grp_offset); 290 } 291 292 static inline struct ftl_ppa 293 ftl_ppa_addr_unpack(const struct spdk_ftl_dev *dev, uint64_t ppa) 294 { 295 struct ftl_ppa res = {}; 296 297 res.lbk = (ppa >> dev->ppaf.lbk_offset) & dev->ppaf.lbk_mask; 298 res.chk = (ppa >> dev->ppaf.chk_offset) & dev->ppaf.chk_mask; 299 res.pu = (ppa >> dev->ppaf.pu_offset) & dev->ppaf.pu_mask; 300 res.grp = (ppa >> dev->ppaf.grp_offset) & dev->ppaf.grp_mask; 301 302 return res; 303 } 304 305 static inline struct ftl_ppa 306 ftl_ppa_to_packed(const struct spdk_ftl_dev *dev, struct ftl_ppa ppa) 307 { 308 struct ftl_ppa p = {}; 309 310 if (ftl_ppa_invalid(ppa)) { 311 p = ftl_to_ppa_packed(FTL_PPA_INVALID); 312 } else if (ftl_ppa_cached(ppa)) { 313 p.pack.cached = 1; 314 p.pack.offset = (uint32_t) ppa.offset; 315 } else { 316 p.pack.ppa = (uint32_t) ftl_ppa_addr_pack(dev, ppa); 317 } 318 319 return p; 320 } 321 322 static inline struct ftl_ppa 323 ftl_ppa_from_packed(const struct spdk_ftl_dev *dev, struct ftl_ppa p) 324 { 325 struct ftl_ppa ppa = {}; 326 327 if (p.pack.ppa == (uint32_t)FTL_PPA_INVALID) { 328 ppa = ftl_to_ppa(FTL_PPA_INVALID); 329 } else if (p.pack.cached) { 330 ppa.cached = 1; 331 ppa.offset = p.pack.offset; 332 } else { 333 ppa = ftl_ppa_addr_unpack(dev, p.pack.ppa); 334 } 335 336 return ppa; 337 } 338 339 static inline unsigned int 340 ftl_ppa_flatten_punit(const struct spdk_ftl_dev *dev, struct ftl_ppa ppa) 341 { 342 return ppa.pu * dev->geo.num_grp + ppa.grp - dev->range.begin; 343 } 344 345 static inline int 346 ftl_ppa_in_range(const struct spdk_ftl_dev *dev, struct ftl_ppa ppa) 347 { 348 unsigned int punit = ftl_ppa_flatten_punit(dev, ppa) + dev->range.begin; 349 350 if (punit >= dev->range.begin && punit <= dev->range.end) { 351 return 1; 352 } 353 354 return 0; 355 } 356 357 #define _ftl_l2p_set(l2p, off, val, bits) \ 358 __atomic_store_n(((uint##bits##_t *)(l2p)) + (off), val, __ATOMIC_SEQ_CST) 359 360 #define _ftl_l2p_set32(l2p, off, val) \ 361 _ftl_l2p_set(l2p, off, val, 32) 362 363 #define _ftl_l2p_set64(l2p, off, val) \ 364 _ftl_l2p_set(l2p, off, val, 64) 365 366 #define _ftl_l2p_get(l2p, off, bits) \ 367 __atomic_load_n(((uint##bits##_t *)(l2p)) + (off), __ATOMIC_SEQ_CST) 368 369 #define _ftl_l2p_get32(l2p, off) \ 370 _ftl_l2p_get(l2p, off, 32) 371 372 #define _ftl_l2p_get64(l2p, off) \ 373 _ftl_l2p_get(l2p, off, 64) 374 375 #define ftl_ppa_cmp(p1, p2) \ 376 ((p1).ppa == (p2).ppa) 377 378 static inline void 379 ftl_l2p_set(struct spdk_ftl_dev *dev, uint64_t lba, struct ftl_ppa ppa) 380 { 381 assert(dev->num_lbas > lba); 382 383 if (ftl_ppa_packed(dev)) { 384 _ftl_l2p_set32(dev->l2p, lba, ftl_ppa_to_packed(dev, ppa).ppa); 385 } else { 386 _ftl_l2p_set64(dev->l2p, lba, ppa.ppa); 387 } 388 } 389 390 static inline struct ftl_ppa 391 ftl_l2p_get(struct spdk_ftl_dev *dev, uint64_t lba) 392 { 393 assert(dev->num_lbas > lba); 394 395 if (ftl_ppa_packed(dev)) { 396 return ftl_ppa_from_packed(dev, ftl_to_ppa_packed( 397 _ftl_l2p_get32(dev->l2p, lba))); 398 } else { 399 return ftl_to_ppa(_ftl_l2p_get64(dev->l2p, lba)); 400 } 401 } 402 static inline size_t 403 ftl_dev_num_bands(const struct spdk_ftl_dev *dev) 404 { 405 return dev->geo.num_chk; 406 } 407 408 static inline size_t 409 ftl_dev_lbks_in_chunk(const struct spdk_ftl_dev *dev) 410 { 411 return dev->geo.clba; 412 } 413 414 static inline size_t 415 ftl_dev_num_punits(const struct spdk_ftl_dev *dev) 416 { 417 return dev->range.end - dev->range.begin + 1; 418 } 419 420 static inline uint64_t 421 ftl_num_band_lbks(const struct spdk_ftl_dev *dev) 422 { 423 return ftl_dev_num_punits(dev) * ftl_dev_lbks_in_chunk(dev); 424 } 425 426 static inline size_t 427 ftl_vld_map_size(const struct spdk_ftl_dev *dev) 428 { 429 return (size_t)spdk_divide_round_up(ftl_num_band_lbks(dev), CHAR_BIT); 430 } 431 432 #endif /* FTL_CORE_H */ 433