1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #ifndef FTL_IO_H 35 #define FTL_IO_H 36 37 #include "spdk/stdinc.h" 38 #include "spdk/nvme.h" 39 #include "spdk/ftl.h" 40 41 #include "ftl_addr.h" 42 #include "ftl_trace.h" 43 44 struct spdk_ftl_dev; 45 struct ftl_band; 46 struct ftl_batch; 47 struct ftl_io; 48 49 typedef int (*ftl_md_pack_fn)(struct ftl_band *); 50 typedef void (*ftl_io_fn)(struct ftl_io *, void *, int); 51 52 /* IO flags */ 53 enum ftl_io_flags { 54 /* Indicates whether IO is already initialized */ 55 FTL_IO_INITIALIZED = (1 << 0), 56 /* Internal based IO (defrag, metadata etc.) */ 57 FTL_IO_INTERNAL = (1 << 1), 58 /* Indicates that the IO should not go through if there's */ 59 /* already another one scheduled to the same LBA */ 60 FTL_IO_WEAK = (1 << 2), 61 /* Indicates that the IO is used for padding */ 62 FTL_IO_PAD = (1 << 3), 63 /* The IO operates on metadata */ 64 FTL_IO_MD = (1 << 4), 65 /* Using physical instead of logical address */ 66 FTL_IO_PHYSICAL_MODE = (1 << 5), 67 /* Indicates that IO contains noncontiguous LBAs */ 68 FTL_IO_VECTOR_LBA = (1 << 6), 69 /* The IO is directed to non-volatile cache */ 70 FTL_IO_CACHE = (1 << 7), 71 /* Indicates that physical address should be taken from IO struct, */ 72 /* not assigned by wptr, only works if wptr is also in direct mode */ 73 FTL_IO_DIRECT_ACCESS = (1 << 8), 74 /* Bypass the non-volatile cache */ 75 FTL_IO_BYPASS_CACHE = (1 << 9), 76 }; 77 78 enum ftl_io_type { 79 FTL_IO_READ, 80 FTL_IO_WRITE, 81 FTL_IO_ERASE, 82 }; 83 84 #define FTL_IO_MAX_IOVEC 64 85 86 struct ftl_io_init_opts { 87 struct spdk_ftl_dev *dev; 88 89 /* IO descriptor */ 90 struct ftl_io *io; 91 92 /* Parent request */ 93 struct ftl_io *parent; 94 95 /* Size of IO descriptor */ 96 size_t size; 97 98 /* IO flags */ 99 int flags; 100 101 /* IO type */ 102 enum ftl_io_type type; 103 104 /* Transfer batch, set for IO going through the write buffer */ 105 struct ftl_batch *batch; 106 107 /* Band to which the IO is directed */ 108 struct ftl_band *band; 109 110 /* Number of logical blocks */ 111 size_t num_blocks; 112 113 /* Data */ 114 struct iovec iovs[FTL_IO_MAX_IOVEC]; 115 int iovcnt; 116 117 /* Metadata */ 118 void *md; 119 120 /* Callback's function */ 121 ftl_io_fn cb_fn; 122 123 /* Callback's context */ 124 void *cb_ctx; 125 }; 126 127 struct ftl_io_channel; 128 129 struct ftl_wbuf_entry { 130 /* IO channel that owns the write buffer entry */ 131 struct ftl_io_channel *ioch; 132 /* Data payload (single block) */ 133 void *payload; 134 /* Index within the IO channel's wbuf_entries array */ 135 uint32_t index; 136 uint32_t io_flags; 137 /* Points at the band the data is copied from. Only valid for internal 138 * requests coming from reloc. 139 */ 140 struct ftl_band *band; 141 /* Physical address of that particular block. Valid once the data has 142 * been written out. 143 */ 144 struct ftl_addr addr; 145 /* Logical block address */ 146 uint64_t lba; 147 148 /* Trace ID of the requests the entry is part of */ 149 uint64_t trace; 150 151 /* Indicates that the entry was written out and is still present in the 152 * L2P table. 153 */ 154 bool valid; 155 /* Lock that protects the entry from being evicted from the L2P */ 156 pthread_spinlock_t lock; 157 TAILQ_ENTRY(ftl_wbuf_entry) tailq; 158 }; 159 160 #define FTL_IO_CHANNEL_INDEX_INVALID ((uint64_t)-1) 161 162 struct ftl_io_channel { 163 /* Device */ 164 struct spdk_ftl_dev *dev; 165 /* IO pool element size */ 166 size_t elem_size; 167 /* Index within the IO channel array */ 168 uint64_t index; 169 /* IO pool */ 170 struct spdk_mempool *io_pool; 171 /* Underlying device IO channel */ 172 struct spdk_io_channel *base_ioch; 173 /* Persistent cache IO channel */ 174 struct spdk_io_channel *cache_ioch; 175 /* Poller used for completing write requests and retrying IO */ 176 struct spdk_poller *poller; 177 /* Write completion queue */ 178 TAILQ_HEAD(, ftl_io) write_cmpl_queue; 179 TAILQ_HEAD(, ftl_io) retry_queue; 180 TAILQ_ENTRY(ftl_io_channel) tailq; 181 182 /* Array of write buffer entries */ 183 struct ftl_wbuf_entry *wbuf_entries; 184 /* Write buffer data payload */ 185 void *wbuf_payload; 186 /* Number of write buffer entries */ 187 uint32_t num_entries; 188 /* Write buffer queues */ 189 struct spdk_ring *free_queue; 190 struct spdk_ring *submit_queue; 191 /* Maximum number of concurrent user writes */ 192 uint32_t qdepth_limit; 193 /* Current number of concurrent user writes */ 194 uint32_t qdepth_current; 195 /* Means that the IO channel is being flushed */ 196 bool flush; 197 }; 198 199 /* General IO descriptor */ 200 struct ftl_io { 201 /* Device */ 202 struct spdk_ftl_dev *dev; 203 204 /* IO channel */ 205 struct spdk_io_channel *ioch; 206 207 union { 208 /* LBA table */ 209 uint64_t *vector; 210 211 /* First LBA */ 212 uint64_t single; 213 } lba; 214 215 /* First block address */ 216 struct ftl_addr addr; 217 218 /* Number of processed blocks */ 219 size_t pos; 220 221 /* Number of blocks */ 222 size_t num_blocks; 223 224 /* IO vector pointer */ 225 struct iovec *iov; 226 227 /* IO vector buffer for internal requests */ 228 struct iovec iov_buf[FTL_IO_MAX_IOVEC]; 229 230 /* Metadata */ 231 void *md; 232 233 /* Number of IO vectors */ 234 size_t iov_cnt; 235 236 /* Position within the iovec */ 237 size_t iov_pos; 238 239 /* Offset within the iovec (in blocks) */ 240 size_t iov_off; 241 242 /* Transfer batch (valid only for writes going through the write buffer) */ 243 struct ftl_batch *batch; 244 245 /* Band this IO is being written to */ 246 struct ftl_band *band; 247 248 /* Request status */ 249 int status; 250 251 /* Number of split requests */ 252 size_t req_cnt; 253 254 /* Callback's function */ 255 ftl_io_fn cb_fn; 256 257 /* Callback's context */ 258 void *cb_ctx; 259 260 /* User callback function */ 261 spdk_ftl_fn user_fn; 262 263 /* Flags */ 264 int flags; 265 266 /* IO type */ 267 enum ftl_io_type type; 268 269 /* Done flag */ 270 bool done; 271 272 /* Parent request */ 273 struct ftl_io *parent; 274 /* Child requests list */ 275 LIST_HEAD(, ftl_io) children; 276 /* Child list link */ 277 LIST_ENTRY(ftl_io) child_entry; 278 /* Children lock */ 279 pthread_spinlock_t lock; 280 281 /* Trace group id */ 282 uint64_t trace; 283 284 /* Used by retry and write completion queues */ 285 TAILQ_ENTRY(ftl_io) ioch_entry; 286 }; 287 288 /* Metadata IO */ 289 struct ftl_md_io { 290 /* Parent IO structure */ 291 struct ftl_io io; 292 293 /* Serialization/deserialization callback */ 294 ftl_md_pack_fn pack_fn; 295 296 /* Callback's function */ 297 ftl_io_fn cb_fn; 298 299 /* Callback's context */ 300 void *cb_ctx; 301 }; 302 303 static inline bool 304 ftl_io_mode_physical(const struct ftl_io *io) 305 { 306 return io->flags & FTL_IO_PHYSICAL_MODE; 307 } 308 309 static inline bool 310 ftl_io_mode_logical(const struct ftl_io *io) 311 { 312 return !ftl_io_mode_physical(io); 313 } 314 315 static inline bool 316 ftl_io_done(const struct ftl_io *io) 317 { 318 return io->req_cnt == 0 && io->pos == io->num_blocks; 319 } 320 321 struct ftl_io *ftl_io_alloc(struct spdk_io_channel *ch); 322 struct ftl_io *ftl_io_alloc_child(struct ftl_io *parent); 323 void ftl_io_fail(struct ftl_io *io, int status); 324 void ftl_io_free(struct ftl_io *io); 325 struct ftl_io *ftl_io_init_internal(const struct ftl_io_init_opts *opts); 326 void ftl_io_reinit(struct ftl_io *io, ftl_io_fn cb, 327 void *ctx, int flags, int type); 328 void ftl_io_clear(struct ftl_io *io); 329 void ftl_io_inc_req(struct ftl_io *io); 330 void ftl_io_dec_req(struct ftl_io *io); 331 struct iovec *ftl_io_iovec(struct ftl_io *io); 332 uint64_t ftl_io_current_lba(const struct ftl_io *io); 333 uint64_t ftl_io_get_lba(const struct ftl_io *io, size_t offset); 334 void ftl_io_advance(struct ftl_io *io, size_t num_blocks); 335 size_t ftl_iovec_num_blocks(struct iovec *iov, size_t iov_cnt); 336 void *ftl_io_iovec_addr(struct ftl_io *io); 337 size_t ftl_io_iovec_len_left(struct ftl_io *io); 338 struct ftl_io *ftl_io_wbuf_init(struct spdk_ftl_dev *dev, struct ftl_addr addr, 339 struct ftl_band *band, struct ftl_batch *batch, ftl_io_fn cb); 340 struct ftl_io *ftl_io_erase_init(struct ftl_band *band, size_t num_blocks, ftl_io_fn cb); 341 struct ftl_io *ftl_io_user_init(struct spdk_io_channel *ioch, uint64_t lba, size_t num_blocks, 342 struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn, 343 void *cb_arg, int type); 344 void *ftl_io_get_md(const struct ftl_io *io); 345 void ftl_io_complete(struct ftl_io *io); 346 void ftl_io_shrink_iovec(struct ftl_io *io, size_t num_blocks); 347 void ftl_io_process_error(struct ftl_io *io, const struct spdk_nvme_cpl *status); 348 void ftl_io_reset(struct ftl_io *io); 349 void ftl_io_call_foreach_child(struct ftl_io *io, int (*callback)(struct ftl_io *)); 350 351 #endif /* FTL_IO_H */ 352