1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright (C) 2018 Intel Corporation.
3 * All rights reserved.
4 */
5
6 #ifndef FTL_IO_H
7 #define FTL_IO_H
8
9 #include "spdk/stdinc.h"
10 #include "spdk/nvme.h"
11 #include "spdk/ftl.h"
12 #include "spdk/bdev.h"
13 #include "spdk/util.h"
14
15 #include "ftl_internal.h"
16 #include "ftl_trace.h"
17 #include "ftl_l2p.h"
18 #include "utils/ftl_md.h"
19
20 struct spdk_ftl_dev;
21 struct ftl_band;
22 struct ftl_io;
23
24 typedef void (*ftl_io_fn)(struct ftl_io *, void *, int);
25
26 /* IO flags */
27 enum ftl_io_flags {
28 /* Indicates whether IO is already initialized */
29 FTL_IO_INITIALIZED = (1 << 0),
30 /* Indicated whether the user IO pinned the L2P pages containing LBAs */
31 FTL_IO_PINNED = (1 << 1),
32 };
33
34 enum ftl_io_type {
35 FTL_IO_READ,
36 FTL_IO_WRITE,
37 FTL_IO_TRIM,
38 };
39
40 #define FTL_IO_MAX_IOVEC 4
41
42 struct ftl_io_channel {
43 /* Device */
44 struct spdk_ftl_dev *dev;
45 /* Entry of IO channels queue/list */
46 TAILQ_ENTRY(ftl_io_channel) entry;
47 /* IO map pool */
48 struct ftl_mempool *map_pool;
49 /* Poller used for completing user requests and retrying IO */
50 struct spdk_poller *poller;
51 /* Submission queue */
52 struct spdk_ring *sq;
53 /* Completion queue */
54 struct spdk_ring *cq;
55 };
56
57 /* General IO descriptor for user requests */
58 struct ftl_io {
59 /* Device */
60 struct spdk_ftl_dev *dev;
61
62 /* IO channel */
63 struct spdk_io_channel *ioch;
64
65 /* LBA address */
66 uint64_t lba;
67
68 /* First address of write when sent to cache device */
69 ftl_addr addr;
70
71 /* Number of processed blocks */
72 size_t pos;
73
74 /* Number of blocks */
75 size_t num_blocks;
76
77 /* IO vector pointer */
78 struct iovec *iov;
79
80 /* Metadata */
81 void *md;
82
83 /* Number of IO vectors */
84 size_t iov_cnt;
85
86 /* Position within the io vector array */
87 size_t iov_pos;
88
89 /* Offset within the iovec (in blocks) */
90 size_t iov_off;
91
92 /* Band this IO is being written to */
93 struct ftl_band *band;
94
95 /* Request status */
96 int status;
97
98 /* Number of split requests */
99 size_t req_cnt;
100
101 /* Callback's context */
102 void *cb_ctx;
103
104 /* User callback function */
105 spdk_ftl_fn user_fn;
106
107 /* Flags */
108 int flags;
109
110 /* IO type */
111 enum ftl_io_type type;
112
113 /* Done flag */
114 bool done;
115
116 /* Trace group id */
117 uint64_t trace;
118
119 /* Used by retry and write completion queues */
120 TAILQ_ENTRY(ftl_io) queue_entry;
121
122 /* Reference to the chunk within NV cache */
123 struct ftl_nv_cache_chunk *nv_cache_chunk;
124
125 /* For l2p pinning */
126 struct ftl_l2p_pin_ctx l2p_pin_ctx;
127
128 /* Logical to physical mapping for this IO, number of entries equals to
129 * number of transfer blocks */
130 ftl_addr *map;
131
132 struct spdk_bdev_io_wait_entry bdev_io_wait;
133 };
134
135 /* */
136 struct ftl_rq_entry {
137 /* Data payload of single entry (block) */
138 void *io_payload;
139
140 void *io_md;
141
142 /*
143 * Physical address of block described by ftl_rq_entry.
144 * Valid after write command is completed (due to potential append reordering)
145 */
146 ftl_addr addr;
147
148 /* Logical block address */
149 uint64_t lba;
150
151 /* Sequence id of original chunk where this user data was written to */
152 uint64_t seq_id;
153
154 /* Index of this entry within FTL request */
155 const uint64_t index;
156
157 struct {
158 void *priv;
159 } owner;
160
161 /* If request issued in iterative way, it contains IO information */
162 struct {
163 struct ftl_band *band;
164 } io;
165
166 /* For l2p pinning */
167 struct ftl_l2p_pin_ctx l2p_pin_ctx;
168
169 struct {
170 uint64_t offset_blocks;
171 uint64_t num_blocks;
172 struct spdk_bdev_io_wait_entry wait_entry;
173 } bdev_io;
174 };
175
176 /*
177 * Descriptor used for internal requests (compaction and reloc). May be split into multiple
178 * IO requests (as valid blocks that need to be relocated may not be contiguous) - utilizing
179 * the ftl_rq_entry array
180 */
181 struct ftl_rq {
182 struct spdk_ftl_dev *dev;
183
184 /* Request queue entry */
185 TAILQ_ENTRY(ftl_rq) qentry;
186
187 /* Number of block within the request */
188 uint64_t num_blocks;
189
190 /* Extended metadata for IO. Its size is io_md_size * num_blocks */
191 void *io_md;
192
193 /* Size of extended metadata size for one entry */
194 uint64_t io_md_size;
195
196 /* Payload for IO */
197 void *io_payload;
198
199 /* Request result status */
200 bool success;
201
202 /* Fields for owner of this request */
203 struct {
204 /* End request callback */
205 void (*cb)(struct ftl_rq *rq);
206
207 /* IO error request callback */
208 void (*error)(struct ftl_rq *rq, struct ftl_band *band,
209 uint64_t idx, uint64_t count);
210
211 /* Owner context */
212 void *priv;
213
214 /* This is compaction IO */
215 bool compaction;
216 } owner;
217
218 /* Iterator fields for processing state of the request */
219 struct {
220 uint32_t idx;
221
222 uint32_t count;
223
224 /* Queue depth on this request */
225 uint32_t qd;
226
227 uint32_t remaining;
228 int status;
229 } iter;
230
231 /* Private fields for issuing IO */
232 struct {
233 /* Request physical address, on IO completion set for append device */
234 ftl_addr addr;
235
236 /* Band to which IO is issued */
237 struct ftl_band *band;
238
239 struct spdk_bdev_io_wait_entry bdev_io_wait;
240 } io;
241
242 /* For writing P2L metadata */
243 struct ftl_md_io_entry_ctx md_persist_entry_ctx;
244
245 struct ftl_rq_entry entries[];
246 };
247
248 /* Used for reading/writing P2L map during runtime and recovery */
249 struct ftl_basic_rq {
250 struct spdk_ftl_dev *dev;
251
252 /* Request queue entry */
253 TAILQ_ENTRY(ftl_basic_rq) qentry;
254
255 /* Number of block within the request */
256 uint64_t num_blocks;
257
258 /* Payload for IO */
259 void *io_payload;
260
261 /* Request result status */
262 bool success;
263
264 /* Fields for owner of this request */
265 struct {
266 /* End request callback */
267 void (*cb)(struct ftl_basic_rq *brq);
268
269 /* Owner context */
270 void *priv;
271 } owner;
272
273 /* Private fields for issuing IO */
274 struct {
275 /* Request physical address, on IO completion set for append device */
276 ftl_addr addr;
277
278 /* Band to which IO is issued */
279 struct ftl_band *band;
280
281 /* Chunk to which IO is issued */
282 struct ftl_nv_cache_chunk *chunk;
283
284 struct spdk_bdev_io_wait_entry bdev_io_wait;
285 } io;
286 };
287
288 static inline bool
ftl_rq_entry_loop_assert(struct ftl_rq * rq,struct ftl_rq_entry * entry,uint32_t count)289 ftl_rq_entry_loop_assert(struct ftl_rq *rq, struct ftl_rq_entry *entry, uint32_t count)
290 {
291 assert(entry >= rq->entries);
292 assert(((uintptr_t)entry - (uintptr_t)rq->entries) % sizeof(*entry) == 0);
293 assert(count <= rq->num_blocks);
294
295 return true;
296 }
297
298 #define FTL_RQ_ENTRY_LOOP_FROM(rq, from, entry, count) \
299 for ((entry) = (from); \
300 (entry) < (&(rq)->entries[count]) && ftl_rq_entry_loop_assert(rq, entry, count); (entry)++)
301
302 #define FTL_RQ_ENTRY_LOOP(rq, entry, count) \
303 FTL_RQ_ENTRY_LOOP_FROM(rq, (rq)->entries, entry, count)
304
305 void ftl_io_fail(struct ftl_io *io, int status);
306 void ftl_io_clear(struct ftl_io *io);
307 void ftl_io_inc_req(struct ftl_io *io);
308 void ftl_io_dec_req(struct ftl_io *io);
309 struct iovec *ftl_io_iovec(struct ftl_io *io);
310 uint64_t ftl_io_current_lba(const struct ftl_io *io);
311 uint64_t ftl_io_get_lba(const struct ftl_io *io, size_t offset);
312 void ftl_io_advance(struct ftl_io *io, size_t num_blocks);
313 size_t ftl_iovec_num_blocks(struct iovec *iov, size_t iov_cnt);
314 void *ftl_io_iovec_addr(struct ftl_io *io);
315 size_t ftl_io_iovec_len_left(struct ftl_io *io);
316 int ftl_io_init(struct spdk_io_channel *ioch, struct ftl_io *io, uint64_t lba,
317 size_t num_blocks, struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn,
318 void *cb_arg, int type);
319 void ftl_io_complete(struct ftl_io *io);
320 void ftl_rq_del(struct ftl_rq *rq);
321 struct ftl_rq *ftl_rq_new(struct spdk_ftl_dev *dev, uint32_t io_md_size);
322 void ftl_rq_unpin(struct ftl_rq *rq);
323
324 static inline void
ftl_basic_rq_init(struct spdk_ftl_dev * dev,struct ftl_basic_rq * brq,void * io_payload,uint64_t num_blocks)325 ftl_basic_rq_init(struct spdk_ftl_dev *dev, struct ftl_basic_rq *brq,
326 void *io_payload, uint64_t num_blocks)
327 {
328 brq->dev = dev;
329 brq->io_payload = io_payload;
330 brq->num_blocks = num_blocks;
331 brq->success = false;
332 }
333
334 static inline void
ftl_basic_rq_set_owner(struct ftl_basic_rq * brq,void (* cb)(struct ftl_basic_rq * brq),void * priv)335 ftl_basic_rq_set_owner(struct ftl_basic_rq *brq, void (*cb)(struct ftl_basic_rq *brq), void *priv)
336 {
337 brq->owner.cb = cb;
338 brq->owner.priv = priv;
339 }
340
341 static inline struct ftl_rq *
ftl_rq_from_entry(struct ftl_rq_entry * entry)342 ftl_rq_from_entry(struct ftl_rq_entry *entry)
343 {
344 uint64_t idx = entry->index;
345 struct ftl_rq *rq = SPDK_CONTAINEROF(entry, struct ftl_rq, entries[idx]);
346 return rq;
347 }
348
349
350 static inline bool
ftl_io_done(const struct ftl_io * io)351 ftl_io_done(const struct ftl_io *io)
352 {
353 return io->req_cnt == 0 && io->pos == io->num_blocks;
354 }
355
356 #endif /* FTL_IO_H */
357