xref: /spdk/lib/ftl/ftl_io.c (revision 0ed66e7ef6965ba59db2c57aa64e9898f12c302d) 
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright (c) Intel Corporation.
5  *   All rights reserved.
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of Intel Corporation nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33 
34 #include "spdk/stdinc.h"
35 #include "spdk/ftl.h"
36 
37 #include "ftl_io.h"
38 #include "ftl_core.h"
39 #include "ftl_rwb.h"
40 #include "ftl_band.h"
41 
42 void
43 ftl_io_inc_req(struct ftl_io *io)
44 {
45 	struct ftl_band *band = io->band;
46 
47 	if (io->type != FTL_IO_READ && io->type != FTL_IO_ERASE) {
48 		ftl_band_acquire_md(band);
49 	}
50 
51 	__atomic_fetch_add(&io->dev->num_inflight, 1, __ATOMIC_SEQ_CST);
52 
53 	++io->req_cnt;
54 }
55 
56 void
57 ftl_io_dec_req(struct ftl_io *io)
58 {
59 	struct ftl_band *band = io->band;
60 	unsigned long num_inflight __attribute__((unused));
61 
62 	if (io->type != FTL_IO_READ && io->type != FTL_IO_ERASE) {
63 		ftl_band_release_md(band);
64 	}
65 
66 	num_inflight = __atomic_fetch_sub(&io->dev->num_inflight, 1, __ATOMIC_SEQ_CST);
67 
68 	assert(num_inflight > 0);
69 	assert(io->req_cnt > 0);
70 
71 	--io->req_cnt;
72 }
73 
74 struct iovec *
75 ftl_io_iovec(struct ftl_io *io)
76 {
77 	if (io->iov_cnt > 1) {
78 		return io->iovs;
79 	} else {
80 		return &io->iov;
81 	}
82 }
83 
84 uint64_t
85 ftl_io_current_lba(struct ftl_io *io)
86 {
87 	if (io->flags & FTL_IO_VECTOR_LBA) {
88 		return io->lbas[io->pos];
89 	} else {
90 		return io->lba + io->pos;
91 	}
92 }
93 
94 void
95 ftl_io_update_iovec(struct ftl_io *io, size_t lbk_cnt)
96 {
97 	struct iovec *iov = ftl_io_iovec(io);
98 	size_t iov_lbks;
99 
100 	io->pos += lbk_cnt;
101 
102 	while (lbk_cnt > 0) {
103 		assert(io->iov_pos < io->iov_cnt);
104 		iov_lbks = iov[io->iov_pos].iov_len / PAGE_SIZE;
105 
106 		if (io->iov_off + lbk_cnt < iov_lbks) {
107 			io->iov_off += lbk_cnt;
108 			break;
109 		}
110 
111 		assert(iov_lbks > io->iov_off);
112 		lbk_cnt -= (iov_lbks - io->iov_off);
113 		io->iov_off = 0;
114 		io->iov_pos++;
115 	}
116 }
117 
118 size_t
119 ftl_iovec_num_lbks(struct iovec *iov, size_t iov_cnt)
120 {
121 	size_t lbks = 0, i = 0;
122 
123 	for (; i < iov_cnt; ++i) {
124 		lbks += iov[i].iov_len / PAGE_SIZE;
125 	}
126 
127 	return lbks;
128 }
129 
130 void *
131 ftl_io_iovec_addr(struct ftl_io *io)
132 {
133 	assert(io->iov_pos < io->iov_cnt);
134 	assert(io->iov_off * PAGE_SIZE < ftl_io_iovec(io)[io->iov_pos].iov_len);
135 
136 	return (char *)ftl_io_iovec(io)[io->iov_pos].iov_base +
137 	       io->iov_off * PAGE_SIZE;
138 }
139 
140 size_t
141 ftl_io_iovec_len_left(struct ftl_io *io)
142 {
143 	struct iovec *iov = ftl_io_iovec(io);
144 	return iov[io->iov_pos].iov_len / PAGE_SIZE - io->iov_off;
145 }
146 
147 int
148 ftl_io_init_iovec(struct ftl_io *io, void *buf,
149 		  size_t iov_cnt, size_t req_size)
150 {
151 	struct iovec *iov;
152 	size_t i;
153 
154 	if (iov_cnt > 1) {
155 		iov = io->iovs = calloc(iov_cnt, sizeof(*iov));
156 		if (!iov) {
157 			return -ENOMEM;
158 		}
159 	} else {
160 		iov = &io->iov;
161 	}
162 
163 	io->iov_pos = 0;
164 	io->iov_cnt = iov_cnt;
165 	for (i = 0; i < iov_cnt; ++i) {
166 		iov[i].iov_base = (char *)buf + i * req_size * PAGE_SIZE;
167 		iov[i].iov_len = req_size * PAGE_SIZE;
168 	}
169 
170 	return 0;
171 }
172 
173 static void
174 ftl_io_init(struct ftl_io *io, struct spdk_ftl_dev *dev,
175 	    spdk_ftl_fn fn, void *ctx, int flags, int type)
176 {
177 	io->flags |= flags | FTL_IO_INITIALIZED;
178 	io->type = type;
179 	io->dev = dev;
180 	io->lba = FTL_LBA_INVALID;
181 	io->cb.fn = fn;
182 	io->cb.ctx = ctx;
183 	io->trace = ftl_trace_alloc_id(dev);
184 }
185 
186 struct ftl_io *
187 ftl_io_init_internal(const struct ftl_io_init_opts *opts)
188 {
189 	struct ftl_io *io = opts->io;
190 	struct spdk_ftl_dev *dev = opts->dev;
191 
192 	if (!io) {
193 		io = ftl_io_alloc(dev->ioch);
194 		if (!io) {
195 			return NULL;
196 		}
197 	}
198 
199 	ftl_io_clear(io);
200 	ftl_io_init(io, dev, opts->fn, io, opts->flags | FTL_IO_INTERNAL, opts->type);
201 
202 	io->lbk_cnt = opts->iov_cnt * opts->req_size;
203 	io->rwb_batch = opts->rwb_batch;
204 	io->band = opts->band;
205 	io->md = opts->md;
206 
207 	if (ftl_io_init_iovec(io, opts->data, opts->iov_cnt, opts->req_size)) {
208 		if (!opts->io) {
209 			ftl_io_free(io);
210 		}
211 		return NULL;
212 	}
213 
214 	return io;
215 }
216 
217 struct ftl_io *
218 ftl_io_rwb_init(struct spdk_ftl_dev *dev, struct ftl_band *band,
219 		struct ftl_rwb_batch *batch, spdk_ftl_fn cb)
220 {
221 	struct ftl_io_init_opts opts = {
222 		.dev		= dev,
223 		.io		= NULL,
224 		.rwb_batch	= batch,
225 		.band		= band,
226 		.size		= sizeof(struct ftl_io),
227 		.flags		= 0,
228 		.type		= FTL_IO_WRITE,
229 		.iov_cnt	= 1,
230 		.req_size	= dev->xfer_size,
231 		.fn		= cb,
232 		.data		= ftl_rwb_batch_get_data(batch),
233 		.md		= ftl_rwb_batch_get_md(batch),
234 	};
235 
236 	return ftl_io_init_internal(&opts);
237 }
238 
239 struct ftl_io *
240 ftl_io_erase_init(struct ftl_band *band, size_t lbk_cnt, spdk_ftl_fn cb)
241 {
242 	struct ftl_io *io;
243 	struct ftl_io_init_opts opts = {
244 		.dev		= band->dev,
245 		.io		= NULL,
246 		.rwb_batch	= NULL,
247 		.band		= band,
248 		.size		= sizeof(struct ftl_io),
249 		.flags		= FTL_IO_PPA_MODE,
250 		.type		= FTL_IO_ERASE,
251 		.iov_cnt	= 0,
252 		.req_size	= 1,
253 		.fn		= cb,
254 		.data		= NULL,
255 		.md		= NULL,
256 	};
257 
258 	io = ftl_io_init_internal(&opts);
259 	io->lbk_cnt = lbk_cnt;
260 
261 	return io;
262 }
263 
264 void
265 ftl_io_user_init(struct spdk_ftl_dev *dev, struct ftl_io *io, uint64_t lba, size_t lbk_cnt,
266 		 struct iovec *iov, size_t iov_cnt,
267 		 spdk_ftl_fn cb_fn, void *cb_arg, int type)
268 {
269 	if (io->flags & FTL_IO_INITIALIZED) {
270 		return;
271 	}
272 
273 	ftl_io_init(io, dev, cb_fn, cb_arg, 0, type);
274 
275 	io->lba = lba;
276 	io->lbk_cnt = lbk_cnt;
277 	io->iov_cnt = iov_cnt;
278 
279 	if (iov_cnt > 1) {
280 		io->iovs = iov;
281 	} else {
282 		io->iov = *iov;
283 	}
284 
285 	ftl_trace_lba_io_init(io->dev, io);
286 }
287 
288 void
289 ftl_io_complete(struct ftl_io *io)
290 {
291 	int keep_alive = io->flags & FTL_IO_KEEP_ALIVE;
292 
293 	io->flags &= ~FTL_IO_INITIALIZED;
294 	io->cb.fn(io->cb.ctx, io->status);
295 
296 	if (!keep_alive) {
297 		ftl_io_free(io);
298 	}
299 }
300 
301 void
302 ftl_io_process_error(struct ftl_io *io, const struct spdk_nvme_cpl *status)
303 {
304 	/* TODO: add error handling for specifc cases */
305 	if (status->status.sct == SPDK_NVME_SCT_MEDIA_ERROR &&
306 	    status->status.sc == SPDK_OCSSD_SC_READ_HIGH_ECC) {
307 		return;
308 	}
309 
310 	io->status = -EIO;
311 }
312 
313 void *
314 ftl_io_get_md(const struct ftl_io *io)
315 {
316 	if (!io->md) {
317 		return NULL;
318 	}
319 
320 	return (char *)io->md + io->pos * FTL_BLOCK_SIZE;
321 }
322 
323 struct ftl_io *
324 ftl_io_alloc(struct spdk_io_channel *ch)
325 {
326 	struct ftl_io *io;
327 	struct ftl_io_channel *ioch = spdk_io_channel_get_ctx(ch);
328 
329 	io = spdk_mempool_get(ioch->io_pool);
330 	if (!io) {
331 		return NULL;
332 	}
333 
334 	memset(io, 0, ioch->elem_size);
335 	io->ch = ch;
336 	return io;
337 }
338 
339 void
340 ftl_io_reinit(struct ftl_io *io, spdk_ftl_fn fn, void *ctx, int flags, int type)
341 {
342 	ftl_io_clear(io);
343 	ftl_io_init(io, io->dev, fn, ctx, flags, type);
344 }
345 
346 void
347 ftl_io_clear(struct ftl_io *io)
348 {
349 	io->pos = 0;
350 	io->req_cnt = 0;
351 	io->iov_pos = 0;
352 	io->iov_off = 0;
353 	io->flags = 0;
354 	io->rwb_batch = NULL;
355 	io->band = NULL;
356 }
357 
358 void
359 ftl_io_free(struct ftl_io *io)
360 {
361 	struct ftl_io_channel *ioch;
362 
363 	if (!io) {
364 		return;
365 	}
366 
367 	if ((io->flags & FTL_IO_INTERNAL) && io->iov_cnt > 1) {
368 		free(io->iovs);
369 	}
370 
371 	ioch = spdk_io_channel_get_ctx(io->ch);
372 	spdk_mempool_put(ioch->io_pool, io);
373 }
374