xref: /spdk/lib/ftl/ftl_io.c (revision e967dcd245f096f102d811e5c6d8aeb96c172e3e) 
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright (c) Intel Corporation.
5  *   All rights reserved.
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of Intel Corporation nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33 
34 #include "spdk/stdinc.h"
35 #include "spdk/ftl.h"
36 #include "spdk/likely.h"
37 
38 #include "ftl_io.h"
39 #include "ftl_core.h"
40 #include "ftl_rwb.h"
41 #include "ftl_band.h"
42 
43 void
44 ftl_io_inc_req(struct ftl_io *io)
45 {
46 	struct ftl_band *band = io->band;
47 
48 	if (!(io->flags & FTL_IO_CACHE) && io->type != FTL_IO_READ && io->type != FTL_IO_ERASE) {
49 		ftl_band_acquire_md(band);
50 	}
51 
52 	__atomic_fetch_add(&io->dev->num_inflight, 1, __ATOMIC_SEQ_CST);
53 
54 	++io->req_cnt;
55 }
56 
57 void
58 ftl_io_dec_req(struct ftl_io *io)
59 {
60 	struct ftl_band *band = io->band;
61 	unsigned long num_inflight __attribute__((unused));
62 
63 	if (!(io->flags & FTL_IO_CACHE) && io->type != FTL_IO_READ && io->type != FTL_IO_ERASE) {
64 		ftl_band_release_md(band);
65 	}
66 
67 	num_inflight = __atomic_fetch_sub(&io->dev->num_inflight, 1, __ATOMIC_SEQ_CST);
68 
69 	assert(num_inflight > 0);
70 	assert(io->req_cnt > 0);
71 
72 	--io->req_cnt;
73 }
74 
75 struct iovec *
76 ftl_io_iovec(struct ftl_io *io)
77 {
78 	if (io->iov_cnt > 1) {
79 		return io->iov.vector;
80 	} else {
81 		return &io->iov.single;
82 	}
83 }
84 
85 uint64_t
86 ftl_io_get_lba(const struct ftl_io *io, size_t offset)
87 {
88 	assert(offset < io->lbk_cnt);
89 
90 	if (io->flags & FTL_IO_VECTOR_LBA) {
91 		return io->lba.vector[offset];
92 	} else {
93 		return io->lba.single + offset;
94 	}
95 }
96 
97 uint64_t
98 ftl_io_current_lba(const struct ftl_io *io)
99 {
100 	return ftl_io_get_lba(io, io->pos);
101 }
102 
103 void
104 ftl_io_advance(struct ftl_io *io, size_t lbk_cnt)
105 {
106 	struct iovec *iov = ftl_io_iovec(io);
107 	size_t iov_lbks;
108 
109 	io->pos += lbk_cnt;
110 
111 	if (io->iov_cnt == 0) {
112 		return;
113 	}
114 
115 	while (lbk_cnt > 0) {
116 		assert(io->iov_pos < io->iov_cnt);
117 		iov_lbks = iov[io->iov_pos].iov_len / PAGE_SIZE;
118 
119 		if (io->iov_off + lbk_cnt < iov_lbks) {
120 			io->iov_off += lbk_cnt;
121 			break;
122 		}
123 
124 		assert(iov_lbks > io->iov_off);
125 		lbk_cnt -= (iov_lbks - io->iov_off);
126 		io->iov_off = 0;
127 		io->iov_pos++;
128 	}
129 }
130 
131 size_t
132 ftl_iovec_num_lbks(struct iovec *iov, size_t iov_cnt)
133 {
134 	size_t lbks = 0, i = 0;
135 
136 	for (; i < iov_cnt; ++i) {
137 		lbks += iov[i].iov_len / PAGE_SIZE;
138 	}
139 
140 	return lbks;
141 }
142 
143 void *
144 ftl_io_iovec_addr(struct ftl_io *io)
145 {
146 	assert(io->iov_pos < io->iov_cnt);
147 	assert(io->iov_off * PAGE_SIZE < ftl_io_iovec(io)[io->iov_pos].iov_len);
148 
149 	return (char *)ftl_io_iovec(io)[io->iov_pos].iov_base +
150 	       io->iov_off * PAGE_SIZE;
151 }
152 
153 size_t
154 ftl_io_iovec_len_left(struct ftl_io *io)
155 {
156 	struct iovec *iov = ftl_io_iovec(io);
157 	return iov[io->iov_pos].iov_len / PAGE_SIZE - io->iov_off;
158 }
159 
160 static void
161 _ftl_io_init_iovec(struct ftl_io *io, void *buf, size_t iov_cnt, size_t req_size)
162 {
163 	struct iovec *iov;
164 	size_t i;
165 
166 	io->iov_pos = 0;
167 	io->iov_cnt = iov_cnt;
168 	io->lbk_cnt = iov_cnt * req_size;
169 
170 	iov = ftl_io_iovec(io);
171 	for (i = 0; i < iov_cnt; ++i) {
172 		iov[i].iov_base = (char *)buf + i * req_size * PAGE_SIZE;
173 		iov[i].iov_len = req_size * PAGE_SIZE;
174 	}
175 }
176 
177 static int
178 ftl_io_init_iovec(struct ftl_io *io, void *buf, size_t iov_cnt, size_t req_size)
179 {
180 	if (iov_cnt > 1) {
181 		io->iov.vector = calloc(iov_cnt, sizeof(struct iovec));
182 		if (!io->iov.vector) {
183 			return -ENOMEM;
184 		}
185 	}
186 
187 	_ftl_io_init_iovec(io, buf, iov_cnt, req_size);
188 
189 	return 0;
190 }
191 
192 void
193 ftl_io_shrink_iovec(struct ftl_io *io, char *buf, size_t iov_cnt, size_t req_size)
194 {
195 	assert(io->iov_cnt >= iov_cnt);
196 	assert(io->lbk_cnt >= iov_cnt * req_size);
197 	assert(io->pos == 0 && io->iov_pos == 0 && io->iov_off == 0);
198 
199 	if (iov_cnt == 1 && io->iov_cnt > 1) {
200 		free(io->iov.vector);
201 	}
202 
203 	_ftl_io_init_iovec(io, buf, iov_cnt, req_size);
204 }
205 
206 static void
207 ftl_io_init(struct ftl_io *io, struct spdk_ftl_dev *dev,
208 	    spdk_ftl_fn fn, void *ctx, int flags, int type)
209 {
210 	io->flags |= flags | FTL_IO_INITIALIZED;
211 	io->type = type;
212 	io->dev = dev;
213 	io->lba.single = FTL_LBA_INVALID;
214 	io->ppa.ppa = FTL_PPA_INVALID;
215 	io->cb.fn = fn;
216 	io->cb.ctx = ctx;
217 	io->trace = ftl_trace_alloc_id(dev);
218 }
219 
220 struct ftl_io *
221 ftl_io_init_internal(const struct ftl_io_init_opts *opts)
222 {
223 	struct ftl_io *io = opts->io;
224 	struct spdk_ftl_dev *dev = opts->dev;
225 
226 	if (!io) {
227 		if (opts->parent) {
228 			io = ftl_io_alloc_child(opts->parent);
229 		} else {
230 			io = ftl_io_alloc(dev->ioch);
231 		}
232 
233 		if (!io) {
234 			return NULL;
235 		}
236 	}
237 
238 	ftl_io_clear(io);
239 	ftl_io_init(io, dev, opts->fn, io, opts->flags | FTL_IO_INTERNAL, opts->type);
240 
241 	io->rwb_batch = opts->rwb_batch;
242 	io->band = opts->band;
243 	io->md = opts->md;
244 
245 	if (ftl_io_init_iovec(io, opts->data, opts->iov_cnt, opts->req_size)) {
246 		if (!opts->io) {
247 			ftl_io_free(io);
248 		}
249 		return NULL;
250 	}
251 
252 	return io;
253 }
254 
255 struct ftl_io *
256 ftl_io_rwb_init(struct spdk_ftl_dev *dev, struct ftl_band *band,
257 		struct ftl_rwb_batch *batch, spdk_ftl_fn cb)
258 {
259 	struct ftl_io_init_opts opts = {
260 		.dev		= dev,
261 		.io		= NULL,
262 		.rwb_batch	= batch,
263 		.band		= band,
264 		.size		= sizeof(struct ftl_io),
265 		.flags		= 0,
266 		.type		= FTL_IO_WRITE,
267 		.iov_cnt	= 1,
268 		.req_size	= dev->xfer_size,
269 		.fn		= cb,
270 		.data		= ftl_rwb_batch_get_data(batch),
271 		.md		= ftl_rwb_batch_get_md(batch),
272 	};
273 
274 	return ftl_io_init_internal(&opts);
275 }
276 
277 struct ftl_io *
278 ftl_io_erase_init(struct ftl_band *band, size_t lbk_cnt, spdk_ftl_fn cb)
279 {
280 	struct ftl_io *io;
281 	struct ftl_io_init_opts opts = {
282 		.dev		= band->dev,
283 		.io		= NULL,
284 		.rwb_batch	= NULL,
285 		.band		= band,
286 		.size		= sizeof(struct ftl_io),
287 		.flags		= FTL_IO_PPA_MODE,
288 		.type		= FTL_IO_ERASE,
289 		.iov_cnt	= 0,
290 		.req_size	= 1,
291 		.fn		= cb,
292 		.data		= NULL,
293 		.md		= NULL,
294 	};
295 
296 	io = ftl_io_init_internal(&opts);
297 	if (!io) {
298 		return NULL;
299 	}
300 
301 	io->lbk_cnt = lbk_cnt;
302 
303 	return io;
304 }
305 
306 void
307 ftl_io_user_init(struct spdk_ftl_dev *dev, struct ftl_io *io, uint64_t lba, size_t lbk_cnt,
308 		 struct iovec *iov, size_t iov_cnt,
309 		 spdk_ftl_fn cb_fn, void *cb_arg, int type)
310 {
311 	if (io->flags & FTL_IO_INITIALIZED) {
312 		return;
313 	}
314 
315 	ftl_io_init(io, dev, cb_fn, cb_arg, 0, type);
316 
317 	io->lba.single = lba;
318 	io->lbk_cnt = lbk_cnt;
319 	io->iov_cnt = iov_cnt;
320 
321 	if (iov_cnt > 1) {
322 		io->iov.vector = iov;
323 	} else {
324 		io->iov.single = *iov;
325 	}
326 
327 	ftl_trace_lba_io_init(io->dev, io);
328 }
329 
330 static void
331 _ftl_io_free(struct ftl_io *io)
332 {
333 	struct ftl_io_channel *ioch;
334 
335 	assert(LIST_EMPTY(&io->children));
336 
337 	if ((io->flags & FTL_IO_INTERNAL) && io->iov_cnt > 1) {
338 		free(io->iov.vector);
339 	}
340 
341 	if (pthread_spin_destroy(&io->lock)) {
342 		SPDK_ERRLOG("pthread_spin_destroy failed\n");
343 	}
344 
345 	ioch = spdk_io_channel_get_ctx(io->ioch);
346 	spdk_mempool_put(ioch->io_pool, io);
347 }
348 
349 static bool
350 ftl_io_remove_child(struct ftl_io *io)
351 {
352 	struct ftl_io *parent = io->parent;
353 	bool parent_done;
354 
355 	pthread_spin_lock(&parent->lock);
356 	LIST_REMOVE(io, child_entry);
357 	parent_done = parent->done && LIST_EMPTY(&parent->children);
358 	parent->status = parent->status ? : io->status;
359 	pthread_spin_unlock(&parent->lock);
360 
361 	return parent_done;
362 }
363 
364 void
365 ftl_io_complete(struct ftl_io *io)
366 {
367 	struct ftl_io *parent = io->parent;
368 	bool complete, keep_alive = io->flags & FTL_IO_KEEP_ALIVE;
369 
370 	io->flags &= ~FTL_IO_INITIALIZED;
371 
372 	pthread_spin_lock(&io->lock);
373 	complete = LIST_EMPTY(&io->children);
374 	io->done = true;
375 	pthread_spin_unlock(&io->lock);
376 
377 	if (complete) {
378 		if (io->cb.fn) {
379 			io->cb.fn(io->cb.ctx, io->status);
380 		}
381 
382 		if (parent && ftl_io_remove_child(io)) {
383 			ftl_io_complete(parent);
384 		}
385 
386 		if (!keep_alive) {
387 			_ftl_io_free(io);
388 		}
389 	}
390 }
391 
392 struct ftl_io *
393 ftl_io_alloc_child(struct ftl_io *parent)
394 {
395 	struct ftl_io *io;
396 
397 	io = ftl_io_alloc(parent->ioch);
398 	if (spdk_unlikely(!io)) {
399 		return NULL;
400 	}
401 
402 	io->parent = parent;
403 
404 	pthread_spin_lock(&parent->lock);
405 	LIST_INSERT_HEAD(&parent->children, io, child_entry);
406 	pthread_spin_unlock(&parent->lock);
407 
408 	return io;
409 }
410 
411 void
412 ftl_io_process_error(struct ftl_io *io, const struct spdk_nvme_cpl *status)
413 {
414 	/* TODO: add error handling for specifc cases */
415 	if (status->status.sct == SPDK_NVME_SCT_MEDIA_ERROR &&
416 	    status->status.sc == SPDK_OCSSD_SC_READ_HIGH_ECC) {
417 		return;
418 	}
419 
420 	io->status = -EIO;
421 }
422 
423 void ftl_io_fail(struct ftl_io *io, int status)
424 {
425 	io->status = status;
426 	ftl_io_advance(io, io->lbk_cnt - io->pos);
427 }
428 
429 void *
430 ftl_io_get_md(const struct ftl_io *io)
431 {
432 	if (!io->md) {
433 		return NULL;
434 	}
435 
436 	return (char *)io->md + io->pos * FTL_BLOCK_SIZE;
437 }
438 
439 struct ftl_io *
440 ftl_io_alloc(struct spdk_io_channel *ch)
441 {
442 	struct ftl_io *io;
443 	struct ftl_io_channel *ioch = spdk_io_channel_get_ctx(ch);
444 
445 	io = spdk_mempool_get(ioch->io_pool);
446 	if (!io) {
447 		return NULL;
448 	}
449 
450 	memset(io, 0, ioch->elem_size);
451 	io->ioch = ch;
452 
453 	if (pthread_spin_init(&io->lock, PTHREAD_PROCESS_PRIVATE)) {
454 		SPDK_ERRLOG("pthread_spin_init failed\n");
455 		spdk_mempool_put(ioch->io_pool, io);
456 		return NULL;
457 	}
458 
459 	return io;
460 }
461 
462 void
463 ftl_io_reinit(struct ftl_io *io, spdk_ftl_fn fn, void *ctx, int flags, int type)
464 {
465 	ftl_io_clear(io);
466 	ftl_io_init(io, io->dev, fn, ctx, flags, type);
467 }
468 
469 void
470 ftl_io_clear(struct ftl_io *io)
471 {
472 	ftl_io_reset(io);
473 
474 	io->flags = 0;
475 	io->rwb_batch = NULL;
476 	io->band = NULL;
477 }
478 
479 void
480 ftl_io_reset(struct ftl_io *io)
481 {
482 	io->req_cnt = io->pos = io->iov_pos = io->iov_off = 0;
483 	io->done = false;
484 }
485 
486 void
487 ftl_io_free(struct ftl_io *io)
488 {
489 	struct ftl_io *parent = io->parent;
490 
491 	if (!io) {
492 		return;
493 	}
494 
495 	if (parent && ftl_io_remove_child(io)) {
496 		ftl_io_complete(parent);
497 	}
498 
499 	_ftl_io_free(io);
500 }
501