xref: /spdk/lib/ftl/ftl_io.c (revision 1966f1eef3f99d9ff26320cff449c6c03017ea66) 
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright (c) Intel Corporation.
5  *   All rights reserved.
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of Intel Corporation nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33 
34 #include "spdk/stdinc.h"
35 #include "spdk/ftl.h"
36 #include "spdk/likely.h"
37 #include "spdk/util.h"
38 
39 #include "ftl_io.h"
40 #include "ftl_core.h"
41 #include "ftl_rwb.h"
42 #include "ftl_band.h"
43 
44 void
45 ftl_io_inc_req(struct ftl_io *io)
46 {
47 	struct ftl_band *band = io->band;
48 
49 	if (!(io->flags & FTL_IO_CACHE) && io->type != FTL_IO_READ && io->type != FTL_IO_ERASE) {
50 		ftl_band_acquire_lba_map(band);
51 	}
52 
53 	__atomic_fetch_add(&io->dev->num_inflight, 1, __ATOMIC_SEQ_CST);
54 
55 	++io->req_cnt;
56 }
57 
58 void
59 ftl_io_dec_req(struct ftl_io *io)
60 {
61 	struct ftl_band *band = io->band;
62 	unsigned long num_inflight __attribute__((unused));
63 
64 	if (!(io->flags & FTL_IO_CACHE) && io->type != FTL_IO_READ && io->type != FTL_IO_ERASE) {
65 		ftl_band_release_lba_map(band);
66 	}
67 
68 	num_inflight = __atomic_fetch_sub(&io->dev->num_inflight, 1, __ATOMIC_SEQ_CST);
69 
70 	assert(num_inflight > 0);
71 	assert(io->req_cnt > 0);
72 
73 	--io->req_cnt;
74 }
75 
76 struct iovec *
77 ftl_io_iovec(struct ftl_io *io)
78 {
79 	return &io->iov[0];
80 }
81 
82 uint64_t
83 ftl_io_get_lba(const struct ftl_io *io, size_t offset)
84 {
85 	assert(offset < io->lbk_cnt);
86 
87 	if (io->flags & FTL_IO_VECTOR_LBA) {
88 		return io->lba.vector[offset];
89 	} else {
90 		return io->lba.single + offset;
91 	}
92 }
93 
94 uint64_t
95 ftl_io_current_lba(const struct ftl_io *io)
96 {
97 	return ftl_io_get_lba(io, io->pos);
98 }
99 
100 void
101 ftl_io_advance(struct ftl_io *io, size_t lbk_cnt)
102 {
103 	struct iovec *iov = ftl_io_iovec(io);
104 	size_t iov_lbks, lbk_left = lbk_cnt;
105 
106 	io->pos += lbk_cnt;
107 
108 	if (io->iov_cnt != 0) {
109 		while (lbk_left > 0) {
110 			assert(io->iov_pos < io->iov_cnt);
111 			iov_lbks = iov[io->iov_pos].iov_len / FTL_BLOCK_SIZE;
112 
113 			if (io->iov_off + lbk_left < iov_lbks) {
114 				io->iov_off += lbk_left;
115 				break;
116 			}
117 
118 			assert(iov_lbks > io->iov_off);
119 			lbk_left -= (iov_lbks - io->iov_off);
120 			io->iov_off = 0;
121 			io->iov_pos++;
122 		}
123 	}
124 
125 	if (io->parent) {
126 		ftl_io_advance(io->parent, lbk_cnt);
127 	}
128 }
129 
130 size_t
131 ftl_iovec_num_lbks(struct iovec *iov, size_t iov_cnt)
132 {
133 	size_t lbks = 0, i = 0;
134 
135 	for (; i < iov_cnt; ++i) {
136 		lbks += iov[i].iov_len / FTL_BLOCK_SIZE;
137 	}
138 
139 	return lbks;
140 }
141 
142 void *
143 ftl_io_iovec_addr(struct ftl_io *io)
144 {
145 	assert(io->iov_pos < io->iov_cnt);
146 	assert(io->iov_off * FTL_BLOCK_SIZE < ftl_io_iovec(io)[io->iov_pos].iov_len);
147 
148 	return (char *)ftl_io_iovec(io)[io->iov_pos].iov_base +
149 	       io->iov_off * FTL_BLOCK_SIZE;
150 }
151 
152 size_t
153 ftl_io_iovec_len_left(struct ftl_io *io)
154 {
155 	struct iovec *iov = ftl_io_iovec(io);
156 	return iov[io->iov_pos].iov_len / FTL_BLOCK_SIZE - io->iov_off;
157 }
158 
159 static void
160 _ftl_io_init_iovec(struct ftl_io *io, const struct iovec *iov, size_t iov_cnt, size_t lbk_cnt)
161 {
162 	size_t iov_off;
163 
164 	io->iov_pos = 0;
165 	io->iov_cnt = iov_cnt;
166 	io->lbk_cnt = lbk_cnt;
167 
168 	memcpy(io->iov, iov, iov_cnt * sizeof(*iov));
169 
170 	if (lbk_cnt == 0) {
171 		for (iov_off = 0; iov_off < iov_cnt; ++iov_off) {
172 			io->lbk_cnt += iov[iov_off].iov_len / FTL_BLOCK_SIZE;
173 		}
174 	}
175 }
176 
177 static void _ftl_io_free(struct ftl_io *io);
178 
179 static int
180 ftl_io_add_child(struct ftl_io *io, const struct iovec *iov, size_t iov_cnt)
181 {
182 	struct ftl_io *child;
183 
184 	child = ftl_io_alloc_child(io);
185 	if (spdk_unlikely(!child)) {
186 		return -ENOMEM;
187 	}
188 
189 	_ftl_io_init_iovec(child, iov, iov_cnt, 0);
190 
191 	if (io->flags & FTL_IO_VECTOR_LBA) {
192 		child->lba.vector = io->lba.vector + io->lbk_cnt;
193 	} else {
194 		child->lba.single = io->lba.single + io->lbk_cnt;
195 	}
196 
197 	io->lbk_cnt += child->lbk_cnt;
198 	return 0;
199 }
200 
201 static int
202 ftl_io_init_iovec(struct ftl_io *io, const struct iovec *iov, size_t iov_cnt, size_t lbk_cnt)
203 {
204 	struct ftl_io *child;
205 	size_t iov_off = 0, iov_left;
206 	int rc;
207 
208 	if (spdk_likely(iov_cnt <= FTL_IO_MAX_IOVEC)) {
209 		_ftl_io_init_iovec(io, iov, iov_cnt, lbk_cnt);
210 		return 0;
211 	}
212 
213 	while (iov_off < iov_cnt) {
214 		iov_left = spdk_min(iov_cnt - iov_off, FTL_IO_MAX_IOVEC);
215 
216 		rc = ftl_io_add_child(io, &iov[iov_off], iov_left);
217 		if (spdk_unlikely(rc != 0)) {
218 			while ((child = LIST_FIRST(&io->children))) {
219 				assert(LIST_EMPTY(&child->children));
220 				LIST_REMOVE(child, child_entry);
221 				_ftl_io_free(child);
222 			}
223 
224 			return -ENOMEM;
225 		}
226 
227 		iov_off += iov_left;
228 	}
229 
230 	assert(io->lbk_cnt == lbk_cnt);
231 	return 0;
232 }
233 
234 void
235 ftl_io_shrink_iovec(struct ftl_io *io, size_t lbk_cnt)
236 {
237 	size_t iov_off = 0, lbk_off = 0;
238 
239 	assert(io->lbk_cnt >= lbk_cnt);
240 	assert(io->pos == 0 && io->iov_pos == 0 && io->iov_off == 0);
241 
242 	for (; iov_off < io->iov_cnt; ++iov_off) {
243 		size_t num_iov = io->iov[iov_off].iov_len / FTL_BLOCK_SIZE;
244 		size_t num_left = lbk_cnt - lbk_off;
245 
246 		if (num_iov >= num_left) {
247 			io->iov[iov_off].iov_len = num_left * FTL_BLOCK_SIZE;
248 			io->iov_cnt = iov_off + 1;
249 			io->lbk_cnt = lbk_cnt;
250 			break;
251 		}
252 
253 		lbk_off += num_iov;
254 	}
255 }
256 
257 static void
258 ftl_io_init(struct ftl_io *io, struct spdk_ftl_dev *dev,
259 	    ftl_io_fn fn, void *ctx, int flags, int type)
260 {
261 	io->flags |= flags | FTL_IO_INITIALIZED;
262 	io->type = type;
263 	io->dev = dev;
264 	io->lba.single = FTL_LBA_INVALID;
265 	io->ppa.ppa = FTL_PPA_INVALID;
266 	io->cb_fn = fn;
267 	io->cb_ctx = ctx;
268 	io->trace = ftl_trace_alloc_id(dev);
269 }
270 
271 struct ftl_io *
272 ftl_io_init_internal(const struct ftl_io_init_opts *opts)
273 {
274 	struct ftl_io *io = opts->io;
275 	struct spdk_ftl_dev *dev = opts->dev;
276 	struct iovec iov = {
277 		.iov_base = opts->data,
278 		.iov_len  = opts->lbk_cnt * FTL_BLOCK_SIZE
279 	};
280 
281 	if (!io) {
282 		if (opts->parent) {
283 			io = ftl_io_alloc_child(opts->parent);
284 		} else {
285 			io = ftl_io_alloc(dev->ioch);
286 		}
287 
288 		if (!io) {
289 			return NULL;
290 		}
291 	}
292 
293 	ftl_io_clear(io);
294 	ftl_io_init(io, dev, opts->cb_fn, opts->cb_ctx, opts->flags | FTL_IO_INTERNAL, opts->type);
295 
296 	io->rwb_batch = opts->rwb_batch;
297 	io->band = opts->band;
298 	io->md = opts->md;
299 
300 	if (ftl_io_init_iovec(io, &iov, 1, opts->lbk_cnt)) {
301 		if (!opts->io) {
302 			ftl_io_free(io);
303 		}
304 		return NULL;
305 	}
306 
307 	if (opts->flags & FTL_IO_VECTOR_LBA) {
308 		io->lba.vector = calloc(io->lbk_cnt, sizeof(uint64_t));
309 		if (!io->lba.vector) {
310 			ftl_io_free(io);
311 			return NULL;
312 		}
313 	}
314 
315 	return io;
316 }
317 
318 struct ftl_io *
319 ftl_io_rwb_init(struct spdk_ftl_dev *dev, struct ftl_band *band,
320 		struct ftl_rwb_batch *batch, ftl_io_fn cb)
321 {
322 	struct ftl_io_init_opts opts = {
323 		.dev		= dev,
324 		.io		= NULL,
325 		.rwb_batch	= batch,
326 		.band		= band,
327 		.size		= sizeof(struct ftl_io),
328 		.flags		= 0,
329 		.type		= FTL_IO_WRITE,
330 		.lbk_cnt	= dev->xfer_size,
331 		.cb_fn		= cb,
332 		.data		= ftl_rwb_batch_get_data(batch),
333 		.md		= ftl_rwb_batch_get_md(batch),
334 	};
335 
336 	return ftl_io_init_internal(&opts);
337 }
338 
339 struct ftl_io *
340 ftl_io_erase_init(struct ftl_band *band, size_t lbk_cnt, ftl_io_fn cb)
341 {
342 	struct ftl_io *io;
343 	struct ftl_io_init_opts opts = {
344 		.dev		= band->dev,
345 		.io		= NULL,
346 		.rwb_batch	= NULL,
347 		.band		= band,
348 		.size		= sizeof(struct ftl_io),
349 		.flags		= FTL_IO_PPA_MODE,
350 		.type		= FTL_IO_ERASE,
351 		.lbk_cnt	= 1,
352 		.cb_fn		= cb,
353 		.data		= NULL,
354 		.md		= NULL,
355 	};
356 
357 	io = ftl_io_init_internal(&opts);
358 	if (!io) {
359 		return NULL;
360 	}
361 
362 	io->lbk_cnt = lbk_cnt;
363 
364 	return io;
365 }
366 
367 static void
368 _ftl_user_cb(struct ftl_io *io, void *arg, int status)
369 {
370 	io->user_fn(arg, status);
371 }
372 
373 struct ftl_io *
374 ftl_io_user_init(struct spdk_io_channel *_ioch, uint64_t lba, size_t lbk_cnt, struct iovec *iov,
375 		 size_t iov_cnt, spdk_ftl_fn cb_fn, void *cb_ctx, int type)
376 {
377 	struct ftl_io_channel *ioch = spdk_io_channel_get_ctx(_ioch);
378 	struct spdk_ftl_dev *dev = ioch->dev;
379 	struct ftl_io *io;
380 
381 	io = ftl_io_alloc(_ioch);
382 	if (spdk_unlikely(!io)) {
383 		return NULL;
384 	}
385 
386 	ftl_io_init(io, dev, _ftl_user_cb, cb_ctx, 0, type);
387 	io->lba.single = lba;
388 	io->user_fn = cb_fn;
389 
390 	if (ftl_io_init_iovec(io, iov, iov_cnt, lbk_cnt)) {
391 		ftl_io_free(io);
392 		return NULL;
393 	}
394 
395 	ftl_trace_lba_io_init(io->dev, io);
396 	return io;
397 }
398 
399 static void
400 _ftl_io_free(struct ftl_io *io)
401 {
402 	struct ftl_io_channel *ioch;
403 
404 	assert(LIST_EMPTY(&io->children));
405 
406 	if (io->flags & FTL_IO_VECTOR_LBA) {
407 		free(io->lba.vector);
408 	}
409 
410 	if (pthread_spin_destroy(&io->lock)) {
411 		SPDK_ERRLOG("pthread_spin_destroy failed\n");
412 	}
413 
414 	ioch = spdk_io_channel_get_ctx(io->ioch);
415 	spdk_mempool_put(ioch->io_pool, io);
416 }
417 
418 static bool
419 ftl_io_remove_child(struct ftl_io *io)
420 {
421 	struct ftl_io *parent = io->parent;
422 	bool parent_done;
423 
424 	pthread_spin_lock(&parent->lock);
425 	LIST_REMOVE(io, child_entry);
426 	parent_done = parent->done && LIST_EMPTY(&parent->children);
427 	parent->status = parent->status ? : io->status;
428 	pthread_spin_unlock(&parent->lock);
429 
430 	return parent_done;
431 }
432 
433 void
434 ftl_io_complete(struct ftl_io *io)
435 {
436 	struct ftl_io *parent = io->parent;
437 	bool complete;
438 
439 	io->flags &= ~FTL_IO_INITIALIZED;
440 
441 	pthread_spin_lock(&io->lock);
442 	complete = LIST_EMPTY(&io->children);
443 	io->done = true;
444 	pthread_spin_unlock(&io->lock);
445 
446 	if (complete) {
447 		if (io->cb_fn) {
448 			io->cb_fn(io, io->cb_ctx, io->status);
449 		}
450 
451 		if (parent && ftl_io_remove_child(io)) {
452 			ftl_io_complete(parent);
453 		}
454 
455 		_ftl_io_free(io);
456 	}
457 }
458 
459 struct ftl_io *
460 ftl_io_alloc_child(struct ftl_io *parent)
461 {
462 	struct ftl_io *io;
463 
464 	io = ftl_io_alloc(parent->ioch);
465 	if (spdk_unlikely(!io)) {
466 		return NULL;
467 	}
468 
469 	ftl_io_init(io, parent->dev, NULL, NULL, parent->flags, parent->type);
470 	io->parent = parent;
471 
472 	pthread_spin_lock(&parent->lock);
473 	LIST_INSERT_HEAD(&parent->children, io, child_entry);
474 	pthread_spin_unlock(&parent->lock);
475 
476 	return io;
477 }
478 
479 void
480 ftl_io_process_error(struct ftl_io *io, const struct spdk_nvme_cpl *status)
481 {
482 	/* TODO: add error handling for specifc cases */
483 	if (status->status.sct == SPDK_NVME_SCT_MEDIA_ERROR &&
484 	    status->status.sc == SPDK_OCSSD_SC_READ_HIGH_ECC) {
485 		return;
486 	}
487 
488 	io->status = -EIO;
489 }
490 
491 void ftl_io_fail(struct ftl_io *io, int status)
492 {
493 	io->status = status;
494 	ftl_io_advance(io, io->lbk_cnt - io->pos);
495 }
496 
497 void *
498 ftl_io_get_md(const struct ftl_io *io)
499 {
500 	if (!io->md) {
501 		return NULL;
502 	}
503 
504 	return (char *)io->md + io->pos * io->dev->md_size;
505 }
506 
507 struct ftl_io *
508 ftl_io_alloc(struct spdk_io_channel *ch)
509 {
510 	struct ftl_io *io;
511 	struct ftl_io_channel *ioch = spdk_io_channel_get_ctx(ch);
512 
513 	io = spdk_mempool_get(ioch->io_pool);
514 	if (!io) {
515 		return NULL;
516 	}
517 
518 	memset(io, 0, ioch->elem_size);
519 	io->ioch = ch;
520 
521 	if (pthread_spin_init(&io->lock, PTHREAD_PROCESS_PRIVATE)) {
522 		SPDK_ERRLOG("pthread_spin_init failed\n");
523 		spdk_mempool_put(ioch->io_pool, io);
524 		return NULL;
525 	}
526 
527 	return io;
528 }
529 
530 void
531 ftl_io_reinit(struct ftl_io *io, ftl_io_fn cb, void *ctx, int flags, int type)
532 {
533 	ftl_io_clear(io);
534 	ftl_io_init(io, io->dev, cb, ctx, flags, type);
535 }
536 
537 void
538 ftl_io_clear(struct ftl_io *io)
539 {
540 	ftl_io_reset(io);
541 
542 	io->flags = 0;
543 	io->rwb_batch = NULL;
544 	io->band = NULL;
545 }
546 
547 void
548 ftl_io_reset(struct ftl_io *io)
549 {
550 	io->req_cnt = io->pos = io->iov_pos = io->iov_off = 0;
551 	io->done = false;
552 }
553 
554 void
555 ftl_io_free(struct ftl_io *io)
556 {
557 	struct ftl_io *parent = io->parent;
558 
559 	if (!io) {
560 		return;
561 	}
562 
563 	if (parent && ftl_io_remove_child(io)) {
564 		ftl_io_complete(parent);
565 	}
566 
567 	_ftl_io_free(io);
568 }
569 
570 void
571 ftl_io_call_foreach_child(struct ftl_io *io, int (*callback)(struct ftl_io *))
572 {
573 	struct ftl_io *child, *tmp;
574 
575 	assert(!io->done);
576 
577 	/*
578 	 * If the IO doesn't have any children, it means that it directly describes a request (i.e.
579 	 * all of the buffers, LBAs, etc. are filled). Otherwise the IO only groups together several
580 	 * requests and may be partially filled, so the callback needs to be called on all of its
581 	 * children instead.
582 	 */
583 	if (LIST_EMPTY(&io->children)) {
584 		callback(io);
585 		return;
586 	}
587 
588 	LIST_FOREACH_SAFE(child, &io->children, child_entry, tmp) {
589 		int rc = callback(child);
590 		if (rc) {
591 			assert(rc != -EAGAIN);
592 			ftl_io_fail(io, rc);
593 			break;
594 		}
595 	}
596 
597 	/*
598 	 * If all the callbacks were processed or an error occurred, treat this IO as completed.
599 	 * Multiple calls to ftl_io_call_foreach_child are not supported, resubmissions are supposed
600 	 * to be handled in the callback.
601 	 */
602 	ftl_io_complete(io);
603 }
604