xref: /spdk/lib/ftl/ftl_io.c (revision ae7b5890ef728af40bd233a5011b924c482603bf) 
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright (c) Intel Corporation.
5  *   All rights reserved.
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of Intel Corporation nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33 
34 #include "spdk/stdinc.h"
35 #include "spdk/ftl.h"
36 #include "spdk/likely.h"
37 #include "spdk/util.h"
38 
39 #include "ftl_io.h"
40 #include "ftl_core.h"
41 #include "ftl_rwb.h"
42 #include "ftl_band.h"
43 #include "ftl_debug.h"
44 
45 void
46 ftl_io_inc_req(struct ftl_io *io)
47 {
48 	struct ftl_band *band = io->band;
49 
50 	if (!(io->flags & FTL_IO_CACHE) && io->type != FTL_IO_READ && io->type != FTL_IO_ERASE) {
51 		ftl_band_acquire_lba_map(band);
52 	}
53 
54 	__atomic_fetch_add(&io->dev->num_inflight, 1, __ATOMIC_SEQ_CST);
55 
56 	++io->req_cnt;
57 }
58 
59 void
60 ftl_io_dec_req(struct ftl_io *io)
61 {
62 	struct ftl_band *band = io->band;
63 	unsigned long num_inflight __attribute__((unused));
64 
65 	if (!(io->flags & FTL_IO_CACHE) && io->type != FTL_IO_READ && io->type != FTL_IO_ERASE) {
66 		ftl_band_release_lba_map(band);
67 	}
68 
69 	num_inflight = __atomic_fetch_sub(&io->dev->num_inflight, 1, __ATOMIC_SEQ_CST);
70 
71 	assert(num_inflight > 0);
72 	assert(io->req_cnt > 0);
73 
74 	--io->req_cnt;
75 }
76 
77 struct iovec *
78 ftl_io_iovec(struct ftl_io *io)
79 {
80 	return &io->iov[0];
81 }
82 
83 uint64_t
84 ftl_io_get_lba(const struct ftl_io *io, size_t offset)
85 {
86 	assert(offset < io->lbk_cnt);
87 
88 	if (io->flags & FTL_IO_VECTOR_LBA) {
89 		return io->lba.vector[offset];
90 	} else {
91 		return io->lba.single + offset;
92 	}
93 }
94 
95 uint64_t
96 ftl_io_current_lba(const struct ftl_io *io)
97 {
98 	return ftl_io_get_lba(io, io->pos);
99 }
100 
101 void
102 ftl_io_advance(struct ftl_io *io, size_t lbk_cnt)
103 {
104 	struct iovec *iov = ftl_io_iovec(io);
105 	size_t iov_lbks, lbk_left = lbk_cnt;
106 
107 	io->pos += lbk_cnt;
108 
109 	if (io->iov_cnt != 0) {
110 		while (lbk_left > 0) {
111 			assert(io->iov_pos < io->iov_cnt);
112 			iov_lbks = iov[io->iov_pos].iov_len / FTL_BLOCK_SIZE;
113 
114 			if (io->iov_off + lbk_left < iov_lbks) {
115 				io->iov_off += lbk_left;
116 				break;
117 			}
118 
119 			assert(iov_lbks > io->iov_off);
120 			lbk_left -= (iov_lbks - io->iov_off);
121 			io->iov_off = 0;
122 			io->iov_pos++;
123 		}
124 	}
125 
126 	if (io->parent) {
127 		ftl_io_advance(io->parent, lbk_cnt);
128 	}
129 }
130 
131 size_t
132 ftl_iovec_num_lbks(struct iovec *iov, size_t iov_cnt)
133 {
134 	size_t lbks = 0, i = 0;
135 
136 	for (; i < iov_cnt; ++i) {
137 		lbks += iov[i].iov_len / FTL_BLOCK_SIZE;
138 	}
139 
140 	return lbks;
141 }
142 
143 void *
144 ftl_io_iovec_addr(struct ftl_io *io)
145 {
146 	assert(io->iov_pos < io->iov_cnt);
147 	assert(io->iov_off * FTL_BLOCK_SIZE < ftl_io_iovec(io)[io->iov_pos].iov_len);
148 
149 	return (char *)ftl_io_iovec(io)[io->iov_pos].iov_base +
150 	       io->iov_off * FTL_BLOCK_SIZE;
151 }
152 
153 size_t
154 ftl_io_iovec_len_left(struct ftl_io *io)
155 {
156 	struct iovec *iov = ftl_io_iovec(io);
157 	return iov[io->iov_pos].iov_len / FTL_BLOCK_SIZE - io->iov_off;
158 }
159 
160 static void
161 _ftl_io_init_iovec(struct ftl_io *io, const struct iovec *iov, size_t iov_cnt, size_t lbk_cnt)
162 {
163 	size_t iov_off;
164 
165 	io->iov_pos = 0;
166 	io->iov_cnt = iov_cnt;
167 	io->lbk_cnt = lbk_cnt;
168 
169 	memcpy(io->iov, iov, iov_cnt * sizeof(*iov));
170 
171 	if (lbk_cnt == 0) {
172 		for (iov_off = 0; iov_off < iov_cnt; ++iov_off) {
173 			io->lbk_cnt += iov[iov_off].iov_len / FTL_BLOCK_SIZE;
174 		}
175 	}
176 }
177 
178 static void _ftl_io_free(struct ftl_io *io);
179 
180 static int
181 ftl_io_add_child(struct ftl_io *io, const struct iovec *iov, size_t iov_cnt)
182 {
183 	struct ftl_io *child;
184 
185 	child = ftl_io_alloc_child(io);
186 	if (spdk_unlikely(!child)) {
187 		return -ENOMEM;
188 	}
189 
190 	_ftl_io_init_iovec(child, iov, iov_cnt, 0);
191 
192 	if (io->flags & FTL_IO_VECTOR_LBA) {
193 		child->lba.vector = io->lba.vector + io->lbk_cnt;
194 	} else {
195 		child->lba.single = io->lba.single + io->lbk_cnt;
196 	}
197 
198 	io->lbk_cnt += child->lbk_cnt;
199 	return 0;
200 }
201 
202 static int
203 ftl_io_init_iovec(struct ftl_io *io, const struct iovec *iov, size_t iov_cnt, size_t lbk_cnt)
204 {
205 	struct ftl_io *child;
206 	size_t iov_off = 0, iov_left;
207 	int rc;
208 
209 	if (spdk_likely(iov_cnt <= FTL_IO_MAX_IOVEC)) {
210 		_ftl_io_init_iovec(io, iov, iov_cnt, lbk_cnt);
211 		return 0;
212 	}
213 
214 	while (iov_off < iov_cnt) {
215 		iov_left = spdk_min(iov_cnt - iov_off, FTL_IO_MAX_IOVEC);
216 
217 		rc = ftl_io_add_child(io, &iov[iov_off], iov_left);
218 		if (spdk_unlikely(rc != 0)) {
219 			while ((child = LIST_FIRST(&io->children))) {
220 				assert(LIST_EMPTY(&child->children));
221 				LIST_REMOVE(child, child_entry);
222 				_ftl_io_free(child);
223 			}
224 
225 			return -ENOMEM;
226 		}
227 
228 		iov_off += iov_left;
229 	}
230 
231 	assert(io->lbk_cnt == lbk_cnt);
232 	return 0;
233 }
234 
235 void
236 ftl_io_shrink_iovec(struct ftl_io *io, size_t lbk_cnt)
237 {
238 	size_t iov_off = 0, lbk_off = 0;
239 
240 	assert(io->lbk_cnt >= lbk_cnt);
241 	assert(io->pos == 0 && io->iov_pos == 0 && io->iov_off == 0);
242 
243 	for (; iov_off < io->iov_cnt; ++iov_off) {
244 		size_t num_iov = io->iov[iov_off].iov_len / FTL_BLOCK_SIZE;
245 		size_t num_left = lbk_cnt - lbk_off;
246 
247 		if (num_iov >= num_left) {
248 			io->iov[iov_off].iov_len = num_left * FTL_BLOCK_SIZE;
249 			io->iov_cnt = iov_off + 1;
250 			io->lbk_cnt = lbk_cnt;
251 			break;
252 		}
253 
254 		lbk_off += num_iov;
255 	}
256 }
257 
258 static void
259 ftl_io_init(struct ftl_io *io, struct spdk_ftl_dev *dev,
260 	    ftl_io_fn fn, void *ctx, int flags, int type)
261 {
262 	io->flags |= flags | FTL_IO_INITIALIZED;
263 	io->type = type;
264 	io->dev = dev;
265 	io->lba.single = FTL_LBA_INVALID;
266 	io->ppa.ppa = FTL_PPA_INVALID;
267 	io->cb_fn = fn;
268 	io->cb_ctx = ctx;
269 	io->trace = ftl_trace_alloc_id(dev);
270 }
271 
272 struct ftl_io *
273 ftl_io_init_internal(const struct ftl_io_init_opts *opts)
274 {
275 	struct ftl_io *io = opts->io;
276 	struct ftl_io *parent = opts->parent;
277 	struct spdk_ftl_dev *dev = opts->dev;
278 	struct iovec iov = {
279 		.iov_base = opts->data,
280 		.iov_len  = opts->lbk_cnt * FTL_BLOCK_SIZE
281 	};
282 
283 	if (!io) {
284 		if (parent) {
285 			io = ftl_io_alloc_child(parent);
286 		} else {
287 			io = ftl_io_alloc(dev->ioch);
288 		}
289 
290 		if (!io) {
291 			return NULL;
292 		}
293 	}
294 
295 	ftl_io_clear(io);
296 	ftl_io_init(io, dev, opts->cb_fn, opts->cb_ctx, opts->flags | FTL_IO_INTERNAL, opts->type);
297 
298 	io->rwb_batch = opts->rwb_batch;
299 	io->band = opts->band;
300 	io->md = opts->md;
301 
302 	if (parent) {
303 		if (parent->flags & FTL_IO_VECTOR_LBA) {
304 			io->lba.vector = parent->lba.vector + parent->pos;
305 		} else {
306 			io->lba.single = parent->lba.single + parent->pos;
307 		}
308 	}
309 
310 	if (ftl_io_init_iovec(io, &iov, 1, opts->lbk_cnt)) {
311 		if (!opts->io) {
312 			ftl_io_free(io);
313 		}
314 		return NULL;
315 	}
316 
317 	if (opts->flags & FTL_IO_VECTOR_LBA) {
318 		io->lba.vector = calloc(io->lbk_cnt, sizeof(uint64_t));
319 		if (!io->lba.vector) {
320 			ftl_io_free(io);
321 			return NULL;
322 		}
323 	}
324 
325 	return io;
326 }
327 
328 struct ftl_io *
329 ftl_io_rwb_init(struct spdk_ftl_dev *dev, struct ftl_band *band,
330 		struct ftl_rwb_batch *batch, ftl_io_fn cb)
331 {
332 	struct ftl_io_init_opts opts = {
333 		.dev		= dev,
334 		.io		= NULL,
335 		.rwb_batch	= batch,
336 		.band		= band,
337 		.size		= sizeof(struct ftl_io),
338 		.flags		= 0,
339 		.type		= FTL_IO_WRITE,
340 		.lbk_cnt	= dev->xfer_size,
341 		.cb_fn		= cb,
342 		.data		= ftl_rwb_batch_get_data(batch),
343 		.md		= ftl_rwb_batch_get_md(batch),
344 	};
345 
346 	return ftl_io_init_internal(&opts);
347 }
348 
349 struct ftl_io *
350 ftl_io_erase_init(struct ftl_band *band, size_t lbk_cnt, ftl_io_fn cb)
351 {
352 	struct ftl_io *io;
353 	struct ftl_io_init_opts opts = {
354 		.dev		= band->dev,
355 		.io		= NULL,
356 		.rwb_batch	= NULL,
357 		.band		= band,
358 		.size		= sizeof(struct ftl_io),
359 		.flags		= FTL_IO_PPA_MODE,
360 		.type		= FTL_IO_ERASE,
361 		.lbk_cnt	= 1,
362 		.cb_fn		= cb,
363 		.data		= NULL,
364 		.md		= NULL,
365 	};
366 
367 	io = ftl_io_init_internal(&opts);
368 	if (!io) {
369 		return NULL;
370 	}
371 
372 	io->lbk_cnt = lbk_cnt;
373 
374 	return io;
375 }
376 
377 static void
378 _ftl_user_cb(struct ftl_io *io, void *arg, int status)
379 {
380 	io->user_fn(arg, status);
381 }
382 
383 struct ftl_io *
384 ftl_io_user_init(struct spdk_io_channel *_ioch, uint64_t lba, size_t lbk_cnt, struct iovec *iov,
385 		 size_t iov_cnt, spdk_ftl_fn cb_fn, void *cb_ctx, int type)
386 {
387 	struct ftl_io_channel *ioch = spdk_io_channel_get_ctx(_ioch);
388 	struct spdk_ftl_dev *dev = ioch->dev;
389 	struct ftl_io *io;
390 
391 	io = ftl_io_alloc(_ioch);
392 	if (spdk_unlikely(!io)) {
393 		return NULL;
394 	}
395 
396 	ftl_io_init(io, dev, _ftl_user_cb, cb_ctx, 0, type);
397 	io->lba.single = lba;
398 	io->user_fn = cb_fn;
399 
400 	if (ftl_io_init_iovec(io, iov, iov_cnt, lbk_cnt)) {
401 		ftl_io_free(io);
402 		return NULL;
403 	}
404 
405 	ftl_trace_lba_io_init(io->dev, io);
406 	return io;
407 }
408 
409 static void
410 _ftl_io_free(struct ftl_io *io)
411 {
412 	struct ftl_io_channel *ioch;
413 
414 	assert(LIST_EMPTY(&io->children));
415 
416 	if (io->flags & FTL_IO_VECTOR_LBA) {
417 		free(io->lba.vector);
418 	}
419 
420 	if (pthread_spin_destroy(&io->lock)) {
421 		SPDK_ERRLOG("pthread_spin_destroy failed\n");
422 	}
423 
424 	ioch = spdk_io_channel_get_ctx(io->ioch);
425 	spdk_mempool_put(ioch->io_pool, io);
426 }
427 
428 static bool
429 ftl_io_remove_child(struct ftl_io *io)
430 {
431 	struct ftl_io *parent = io->parent;
432 	bool parent_done;
433 
434 	pthread_spin_lock(&parent->lock);
435 	LIST_REMOVE(io, child_entry);
436 	parent_done = parent->done && LIST_EMPTY(&parent->children);
437 	parent->status = parent->status ? : io->status;
438 	pthread_spin_unlock(&parent->lock);
439 
440 	return parent_done;
441 }
442 
443 void
444 ftl_io_complete(struct ftl_io *io)
445 {
446 	struct ftl_io *parent = io->parent;
447 	bool complete;
448 
449 	io->flags &= ~FTL_IO_INITIALIZED;
450 
451 	pthread_spin_lock(&io->lock);
452 	complete = LIST_EMPTY(&io->children);
453 	io->done = true;
454 	pthread_spin_unlock(&io->lock);
455 
456 	if (complete) {
457 		if (io->cb_fn) {
458 			io->cb_fn(io, io->cb_ctx, io->status);
459 		}
460 
461 		if (parent && ftl_io_remove_child(io)) {
462 			ftl_io_complete(parent);
463 		}
464 
465 		_ftl_io_free(io);
466 	}
467 }
468 
469 struct ftl_io *
470 ftl_io_alloc_child(struct ftl_io *parent)
471 {
472 	struct ftl_io *io;
473 
474 	io = ftl_io_alloc(parent->ioch);
475 	if (spdk_unlikely(!io)) {
476 		return NULL;
477 	}
478 
479 	ftl_io_init(io, parent->dev, NULL, NULL, parent->flags, parent->type);
480 	io->parent = parent;
481 
482 	pthread_spin_lock(&parent->lock);
483 	LIST_INSERT_HEAD(&parent->children, io, child_entry);
484 	pthread_spin_unlock(&parent->lock);
485 
486 	return io;
487 }
488 
489 void
490 ftl_io_process_error(struct ftl_io *io, const struct spdk_nvme_cpl *status)
491 {
492 	char ppa_buf[128];
493 
494 	/* TODO: add error handling for specifc cases */
495 	if (status->status.sct == SPDK_NVME_SCT_MEDIA_ERROR &&
496 	    status->status.sc == SPDK_OCSSD_SC_READ_HIGH_ECC) {
497 		return;
498 	}
499 
500 	SPDK_ERRLOG("Status code type 0x%x, status code 0x%x for IO type %u @ppa: %s, lba 0x%lx, cnt %lu\n",
501 		    status->status.sct, status->status.sc, io->type, ftl_ppa2str(io->ppa, ppa_buf, sizeof(ppa_buf)),
502 		    ftl_io_get_lba(io, 0), io->lbk_cnt);
503 
504 	io->status = -EIO;
505 }
506 
507 void ftl_io_fail(struct ftl_io *io, int status)
508 {
509 	io->status = status;
510 	ftl_io_advance(io, io->lbk_cnt - io->pos);
511 }
512 
513 void *
514 ftl_io_get_md(const struct ftl_io *io)
515 {
516 	if (!io->md) {
517 		return NULL;
518 	}
519 
520 	return (char *)io->md + io->pos * io->dev->md_size;
521 }
522 
523 struct ftl_io *
524 ftl_io_alloc(struct spdk_io_channel *ch)
525 {
526 	struct ftl_io *io;
527 	struct ftl_io_channel *ioch = spdk_io_channel_get_ctx(ch);
528 
529 	io = spdk_mempool_get(ioch->io_pool);
530 	if (!io) {
531 		return NULL;
532 	}
533 
534 	memset(io, 0, ioch->elem_size);
535 	io->ioch = ch;
536 
537 	if (pthread_spin_init(&io->lock, PTHREAD_PROCESS_PRIVATE)) {
538 		SPDK_ERRLOG("pthread_spin_init failed\n");
539 		spdk_mempool_put(ioch->io_pool, io);
540 		return NULL;
541 	}
542 
543 	return io;
544 }
545 
546 void
547 ftl_io_reinit(struct ftl_io *io, ftl_io_fn cb, void *ctx, int flags, int type)
548 {
549 	ftl_io_clear(io);
550 	ftl_io_init(io, io->dev, cb, ctx, flags, type);
551 }
552 
553 void
554 ftl_io_clear(struct ftl_io *io)
555 {
556 	ftl_io_reset(io);
557 
558 	io->flags = 0;
559 	io->rwb_batch = NULL;
560 	io->band = NULL;
561 }
562 
563 void
564 ftl_io_reset(struct ftl_io *io)
565 {
566 	io->req_cnt = io->pos = io->iov_pos = io->iov_off = 0;
567 	io->done = false;
568 }
569 
570 void
571 ftl_io_free(struct ftl_io *io)
572 {
573 	struct ftl_io *parent;
574 
575 	if (!io) {
576 		return;
577 	}
578 
579 	parent = io->parent;
580 	if (parent && ftl_io_remove_child(io)) {
581 		ftl_io_complete(parent);
582 	}
583 
584 	_ftl_io_free(io);
585 }
586 
587 void
588 ftl_io_call_foreach_child(struct ftl_io *io, int (*callback)(struct ftl_io *))
589 {
590 	struct ftl_io *child, *tmp;
591 
592 	assert(!io->done);
593 
594 	/*
595 	 * If the IO doesn't have any children, it means that it directly describes a request (i.e.
596 	 * all of the buffers, LBAs, etc. are filled). Otherwise the IO only groups together several
597 	 * requests and may be partially filled, so the callback needs to be called on all of its
598 	 * children instead.
599 	 */
600 	if (LIST_EMPTY(&io->children)) {
601 		callback(io);
602 		return;
603 	}
604 
605 	LIST_FOREACH_SAFE(child, &io->children, child_entry, tmp) {
606 		int rc = callback(child);
607 		if (rc) {
608 			assert(rc != -EAGAIN);
609 			ftl_io_fail(io, rc);
610 			break;
611 		}
612 	}
613 
614 	/*
615 	 * If all the callbacks were processed or an error occurred, treat this IO as completed.
616 	 * Multiple calls to ftl_io_call_foreach_child are not supported, resubmissions are supposed
617 	 * to be handled in the callback.
618 	 */
619 	ftl_io_complete(io);
620 }
621