xref: /spdk/lib/ftl/ftl_core.c (revision 510f4c134a21b45ff3a5add9ebc6c6cf7e49aeab) 
1 /*   SPDX-License-Identifier: BSD-3-Clause
2  *   Copyright (c) Intel Corporation.
3  *   All rights reserved.
4  */
5 
6 #include "spdk/likely.h"
7 #include "spdk/stdinc.h"
8 #include "spdk/nvme.h"
9 #include "spdk/thread.h"
10 #include "spdk/bdev_module.h"
11 #include "spdk/string.h"
12 #include "spdk/ftl.h"
13 #include "spdk/crc32.h"
14 
15 #include "ftl_core.h"
16 #include "ftl_band.h"
17 #include "ftl_io.h"
18 #include "ftl_debug.h"
19 #include "ftl_internal.h"
20 #include "mngt/ftl_mngt.h"
21 
22 
23 size_t
24 spdk_ftl_io_size(void)
25 {
26 	return sizeof(struct ftl_io);
27 }
28 
29 static void
30 ftl_io_cmpl_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg)
31 {
32 	struct ftl_io *io = cb_arg;
33 
34 	if (spdk_unlikely(!success)) {
35 		io->status = -EIO;
36 	}
37 
38 	ftl_io_dec_req(io);
39 	if (ftl_io_done(io)) {
40 		ftl_io_complete(io);
41 	}
42 
43 	spdk_bdev_free_io(bdev_io);
44 }
45 
46 static void
47 ftl_band_erase(struct ftl_band *band)
48 {
49 	assert(band->md->state == FTL_BAND_STATE_CLOSED ||
50 	       band->md->state == FTL_BAND_STATE_FREE);
51 
52 	ftl_band_set_state(band, FTL_BAND_STATE_PREP);
53 }
54 
55 static size_t
56 ftl_get_limit(const struct spdk_ftl_dev *dev, int type)
57 {
58 	assert(type < SPDK_FTL_LIMIT_MAX);
59 	return dev->conf.limits[type];
60 }
61 
62 static bool
63 ftl_shutdown_complete(struct spdk_ftl_dev *dev)
64 {
65 	uint64_t i;
66 
67 	if (dev->num_inflight) {
68 		return false;
69 	}
70 
71 	if (!ftl_nv_cache_is_halted(&dev->nv_cache)) {
72 		ftl_nv_cache_halt(&dev->nv_cache);
73 		return false;
74 	}
75 
76 	if (!ftl_writer_is_halted(&dev->writer_user)) {
77 		ftl_writer_halt(&dev->writer_user);
78 		return false;
79 	}
80 
81 	if (!ftl_reloc_is_halted(dev->reloc)) {
82 		ftl_reloc_halt(dev->reloc);
83 		return false;
84 	}
85 
86 	if (!ftl_writer_is_halted(&dev->writer_gc)) {
87 		ftl_writer_halt(&dev->writer_gc);
88 		return false;
89 	}
90 
91 	if (!ftl_nv_cache_chunks_busy(&dev->nv_cache)) {
92 		return false;
93 	}
94 
95 	for (i = 0; i < ftl_get_num_bands(dev); ++i) {
96 		if (dev->bands[i].queue_depth ||
97 		    dev->bands[i].md->state == FTL_BAND_STATE_CLOSING) {
98 			return false;
99 		}
100 	}
101 
102 	if (!ftl_l2p_is_halted(dev)) {
103 		ftl_l2p_halt(dev);
104 		return false;
105 	}
106 
107 	return true;
108 }
109 
110 void
111 ftl_apply_limits(struct spdk_ftl_dev *dev)
112 {
113 	size_t limit;
114 	int i;
115 
116 	/*  Clear existing limit */
117 	dev->limit = SPDK_FTL_LIMIT_MAX;
118 
119 	for (i = SPDK_FTL_LIMIT_CRIT; i < SPDK_FTL_LIMIT_MAX; ++i) {
120 		limit = ftl_get_limit(dev, i);
121 
122 		if (dev->num_free <= limit) {
123 			dev->limit = i;
124 			break;
125 		}
126 	}
127 }
128 
129 void
130 ftl_invalidate_addr(struct spdk_ftl_dev *dev, ftl_addr addr)
131 {
132 	struct ftl_band *band;
133 	struct ftl_p2l_map *p2l_map;
134 
135 	if (ftl_addr_in_nvc(dev, addr)) {
136 		ftl_bitmap_clear(dev->valid_map, addr);
137 		return;
138 	}
139 
140 	band = ftl_band_from_addr(dev, addr);
141 	p2l_map = &band->p2l_map;
142 
143 	/* The bit might be already cleared if two writes are scheduled to the */
144 	/* same LBA at the same time */
145 	if (ftl_bitmap_get(dev->valid_map, addr)) {
146 		assert(p2l_map->num_valid > 0);
147 		ftl_bitmap_clear(dev->valid_map, addr);
148 		p2l_map->num_valid--;
149 	}
150 
151 	/* Invalidate open/full band p2l_map entry to keep p2l and l2p
152 	 * consistency when band is going to close state */
153 	if (FTL_BAND_STATE_OPEN == band->md->state || FTL_BAND_STATE_FULL == band->md->state) {
154 		p2l_map->band_map[ftl_band_block_offset_from_addr(band, addr)] = FTL_LBA_INVALID;
155 	}
156 }
157 
158 static int
159 ftl_read_canceled(int rc)
160 {
161 	return rc == -EFAULT;
162 }
163 
164 static int
165 ftl_get_next_read_addr(struct ftl_io *io, ftl_addr *addr)
166 {
167 	struct spdk_ftl_dev *dev = io->dev;
168 	ftl_addr next_addr;
169 	size_t i;
170 	bool addr_cached = false;
171 
172 	*addr = ftl_l2p_get(dev, ftl_io_current_lba(io));
173 	io->map[io->pos] = *addr;
174 
175 	/* If the address is invalid, skip it */
176 	if (*addr == FTL_ADDR_INVALID) {
177 		return -EFAULT;
178 	}
179 
180 	addr_cached = ftl_addr_in_nvc(dev, *addr);
181 
182 	for (i = 1; i < ftl_io_iovec_len_left(io); ++i) {
183 		next_addr = ftl_l2p_get(dev, ftl_io_get_lba(io, io->pos + i));
184 
185 		if (next_addr == FTL_ADDR_INVALID) {
186 			break;
187 		}
188 
189 		/* It's not enough to check for contiguity, if user data is on the last block
190 		 * of base device and first nvc, then they're 'contiguous', but can't be handled
191 		 * with one read request.
192 		 */
193 		if (addr_cached != ftl_addr_in_nvc(dev, next_addr)) {
194 			break;
195 		}
196 
197 		if (*addr + i != next_addr) {
198 			break;
199 		}
200 
201 		io->map[io->pos + i] = next_addr;
202 	}
203 
204 	return i;
205 }
206 
207 static void ftl_submit_read(struct ftl_io *io);
208 
209 static void
210 _ftl_submit_read(void *_io)
211 {
212 	struct ftl_io *io = _io;
213 
214 	ftl_submit_read(io);
215 }
216 
217 static void
218 ftl_submit_read(struct ftl_io *io)
219 {
220 	struct spdk_ftl_dev *dev = io->dev;
221 	ftl_addr addr;
222 	int rc = 0, num_blocks;
223 
224 	while (io->pos < io->num_blocks) {
225 		num_blocks = ftl_get_next_read_addr(io, &addr);
226 		rc = num_blocks;
227 
228 		/* User LBA doesn't hold valid data (trimmed or never written to), fill with 0 and skip this block */
229 		if (ftl_read_canceled(rc)) {
230 			memset(ftl_io_iovec_addr(io), 0, FTL_BLOCK_SIZE);
231 			ftl_io_advance(io, 1);
232 			continue;
233 		}
234 
235 		assert(num_blocks > 0);
236 
237 		if (ftl_addr_in_nvc(dev, addr)) {
238 			rc = ftl_nv_cache_read(io, addr, num_blocks, ftl_io_cmpl_cb, io);
239 		} else {
240 			rc = spdk_bdev_read_blocks(dev->base_bdev_desc, dev->base_ioch,
241 						   ftl_io_iovec_addr(io),
242 						   addr, num_blocks, ftl_io_cmpl_cb, io);
243 		}
244 
245 		if (spdk_unlikely(rc)) {
246 			if (rc == -ENOMEM) {
247 				struct spdk_bdev *bdev;
248 				struct spdk_io_channel *ch;
249 
250 				if (ftl_addr_in_nvc(dev, addr)) {
251 					bdev = spdk_bdev_desc_get_bdev(dev->nv_cache.bdev_desc);
252 					ch = dev->nv_cache.cache_ioch;
253 				} else {
254 					bdev = spdk_bdev_desc_get_bdev(dev->base_bdev_desc);
255 					ch = dev->base_ioch;
256 				}
257 				io->bdev_io_wait.bdev = bdev;
258 				io->bdev_io_wait.cb_fn = _ftl_submit_read;
259 				io->bdev_io_wait.cb_arg = io;
260 				spdk_bdev_queue_io_wait(bdev, ch, &io->bdev_io_wait);
261 				return;
262 			} else {
263 				ftl_abort();
264 			}
265 		}
266 
267 		ftl_io_inc_req(io);
268 		ftl_io_advance(io, num_blocks);
269 	}
270 
271 	/* If we didn't have to read anything from the device, */
272 	/* complete the request right away */
273 	if (ftl_io_done(io)) {
274 		ftl_io_complete(io);
275 	}
276 }
277 
278 bool
279 ftl_needs_reloc(struct spdk_ftl_dev *dev)
280 {
281 	size_t limit = ftl_get_limit(dev, SPDK_FTL_LIMIT_START);
282 
283 	if (dev->num_free <= limit) {
284 		return true;
285 	}
286 
287 	return false;
288 }
289 
290 void
291 spdk_ftl_dev_get_attrs(const struct spdk_ftl_dev *dev, struct spdk_ftl_attrs *attrs)
292 {
293 	attrs->num_blocks = dev->num_lbas;
294 	attrs->block_size = FTL_BLOCK_SIZE;
295 	attrs->optimum_io_size = dev->xfer_size;
296 }
297 
298 static void
299 ftl_io_pin_cb(struct spdk_ftl_dev *dev, int status, struct ftl_l2p_pin_ctx *pin_ctx)
300 {
301 	struct ftl_io *io = pin_ctx->cb_ctx;
302 
303 	if (spdk_unlikely(status != 0)) {
304 		/* Retry on the internal L2P fault */
305 		io->status = -EAGAIN;
306 		ftl_io_complete(io);
307 		return;
308 	}
309 
310 	io->flags |= FTL_IO_PINNED;
311 	ftl_submit_read(io);
312 }
313 
314 static void
315 ftl_io_pin(struct ftl_io *io)
316 {
317 	if (spdk_unlikely(io->flags & FTL_IO_PINNED)) {
318 		/*
319 		 * The IO is in a retry path and it had been pinned already.
320 		 * Continue with further processing.
321 		 */
322 		ftl_l2p_pin_skip(io->dev, ftl_io_pin_cb, io, &io->l2p_pin_ctx);
323 	} else {
324 		/* First time when pinning the IO */
325 		ftl_l2p_pin(io->dev, io->lba, io->num_blocks,
326 			    ftl_io_pin_cb, io, &io->l2p_pin_ctx);
327 	}
328 }
329 
330 static void
331 start_io(struct ftl_io *io)
332 {
333 	struct ftl_io_channel *ioch = ftl_io_channel_get_ctx(io->ioch);
334 	struct spdk_ftl_dev *dev = io->dev;
335 
336 	io->map = ftl_mempool_get(ioch->map_pool);
337 	if (spdk_unlikely(!io->map)) {
338 		io->status = -ENOMEM;
339 		ftl_io_complete(io);
340 		return;
341 	}
342 
343 	switch (io->type) {
344 	case FTL_IO_READ:
345 		TAILQ_INSERT_TAIL(&dev->rd_sq, io, queue_entry);
346 		break;
347 	case FTL_IO_WRITE:
348 		TAILQ_INSERT_TAIL(&dev->wr_sq, io, queue_entry);
349 		break;
350 	case FTL_IO_UNMAP:
351 	default:
352 		io->status = -EOPNOTSUPP;
353 		ftl_io_complete(io);
354 	}
355 }
356 
357 static int
358 queue_io(struct spdk_ftl_dev *dev, struct ftl_io *io)
359 {
360 	size_t result;
361 	struct ftl_io_channel *ioch = ftl_io_channel_get_ctx(io->ioch);
362 
363 	result = spdk_ring_enqueue(ioch->sq, (void **)&io, 1, NULL);
364 	if (spdk_unlikely(0 == result)) {
365 		return -EAGAIN;
366 	}
367 
368 	return 0;
369 }
370 
371 int
372 spdk_ftl_writev(struct spdk_ftl_dev *dev, struct ftl_io *io, struct spdk_io_channel *ch,
373 		uint64_t lba, uint64_t lba_cnt, struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn,
374 		void *cb_arg)
375 {
376 	int rc;
377 
378 	if (iov_cnt == 0) {
379 		return -EINVAL;
380 	}
381 
382 	if (lba_cnt == 0) {
383 		return -EINVAL;
384 	}
385 
386 	if (lba_cnt != ftl_iovec_num_blocks(iov, iov_cnt)) {
387 		FTL_ERRLOG(dev, "Invalid IO vector to handle, device %s, LBA %"PRIu64"\n",
388 			   dev->conf.name, lba);
389 		return -EINVAL;
390 	}
391 
392 	if (!dev->initialized) {
393 		return -EBUSY;
394 	}
395 
396 	rc = ftl_io_init(ch, io, lba, lba_cnt, iov, iov_cnt, cb_fn, cb_arg, FTL_IO_WRITE);
397 	if (rc) {
398 		return rc;
399 	}
400 
401 	return queue_io(dev, io);
402 }
403 
404 int
405 spdk_ftl_readv(struct spdk_ftl_dev *dev, struct ftl_io *io, struct spdk_io_channel *ch,
406 	       uint64_t lba, uint64_t lba_cnt, struct iovec *iov, size_t iov_cnt, spdk_ftl_fn cb_fn, void *cb_arg)
407 {
408 	int rc;
409 
410 	if (iov_cnt == 0) {
411 		return -EINVAL;
412 	}
413 
414 	if (lba_cnt == 0) {
415 		return -EINVAL;
416 	}
417 
418 	if (lba_cnt != ftl_iovec_num_blocks(iov, iov_cnt)) {
419 		FTL_ERRLOG(dev, "Invalid IO vector to handle, device %s, LBA %"PRIu64"\n",
420 			   dev->conf.name, lba);
421 		return -EINVAL;
422 	}
423 
424 	if (!dev->initialized) {
425 		return -EBUSY;
426 	}
427 
428 	rc = ftl_io_init(ch, io, lba, lba_cnt, iov, iov_cnt, cb_fn, cb_arg, FTL_IO_READ);
429 	if (rc) {
430 		return rc;
431 	}
432 
433 	return queue_io(dev, io);
434 }
435 
436 #define FTL_IO_QUEUE_BATCH 16
437 int
438 ftl_io_channel_poll(void *arg)
439 {
440 	struct ftl_io_channel *ch = arg;
441 	void *ios[FTL_IO_QUEUE_BATCH];
442 	uint64_t i, count;
443 
444 	count = spdk_ring_dequeue(ch->cq, ios, FTL_IO_QUEUE_BATCH);
445 	if (count == 0) {
446 		return SPDK_POLLER_IDLE;
447 	}
448 
449 	for (i = 0; i < count; i++) {
450 		struct ftl_io *io = ios[i];
451 		io->user_fn(io->cb_ctx, io->status);
452 	}
453 
454 	return SPDK_POLLER_BUSY;
455 }
456 
457 static void
458 ftl_process_io_channel(struct spdk_ftl_dev *dev, struct ftl_io_channel *ioch)
459 {
460 	void *ios[FTL_IO_QUEUE_BATCH];
461 	size_t count, i;
462 
463 	count = spdk_ring_dequeue(ioch->sq, ios, FTL_IO_QUEUE_BATCH);
464 	if (count == 0) {
465 		return;
466 	}
467 
468 	for (i = 0; i < count; i++) {
469 		struct ftl_io *io = ios[i];
470 		start_io(io);
471 	}
472 }
473 
474 static void
475 ftl_process_io_queue(struct spdk_ftl_dev *dev)
476 {
477 	struct ftl_io_channel *ioch;
478 	struct ftl_io *io;
479 
480 	/* TODO: Try to figure out a mechanism to batch more requests at the same time,
481 	 * with keeping enough resources (pinned pages), between reads, writes and gc/compaction
482 	 */
483 	if (!TAILQ_EMPTY(&dev->rd_sq)) {
484 		io = TAILQ_FIRST(&dev->rd_sq);
485 		TAILQ_REMOVE(&dev->rd_sq, io, queue_entry);
486 		assert(io->type == FTL_IO_READ);
487 		ftl_io_pin(io);
488 	}
489 
490 	if (!ftl_nv_cache_full(&dev->nv_cache) && !TAILQ_EMPTY(&dev->wr_sq)) {
491 		io = TAILQ_FIRST(&dev->wr_sq);
492 		TAILQ_REMOVE(&dev->wr_sq, io, queue_entry);
493 		assert(io->type == FTL_IO_WRITE);
494 		if (!ftl_nv_cache_write(io)) {
495 			TAILQ_INSERT_HEAD(&dev->wr_sq, io, queue_entry);
496 		}
497 	}
498 
499 	TAILQ_FOREACH(ioch, &dev->ioch_queue, entry) {
500 		ftl_process_io_channel(dev, ioch);
501 	}
502 }
503 
504 int
505 ftl_core_poller(void *ctx)
506 {
507 	struct spdk_ftl_dev *dev = ctx;
508 	uint64_t io_activity_total_old = dev->io_activity_total;
509 
510 	if (dev->halt && ftl_shutdown_complete(dev)) {
511 		spdk_poller_unregister(&dev->core_poller);
512 		return SPDK_POLLER_IDLE;
513 	}
514 
515 	ftl_process_io_queue(dev);
516 	ftl_writer_run(&dev->writer_user);
517 	ftl_writer_run(&dev->writer_gc);
518 	ftl_reloc(dev->reloc);
519 	ftl_nv_cache_process(dev);
520 	ftl_l2p_process(dev);
521 
522 	if (io_activity_total_old != dev->io_activity_total) {
523 		return SPDK_POLLER_BUSY;
524 	}
525 
526 	return SPDK_POLLER_IDLE;
527 }
528 
529 struct ftl_band *
530 ftl_band_get_next_free(struct spdk_ftl_dev *dev)
531 {
532 	struct ftl_band *band = NULL;
533 
534 	if (!TAILQ_EMPTY(&dev->free_bands)) {
535 		band = TAILQ_FIRST(&dev->free_bands);
536 		TAILQ_REMOVE(&dev->free_bands, band, queue_entry);
537 		ftl_band_erase(band);
538 	}
539 
540 	return band;
541 }
542 
543 void *g_ftl_write_buf;
544 void *g_ftl_read_buf;
545 
546 int
547 spdk_ftl_init(void)
548 {
549 	g_ftl_write_buf = spdk_zmalloc(FTL_ZERO_BUFFER_SIZE, FTL_ZERO_BUFFER_SIZE, NULL,
550 				       SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
551 	if (!g_ftl_write_buf) {
552 		return -ENOMEM;
553 	}
554 
555 	g_ftl_read_buf = spdk_zmalloc(FTL_ZERO_BUFFER_SIZE, FTL_ZERO_BUFFER_SIZE, NULL,
556 				      SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA);
557 	if (!g_ftl_read_buf) {
558 		spdk_free(g_ftl_write_buf);
559 		g_ftl_write_buf = NULL;
560 		return -ENOMEM;
561 	}
562 	return 0;
563 }
564 
565 void
566 spdk_ftl_fini(void)
567 {
568 	spdk_free(g_ftl_write_buf);
569 	spdk_free(g_ftl_read_buf);
570 }
571 
572 void
573 spdk_ftl_dev_set_fast_shutdown(struct spdk_ftl_dev *dev, bool fast_shutdown)
574 {
575 	assert(dev);
576 	dev->conf.fast_shutdown = fast_shutdown;
577 }
578 
579 struct spdk_io_channel *
580 spdk_ftl_get_io_channel(struct spdk_ftl_dev *dev)
581 {
582 	return spdk_get_io_channel(dev);
583 }
584 
585 SPDK_LOG_REGISTER_COMPONENT(ftl_core)
586