xref: /spdk/lib/ftl/ftl_p2l_log.c (revision d4d015a572e1af7b2818e44218c1e661a61545ec) 
1 /*   SPDX-License-Identifier: BSD-3-Clause
2  *   Copyright 2023 Solidigm All Rights Reserved
3  */
4 
5 #include "ftl_core.h"
6 #include "ftl_io.h"
7 #include "ftl_layout.h"
8 #include "utils/ftl_defs.h"
9 
10 struct ftl_pl2_log_item {
11 	uint64_t lba;
12 	uint64_t num_blocks;
13 	uint64_t seq_id;
14 	ftl_addr addr;
15 };
16 #define FTL_P2L_LOG_ITEMS_IN_PAGE ((FTL_BLOCK_SIZE - sizeof(union ftl_md_vss)) / sizeof(struct ftl_pl2_log_item))
17 #define FTL_P2L_LOG_PAGE_COUNT_DEFAULT 128
18 
19 struct ftl_p2l_log_page {
20 	union ftl_md_vss hdr;
21 	struct ftl_pl2_log_item items[FTL_P2L_LOG_ITEMS_IN_PAGE];
22 };
23 SPDK_STATIC_ASSERT(sizeof(struct ftl_p2l_log_page) == FTL_BLOCK_SIZE, "Invalid size of P2L page");
24 
25 struct ftl_p2l_log_page_ctrl {
26 	struct ftl_p2l_log_page page;
27 	struct ftl_p2l_log *p2l;
28 	uint64_t entry_idx;
29 	TAILQ_HEAD(, ftl_io) ios;
30 	struct ftl_md_io_entry_ctx md_ctx;
31 };
32 
33 struct ftl_p2l_log {
34 	struct spdk_ftl_dev		*dev;
35 	TAILQ_ENTRY(ftl_p2l_log)	link;
36 	TAILQ_HEAD(, ftl_io)		ios;
37 	struct ftl_md			*md;
38 	uint64_t			seq_id;
39 	struct ftl_mempool		*page_pool;
40 	uint64_t			entry_idx;
41 	uint64_t			entry_max;
42 	ftl_p2l_log_cb			cb_fn;
43 	uint32_t			ref_cnt;
44 	bool				in_use;
45 
46 	struct {
47 		spdk_ftl_fn cb_fn;
48 		void *cb_arg;
49 		ftl_p2l_log_rd_cb cb_rd;
50 		uint64_t qd;
51 		uint64_t idx;
52 		uint64_t seq_id;
53 		int result;
54 	} read_ctx;
55 };
56 
57 static void p2l_log_page_io(struct ftl_p2l_log *p2l, struct ftl_p2l_log_page_ctrl *ctrl);
58 static void ftl_p2l_log_read_process(struct ftl_p2l_log *p2l);
59 
60 static struct ftl_p2l_log *
61 p2l_log_create(struct spdk_ftl_dev *dev, uint32_t region_type)
62 {
63 	struct ftl_p2l_log *p2l;
64 
65 	p2l = calloc(1, sizeof(struct ftl_p2l_log));
66 	if (!p2l) {
67 		return NULL;
68 	}
69 
70 	TAILQ_INIT(&p2l->ios);
71 	p2l->dev = dev;
72 	p2l->md = dev->layout.md[region_type];
73 	p2l->entry_max = ftl_md_get_buffer_size(p2l->md) / FTL_BLOCK_SIZE;
74 	p2l->page_pool = ftl_mempool_create(FTL_P2L_LOG_PAGE_COUNT_DEFAULT,
75 					    sizeof(struct ftl_p2l_log_page_ctrl),
76 					    FTL_BLOCK_SIZE, SPDK_ENV_SOCKET_ID_ANY);
77 	if (!p2l->page_pool) {
78 		goto ERROR;
79 	}
80 
81 	return p2l;
82 ERROR:
83 	free(p2l);
84 	return NULL;
85 }
86 
87 static void
88 p2l_log_destroy(struct ftl_p2l_log *p2l)
89 {
90 	if (!p2l) {
91 		return;
92 	}
93 
94 	ftl_mempool_destroy(p2l->page_pool);
95 	free(p2l);
96 }
97 
98 static struct ftl_p2l_log_page_ctrl *
99 p2l_log_get_page(struct ftl_p2l_log *p2l)
100 {
101 	struct ftl_p2l_log_page_ctrl *ctrl;
102 
103 	ctrl = ftl_mempool_get(p2l->page_pool);
104 	if (!ctrl) {
105 		return NULL;
106 	}
107 
108 	/* Initialize P2L header */
109 	ctrl->page.hdr.p2l_ckpt.seq_id = p2l->seq_id;
110 	ctrl->page.hdr.p2l_ckpt.count = 0;
111 	ctrl->page.hdr.p2l_ckpt.p2l_checksum = 0;
112 	ctrl->page.hdr.p2l_ckpt.idx = ctrl->entry_idx = p2l->entry_idx;
113 
114 	/* Initialize the page control structure */
115 	ctrl->p2l = p2l;
116 	TAILQ_INIT(&ctrl->ios);
117 
118 	/* Increase P2L page index */
119 	p2l->entry_idx++;
120 
121 	/* Check if the index exceeding the buffer size */
122 	ftl_bug(p2l->entry_idx > p2l->entry_max);
123 
124 	return ctrl;
125 }
126 
127 static bool
128 l2p_log_page_is_full(struct ftl_p2l_log_page_ctrl *ctrl)
129 {
130 	return ctrl->page.hdr.p2l_ckpt.count == FTL_P2L_LOG_ITEMS_IN_PAGE;
131 }
132 
133 static void
134 p2l_log_page_free(struct ftl_p2l_log *p2l, struct ftl_p2l_log_page_ctrl *ctrl)
135 {
136 	ftl_mempool_put(p2l->page_pool, ctrl);
137 }
138 
139 static void
140 p2l_log_handle_io_error(struct ftl_p2l_log *p2l, struct ftl_p2l_log_page_ctrl *ctrl)
141 {
142 #ifdef SPDK_FTL_RETRY_ON_ERROR
143 	p2l_log_page_io(p2l, ctrl);
144 #else
145 	ftl_abort();
146 #endif
147 }
148 
149 static uint32_t
150 p2l_log_page_crc(struct ftl_p2l_log_page *page)
151 {
152 	uint32_t crc = 0;
153 	void *buffer = page;
154 	size_t size = sizeof(*page);
155 	size_t offset = offsetof(struct ftl_p2l_log_page, hdr.p2l_ckpt.p2l_checksum);
156 
157 	crc = spdk_crc32c_update(buffer, offset, crc);
158 	buffer += offset + sizeof(page->hdr.p2l_ckpt.p2l_checksum);
159 	size -= offset + sizeof(page->hdr.p2l_ckpt.p2l_checksum);
160 
161 	return spdk_crc32c_update(buffer, size, crc);
162 }
163 
164 static void
165 p2l_log_page_io_cb(int status, void *arg)
166 {
167 	struct ftl_p2l_log_page_ctrl *ctrl = arg;
168 	struct ftl_p2l_log *p2l = ctrl->p2l;
169 	struct ftl_io *io;
170 
171 	if (status) {
172 		p2l_log_handle_io_error(p2l, ctrl);
173 		return;
174 	}
175 
176 	while ((io = TAILQ_FIRST(&ctrl->ios))) {
177 		TAILQ_REMOVE(&ctrl->ios, io, queue_entry);
178 		p2l->cb_fn(io);
179 	}
180 
181 	p2l_log_page_free(p2l, ctrl);
182 }
183 
184 static void
185 p2l_log_page_io(struct ftl_p2l_log *p2l, struct ftl_p2l_log_page_ctrl *ctrl)
186 {
187 	ctrl->page.hdr.p2l_ckpt.p2l_checksum = p2l_log_page_crc(&ctrl->page);
188 
189 	ftl_md_persist_entries(p2l->md, ctrl->page.hdr.p2l_ckpt.idx, 1, &ctrl->page, NULL,
190 			       p2l_log_page_io_cb,
191 			       ctrl, &ctrl->md_ctx);
192 }
193 
194 static void
195 p2l_log_add_io(struct ftl_p2l_log *p2l, struct ftl_p2l_log_page_ctrl *ctrl, struct ftl_io *io)
196 {
197 	uint64_t i = ctrl->page.hdr.p2l_ckpt.count++;
198 
199 	assert(i < FTL_P2L_LOG_ITEMS_IN_PAGE);
200 	ctrl->page.items[i].lba = io->lba;
201 	ctrl->page.items[i].num_blocks = io->num_blocks;
202 	ctrl->page.items[i].seq_id = io->nv_cache_chunk->md->seq_id;
203 	ctrl->page.items[i].addr = io->addr;
204 
205 	/* TODO Make sure P2L map is updated respectively */
206 
207 	TAILQ_REMOVE(&p2l->ios, io, queue_entry);
208 	TAILQ_INSERT_TAIL(&ctrl->ios, io, queue_entry);
209 }
210 
211 void
212 ftl_p2l_log_io(struct ftl_p2l_log *p2l, struct ftl_io *io)
213 {
214 	TAILQ_INSERT_TAIL(&p2l->ios, io, queue_entry);
215 }
216 
217 static void
218 p2l_log_flush(struct ftl_p2l_log *p2l)
219 {
220 	struct ftl_p2l_log_page_ctrl *ctrl = NULL;
221 	struct ftl_io *io;
222 
223 	while ((io = TAILQ_FIRST(&p2l->ios))) {
224 		if (!ctrl) {
225 			ctrl = p2l_log_get_page(p2l);
226 			if (!ctrl) {
227 				/* No page at the moment, try next time */
228 				break;
229 			}
230 		}
231 
232 		p2l_log_add_io(p2l, ctrl, io);
233 
234 		if (l2p_log_page_is_full(ctrl)) {
235 			p2l_log_page_io(p2l, ctrl);
236 			ctrl = NULL;
237 		}
238 	}
239 
240 	if (ctrl) {
241 		p2l_log_page_io(p2l, ctrl);
242 	}
243 }
244 
245 void
246 ftl_p2l_log_flush(struct spdk_ftl_dev *dev)
247 {
248 	struct ftl_p2l_log *p2l;
249 
250 	TAILQ_FOREACH(p2l, &dev->p2l_ckpt.log.inuse, link) {
251 		p2l_log_flush(p2l);
252 	}
253 }
254 
255 uint64_t
256 ftl_p2l_log_get_md_blocks_required(struct spdk_ftl_dev *dev, uint64_t write_unit_blocks,
257 				   uint64_t max_user_data_blocks)
258 {
259 	return spdk_divide_round_up(max_user_data_blocks, write_unit_blocks);
260 }
261 
262 int
263 ftl_p2l_log_init(struct spdk_ftl_dev *dev)
264 {
265 	struct ftl_p2l_log *p2l;
266 	uint32_t region_type;
267 
268 	TAILQ_INIT(&dev->p2l_ckpt.log.free);
269 	TAILQ_INIT(&dev->p2l_ckpt.log.inuse);
270 
271 	for (region_type = FTL_LAYOUT_REGION_TYPE_P2L_LOG_IO_MIN;
272 	     region_type <= FTL_LAYOUT_REGION_TYPE_P2L_LOG_IO_MAX;
273 	     region_type++) {
274 		p2l = p2l_log_create(dev, region_type);
275 		if (!p2l) {
276 			return -ENOMEM;
277 		}
278 
279 		TAILQ_INSERT_TAIL(&dev->p2l_ckpt.log.free, p2l, link);
280 	}
281 
282 	return 0;
283 }
284 
285 void
286 ftl_p2l_log_deinit(struct spdk_ftl_dev *dev)
287 {
288 	struct ftl_p2l_log *p2l, *p2l_next;
289 
290 	TAILQ_FOREACH_SAFE(p2l, &dev->p2l_ckpt.log.free, link, p2l_next) {
291 		TAILQ_REMOVE(&dev->p2l_ckpt.log.free, p2l, link);
292 		p2l_log_destroy(p2l);
293 	}
294 
295 	TAILQ_FOREACH_SAFE(p2l, &dev->p2l_ckpt.log.inuse, link, p2l_next) {
296 		TAILQ_REMOVE(&dev->p2l_ckpt.log.inuse, p2l, link);
297 		p2l_log_destroy(p2l);
298 	}
299 }
300 
301 enum ftl_layout_region_type
302 ftl_p2l_log_type(struct ftl_p2l_log *p2l) {
303 	return p2l->md->region->type;
304 }
305 
306 struct ftl_p2l_log *
307 ftl_p2l_log_acquire(struct spdk_ftl_dev *dev, uint64_t seq_id, ftl_p2l_log_cb cb)
308 {
309 	struct ftl_p2l_log *p2l;
310 
311 	p2l = TAILQ_FIRST(&dev->p2l_ckpt.log.free);
312 	assert(p2l);
313 	TAILQ_REMOVE(&dev->p2l_ckpt.log.free, p2l, link);
314 	TAILQ_INSERT_TAIL(&dev->p2l_ckpt.log.inuse, p2l, link);
315 
316 	p2l->entry_idx = 0;
317 	p2l->seq_id = seq_id;
318 	p2l->cb_fn = cb;
319 
320 	return p2l;
321 }
322 
323 void
324 ftl_p2l_log_release(struct spdk_ftl_dev *dev, struct ftl_p2l_log *p2l)
325 {
326 	assert(p2l);
327 
328 	/* TODO: Add assert if no ongoing IOs on the P2L log */
329 	/* TODO: Add assert if the P2L log already open */
330 
331 	TAILQ_REMOVE(&dev->p2l_ckpt.log.inuse, p2l, link);
332 	TAILQ_INSERT_TAIL(&dev->p2l_ckpt.log.free, p2l, link);
333 }
334 
335 static struct ftl_p2l_log *
336 p2l_log_get(struct spdk_ftl_dev *dev, enum ftl_layout_region_type type)
337 {
338 	struct ftl_p2l_log *p2l_Log;
339 
340 	TAILQ_FOREACH(p2l_Log, &dev->p2l_ckpt.log.free, link) {
341 		if (type == p2l_Log->md->region->type) {
342 			return p2l_Log;
343 		}
344 	}
345 
346 	TAILQ_FOREACH(p2l_Log, &dev->p2l_ckpt.log.inuse, link) {
347 		if (type == p2l_Log->md->region->type) {
348 			return p2l_Log;
349 		}
350 	}
351 
352 	return NULL;
353 }
354 
355 static bool
356 p2l_log_read_in_progress(struct ftl_p2l_log *p2l)
357 {
358 	return p2l->read_ctx.cb_fn ? true : false;
359 }
360 
361 static bool
362 ftl_p2l_log_read_is_next(struct ftl_p2l_log *p2l)
363 {
364 	if (p2l->read_ctx.result) {
365 		return false;
366 	}
367 
368 	return p2l->read_ctx.idx < p2l->entry_max;
369 }
370 
371 static bool
372 ftl_p2l_log_read_is_qd(struct ftl_p2l_log *p2l)
373 {
374 	return p2l->read_ctx.qd > 0;
375 }
376 
377 static bool
378 ftl_p2l_log_read_is_finished(struct ftl_p2l_log *p2l)
379 {
380 	if (ftl_p2l_log_read_is_next(p2l) || ftl_p2l_log_read_is_qd(p2l)) {
381 		return false;
382 	}
383 
384 	return true;
385 }
386 
387 static void
388 ftl_p2l_log_read_finish(struct ftl_p2l_log *p2l)
389 {
390 	spdk_ftl_fn cb_fn = p2l->read_ctx.cb_fn;
391 	void *cb_arg = p2l->read_ctx.cb_arg;
392 	int result = p2l->read_ctx.result;
393 
394 	memset(&p2l->read_ctx, 0, sizeof(p2l->read_ctx));
395 	cb_fn(cb_arg, result);
396 }
397 
398 static void
399 ftl_p2l_log_read_visit(struct ftl_p2l_log *p2l, struct ftl_p2l_log_page_ctrl *ctrl)
400 {
401 	struct ftl_p2l_log_page *page = &ctrl->page;
402 	struct spdk_ftl_dev *dev = p2l->dev;
403 	ftl_p2l_log_rd_cb cb_rd = p2l->read_ctx.cb_rd;
404 	void *cb_arg = p2l->read_ctx.cb_arg;
405 	uint64_t crc = p2l_log_page_crc(&ctrl->page);
406 	uint64_t i, j;
407 	int rc = 0;
408 
409 	ftl_bug(ctrl->entry_idx > p2l->entry_max);
410 
411 	if (p2l->read_ctx.seq_id != page->hdr.p2l_ckpt.seq_id) {
412 		/* This page contains entires older than the owner's sequence ID */
413 		return;
414 	}
415 
416 	if (ctrl->entry_idx != page->hdr.p2l_ckpt.idx) {
417 		FTL_ERRLOG(p2l->dev, "Read P2L IO Logs ERROR, invalid index, type %d\n",
418 			   p2l->md->region->type);
419 		p2l->read_ctx.result = -EINVAL;
420 		return;
421 	}
422 
423 	if (crc != page->hdr.p2l_ckpt.p2l_checksum) {
424 		FTL_ERRLOG(p2l->dev, "Read P2L IO Log ERROR, CRC problem, type %d\n",
425 			   p2l->md->region->type);
426 		p2l->read_ctx.result = -EINVAL;
427 		return;
428 	}
429 
430 	if (page->hdr.p2l_ckpt.count > SPDK_COUNTOF(page->items)) {
431 		FTL_ERRLOG(p2l->dev, "Read P2L IO Log ERROR, inconsistent format, type %d\n",
432 			   p2l->md->region->type);
433 		p2l->read_ctx.result = -EINVAL;
434 		return;
435 	}
436 
437 	for (i = 0; i < page->hdr.p2l_ckpt.count; i++) {
438 		struct ftl_pl2_log_item *item = &page->items[i];
439 
440 		for (j = 0; j < item->num_blocks; j++) {
441 			rc = cb_rd(dev, cb_arg, item->lba + j, item->addr + j, item->seq_id);
442 			if (rc) {
443 				p2l->read_ctx.result = rc;
444 				break;
445 			}
446 		}
447 
448 		if (rc) {
449 			break;
450 		}
451 	}
452 }
453 
454 static void
455 ftl_p2l_log_read_cb(int status, void *arg)
456 {
457 	struct ftl_p2l_log_page_ctrl *ctrl = arg;
458 	struct ftl_p2l_log *p2l = ctrl->p2l;
459 
460 	assert(p2l->read_ctx.qd > 0);
461 	p2l->read_ctx.qd--;
462 
463 	if (status) {
464 		p2l->read_ctx.result = status;
465 	} else {
466 		ftl_p2l_log_read_visit(p2l, ctrl);
467 	}
468 
469 	/* Release page control */
470 	ftl_mempool_put(p2l->page_pool, ctrl);
471 	ftl_p2l_log_read_process(p2l);
472 }
473 
474 static void
475 ftl_p2l_log_read_process(struct ftl_p2l_log *p2l)
476 {
477 	struct ftl_p2l_log_page_ctrl *ctrl;
478 
479 	while (ftl_p2l_log_read_is_next(p2l)) {
480 		ctrl = ftl_mempool_get(p2l->page_pool);
481 		if (!ctrl) {
482 			break;
483 		}
484 
485 		ctrl->p2l = p2l;
486 		ctrl->entry_idx = p2l->read_ctx.idx++;
487 
488 		/* Check if the index exceeding the buffer size */
489 		ftl_bug(p2l->read_ctx.idx > p2l->entry_max);
490 
491 		p2l->read_ctx.qd++;
492 		ftl_md_read_entry(p2l->md, ctrl->entry_idx, &ctrl->page, NULL,
493 				  ftl_p2l_log_read_cb, ctrl, &ctrl->md_ctx);
494 	}
495 
496 	if (ftl_p2l_log_read_is_finished(p2l)) {
497 		ftl_p2l_log_read_finish(p2l);
498 	}
499 }
500 
501 int
502 ftl_p2l_log_read(struct spdk_ftl_dev *dev, enum ftl_layout_region_type type, uint64_t seq_id,
503 		 spdk_ftl_fn cb_fn, void *cb_arg, ftl_p2l_log_rd_cb cb_rd)
504 {
505 	struct ftl_p2l_log *p2l_log = p2l_log_get(dev, type);
506 
507 	if (!p2l_log) {
508 		FTL_ERRLOG(dev, "Read P2L IO Log ERROR, no such log, type %d\n", type);
509 		return -ENODEV;
510 	}
511 	if (p2l_log_read_in_progress(p2l_log)) {
512 		FTL_ERRLOG(dev, "Read P2L IO Log ERROR, read busy, type %d\n", type);
513 		return -EBUSY;
514 	}
515 
516 	memset(&p2l_log->read_ctx, 0, sizeof(p2l_log->read_ctx));
517 	p2l_log->read_ctx.cb_fn = cb_fn;
518 	p2l_log->read_ctx.cb_arg = cb_arg;
519 	p2l_log->read_ctx.cb_rd = cb_rd;
520 	p2l_log->read_ctx.seq_id = seq_id;
521 
522 	ftl_p2l_log_read_process(p2l_log);
523 	if (ftl_p2l_log_read_is_qd(p2l_log)) {
524 		/* Read in progress */
525 		return 0;
526 	} else {
527 		FTL_ERRLOG(dev, "Read P2L IO Log ERROR, operation not started, type %d\n", type);
528 		return -EINVAL;
529 	}
530 }
531