1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright 2023 Solidigm All Rights Reserved 3 */ 4 5 #include "ftl_core.h" 6 #include "ftl_io.h" 7 #include "utils/ftl_defs.h" 8 9 struct ftl_pl2_log_item { 10 uint64_t lba; 11 uint64_t num_blocks; 12 uint64_t seq_id; 13 uint64_t addr; 14 }; 15 #define FTL_P2L_LOG_ITEMS_IN_PAGE ((FTL_BLOCK_SIZE - sizeof(union ftl_md_vss)) / sizeof(struct ftl_pl2_log_item)) 16 #define FTL_P2L_LOG_PAGE_COUNT_DEFAULT 128 17 18 struct ftl_p2l_log_page { 19 union ftl_md_vss hdr; 20 struct ftl_pl2_log_item items[FTL_P2L_LOG_ITEMS_IN_PAGE]; 21 }; 22 SPDK_STATIC_ASSERT(sizeof(struct ftl_p2l_log_page) == FTL_BLOCK_SIZE, "Invalid size of P2L page"); 23 24 struct ftl_p2l_log_page_ctrl { 25 struct ftl_p2l_log_page page; 26 struct ftl_p2l_log *p2l; 27 uint64_t entry_idx; 28 TAILQ_HEAD(, ftl_io) ios; 29 struct ftl_md_io_entry_ctx md_ctx; 30 }; 31 32 struct ftl_p2l_log { 33 struct spdk_ftl_dev *dev; 34 TAILQ_ENTRY(ftl_p2l_log) link; 35 TAILQ_HEAD(, ftl_io) ios; 36 struct ftl_md *md; 37 uint64_t seq_id; 38 struct ftl_mempool *page_pool; 39 uint64_t entry_idx; 40 ftl_p2l_log_cb cb_fn; 41 }; 42 43 static void p2l_log_page_io(struct ftl_p2l_log *p2l, struct ftl_p2l_log_page_ctrl *ctrl); 44 45 static struct ftl_p2l_log * 46 p2l_log_create(struct spdk_ftl_dev *dev, uint32_t region_type) 47 { 48 struct ftl_p2l_log *p2l; 49 50 p2l = calloc(1, sizeof(struct ftl_p2l_log)); 51 if (!p2l) { 52 return NULL; 53 } 54 55 TAILQ_INIT(&p2l->ios); 56 p2l->dev = dev; 57 p2l->md = dev->layout.md[region_type]; 58 p2l->page_pool = ftl_mempool_create(FTL_P2L_LOG_PAGE_COUNT_DEFAULT, 59 sizeof(struct ftl_p2l_log_page_ctrl), 60 FTL_BLOCK_SIZE, SPDK_ENV_SOCKET_ID_ANY); 61 if (!p2l->page_pool) { 62 goto ERROR; 63 } 64 65 return p2l; 66 ERROR: 67 free(p2l); 68 return NULL; 69 } 70 71 static void 72 p2l_log_destroy(struct ftl_p2l_log *p2l) 73 { 74 if (!p2l) { 75 return; 76 } 77 78 ftl_mempool_destroy(p2l->page_pool); 79 free(p2l); 80 } 81 82 static struct ftl_p2l_log_page_ctrl * 83 p2l_log_get_page(struct ftl_p2l_log *p2l) 84 { 85 struct ftl_p2l_log_page_ctrl *ctrl; 86 87 ctrl = ftl_mempool_get(p2l->page_pool); 88 if (!ctrl) { 89 return NULL; 90 } 91 92 /* Initialize P2L header */ 93 ctrl->page.hdr.p2l_ckpt.seq_id = p2l->seq_id; 94 ctrl->page.hdr.p2l_ckpt.count = 0; 95 ctrl->page.hdr.p2l_ckpt.p2l_checksum = 0; 96 97 /* Initialize the page control structure */ 98 ctrl->p2l = p2l; 99 ctrl->entry_idx = p2l->entry_idx; 100 TAILQ_INIT(&ctrl->ios); 101 102 /* Increase P2L page index */ 103 p2l->entry_idx++; 104 if (p2l->entry_idx > (ftl_md_get_buffer_size(p2l->md) / FTL_BLOCK_SIZE)) { 105 /* The index exceeding the buffer size */ 106 ftl_abort(); 107 } 108 109 return ctrl; 110 } 111 112 static bool 113 l2p_log_page_is_full(struct ftl_p2l_log_page_ctrl *ctrl) 114 { 115 return ctrl->page.hdr.p2l_ckpt.count == FTL_P2L_LOG_ITEMS_IN_PAGE; 116 } 117 118 static void 119 p2l_log_page_free(struct ftl_p2l_log *p2l, struct ftl_p2l_log_page_ctrl *ctrl) 120 { 121 ftl_mempool_put(p2l->page_pool, ctrl); 122 } 123 124 static void 125 p2l_log_handle_io_error(struct ftl_p2l_log *p2l, struct ftl_p2l_log_page_ctrl *ctrl) 126 { 127 #ifdef SPDK_FTL_RETRY_ON_ERROR 128 p2l_log_page_io(p2l, ctrl); 129 #else 130 ftl_abort(); 131 #endif 132 } 133 134 static uint32_t 135 p2l_log_page_crc(struct ftl_p2l_log_page *page) 136 { 137 uint32_t crc = 0; 138 void *buffer = page; 139 size_t size = sizeof(*page); 140 size_t offset = offsetof(struct ftl_p2l_log_page, hdr.p2l_ckpt.p2l_checksum); 141 142 crc = spdk_crc32c_update(buffer, offset, crc); 143 buffer += offset + sizeof(page->hdr.p2l_ckpt.p2l_checksum); 144 size -= offset + sizeof(page->hdr.p2l_ckpt.p2l_checksum); 145 146 return spdk_crc32c_update(buffer, size, crc); 147 } 148 149 static void 150 p2l_log_page_io_cb(int status, void *arg) 151 { 152 struct ftl_p2l_log_page_ctrl *ctrl = arg; 153 struct ftl_p2l_log *p2l = ctrl->p2l; 154 struct ftl_io *io; 155 156 if (status) { 157 p2l_log_handle_io_error(p2l, ctrl); 158 return; 159 } 160 161 while ((io = TAILQ_FIRST(&ctrl->ios))) { 162 TAILQ_REMOVE(&ctrl->ios, io, queue_entry); 163 p2l->cb_fn(io); 164 } 165 166 p2l_log_page_free(p2l, ctrl); 167 } 168 169 static void 170 p2l_log_page_io(struct ftl_p2l_log *p2l, struct ftl_p2l_log_page_ctrl *ctrl) 171 { 172 ctrl->page.hdr.p2l_ckpt.p2l_checksum = p2l_log_page_crc(&ctrl->page); 173 ftl_md_persist_entries(p2l->md, ctrl->entry_idx, 1, &ctrl->page, NULL, p2l_log_page_io_cb, 174 ctrl, &ctrl->md_ctx); 175 } 176 177 static void 178 p2l_log_add_io(struct ftl_p2l_log *p2l, struct ftl_p2l_log_page_ctrl *ctrl, struct ftl_io *io) 179 { 180 uint64_t i = ctrl->page.hdr.p2l_ckpt.count++; 181 182 assert(i < FTL_P2L_LOG_ITEMS_IN_PAGE); 183 ctrl->page.items[i].lba = io->lba; 184 ctrl->page.items[i].num_blocks = io->num_blocks; 185 ctrl->page.items[i].seq_id = io->nv_cache_chunk->md->seq_id; 186 ctrl->page.items[i].addr = io->addr; 187 188 /* TODO Make sure P2L map is updated respectively */ 189 190 TAILQ_REMOVE(&p2l->ios, io, queue_entry); 191 TAILQ_INSERT_TAIL(&ctrl->ios, io, queue_entry); 192 } 193 194 void 195 ftl_p2l_log_io(struct ftl_p2l_log *p2l, struct ftl_io *io) 196 { 197 TAILQ_INSERT_TAIL(&p2l->ios, io, queue_entry); 198 } 199 200 static void 201 p2l_log_flush(struct ftl_p2l_log *p2l) 202 { 203 struct ftl_p2l_log_page_ctrl *ctrl = NULL; 204 struct ftl_io *io; 205 206 while ((io = TAILQ_FIRST(&p2l->ios))) { 207 if (!ctrl) { 208 ctrl = p2l_log_get_page(p2l); 209 if (!ctrl) { 210 /* No page at the moment, try next time */ 211 break; 212 } 213 } 214 215 p2l_log_add_io(p2l, ctrl, io); 216 217 if (l2p_log_page_is_full(ctrl)) { 218 p2l_log_page_io(p2l, ctrl); 219 ctrl = NULL; 220 } 221 } 222 223 if (ctrl) { 224 p2l_log_page_io(p2l, ctrl); 225 } 226 } 227 228 void 229 ftl_p2l_log_flush(struct spdk_ftl_dev *dev) 230 { 231 struct ftl_p2l_log *p2l; 232 233 TAILQ_FOREACH(p2l, &dev->p2l_ckpt.log.inuse, link) { 234 p2l_log_flush(p2l); 235 } 236 } 237 238 uint64_t 239 ftl_p2l_log_get_md_blocks_required(struct spdk_ftl_dev *dev, uint64_t write_unit_blocks, 240 uint64_t max_user_data_blocks) 241 { 242 return spdk_divide_round_up(max_user_data_blocks, write_unit_blocks); 243 } 244 245 int 246 ftl_p2l_log_init(struct spdk_ftl_dev *dev) 247 { 248 struct ftl_p2l_log *p2l; 249 uint32_t region_type; 250 251 TAILQ_INIT(&dev->p2l_ckpt.log.free); 252 TAILQ_INIT(&dev->p2l_ckpt.log.inuse); 253 254 for (region_type = FTL_LAYOUT_REGION_TYPE_P2L_LOG_IO_MIN; 255 region_type <= FTL_LAYOUT_REGION_TYPE_P2L_LOG_IO_MAX; 256 region_type++) { 257 p2l = p2l_log_create(dev, region_type); 258 if (!p2l) { 259 return -ENOMEM; 260 } 261 262 TAILQ_INSERT_TAIL(&dev->p2l_ckpt.log.free, p2l, link); 263 } 264 265 return 0; 266 } 267 268 void 269 ftl_p2l_log_deinit(struct spdk_ftl_dev *dev) 270 { 271 struct ftl_p2l_log *p2l, *p2l_next; 272 273 TAILQ_FOREACH_SAFE(p2l, &dev->p2l_ckpt.log.free, link, p2l_next) { 274 TAILQ_REMOVE(&dev->p2l_ckpt.log.free, p2l, link); 275 p2l_log_destroy(p2l); 276 } 277 278 TAILQ_FOREACH_SAFE(p2l, &dev->p2l_ckpt.log.inuse, link, p2l_next) { 279 TAILQ_REMOVE(&dev->p2l_ckpt.log.inuse, p2l, link); 280 p2l_log_destroy(p2l); 281 } 282 } 283 284 enum ftl_layout_region_type 285 ftl_p2l_log_type(struct ftl_p2l_log *p2l) { 286 return p2l->md->region->type; 287 } 288 289 struct ftl_p2l_log * 290 ftl_p2l_log_acquire(struct spdk_ftl_dev *dev, uint64_t seq_id, ftl_p2l_log_cb cb) 291 { 292 struct ftl_p2l_log *p2l; 293 294 p2l = TAILQ_FIRST(&dev->p2l_ckpt.log.free); 295 assert(p2l); 296 TAILQ_REMOVE(&dev->p2l_ckpt.log.free, p2l, link); 297 TAILQ_INSERT_TAIL(&dev->p2l_ckpt.log.inuse, p2l, link); 298 299 p2l->entry_idx = 0; 300 p2l->seq_id = seq_id; 301 p2l->cb_fn = cb; 302 303 return p2l; 304 } 305 306 void 307 ftl_p2l_log_release(struct spdk_ftl_dev *dev, struct ftl_p2l_log *p2l) 308 { 309 assert(p2l); 310 311 /* TODO: Add assert if no ongoing IOs on the P2L log */ 312 /* TODO: Add assert if the P2L log already open */ 313 314 TAILQ_REMOVE(&dev->p2l_ckpt.log.inuse, p2l, link); 315 TAILQ_INSERT_TAIL(&dev->p2l_ckpt.log.free, p2l, link); 316 } 317