1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 7 #include "spdk/bdev.h" 8 #include "spdk/bdev_module.h" 9 #include "spdk/ftl.h" 10 #include "spdk/string.h" 11 12 #include "ftl_nv_cache.h" 13 #include "ftl_nv_cache_io.h" 14 #include "ftl_core.h" 15 #include "ftl_band.h" 16 #include "utils/ftl_addr_utils.h" 17 #include "mngt/ftl_mngt.h" 18 19 static inline uint64_t nvc_data_blocks(struct ftl_nv_cache *nv_cache) __attribute__((unused)); 20 static struct ftl_nv_cache_compactor *compactor_alloc(struct spdk_ftl_dev *dev); 21 static void compactor_free(struct spdk_ftl_dev *dev, struct ftl_nv_cache_compactor *compactor); 22 static void compaction_process_ftl_done(struct ftl_rq *rq); 23 24 static inline const struct ftl_layout_region * 25 nvc_data_region(struct ftl_nv_cache *nv_cache) 26 { 27 struct spdk_ftl_dev *dev; 28 29 dev = SPDK_CONTAINEROF(nv_cache, struct spdk_ftl_dev, nv_cache); 30 return &dev->layout.region[FTL_LAYOUT_REGION_TYPE_DATA_NVC]; 31 } 32 33 static inline void 34 nvc_validate_md(struct ftl_nv_cache *nv_cache, 35 struct ftl_nv_cache_chunk_md *chunk_md) 36 { 37 struct ftl_md *md = nv_cache->md; 38 void *buffer = ftl_md_get_buffer(md); 39 uint64_t size = ftl_md_get_buffer_size(md); 40 void *ptr = chunk_md; 41 42 if (ptr < buffer) { 43 ftl_abort(); 44 } 45 46 ptr += sizeof(*chunk_md); 47 if (ptr > buffer + size) { 48 ftl_abort(); 49 } 50 } 51 52 static inline uint64_t 53 nvc_data_offset(struct ftl_nv_cache *nv_cache) 54 { 55 return nvc_data_region(nv_cache)->current.offset; 56 } 57 58 static inline uint64_t 59 nvc_data_blocks(struct ftl_nv_cache *nv_cache) 60 { 61 return nvc_data_region(nv_cache)->current.blocks; 62 } 63 64 size_t 65 ftl_nv_cache_chunk_tail_md_num_blocks(const struct ftl_nv_cache *nv_cache) 66 { 67 struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(nv_cache, 68 struct spdk_ftl_dev, nv_cache); 69 return spdk_divide_round_up(dev->layout.nvc.chunk_data_blocks * dev->layout.l2p.addr_size, 70 FTL_BLOCK_SIZE); 71 } 72 73 static size_t 74 nv_cache_p2l_map_pool_elem_size(const struct ftl_nv_cache *nv_cache) 75 { 76 /* Map pool element holds the whole tail md */ 77 return nv_cache->tail_md_chunk_blocks * FTL_BLOCK_SIZE; 78 } 79 80 static uint64_t 81 get_chunk_idx(struct ftl_nv_cache_chunk *chunk) 82 { 83 struct ftl_nv_cache_chunk *first_chunk = chunk->nv_cache->chunks; 84 85 return (chunk->offset - first_chunk->offset) / chunk->nv_cache->chunk_blocks; 86 } 87 88 int 89 ftl_nv_cache_init(struct spdk_ftl_dev *dev) 90 { 91 struct ftl_nv_cache *nv_cache = &dev->nv_cache; 92 struct ftl_nv_cache_chunk *chunk; 93 struct ftl_nv_cache_chunk_md *md; 94 struct ftl_nv_cache_compactor *compactor; 95 uint64_t i, offset; 96 97 nv_cache->halt = true; 98 99 nv_cache->md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_NVC_MD]; 100 if (!nv_cache->md) { 101 FTL_ERRLOG(dev, "No NV cache metadata object\n"); 102 return -1; 103 } 104 105 nv_cache->md_pool = ftl_mempool_create(dev->conf.user_io_pool_size, 106 nv_cache->md_size * dev->xfer_size, 107 FTL_BLOCK_SIZE, SPDK_ENV_SOCKET_ID_ANY); 108 if (!nv_cache->md_pool) { 109 FTL_ERRLOG(dev, "Failed to initialize NV cache metadata pool\n"); 110 return -1; 111 } 112 113 /* 114 * Initialize chunk info 115 */ 116 nv_cache->chunk_blocks = dev->layout.nvc.chunk_data_blocks; 117 nv_cache->chunk_count = dev->layout.nvc.chunk_count; 118 nv_cache->tail_md_chunk_blocks = ftl_nv_cache_chunk_tail_md_num_blocks(nv_cache); 119 120 /* Allocate chunks */ 121 nv_cache->chunks = calloc(nv_cache->chunk_count, 122 sizeof(nv_cache->chunks[0])); 123 if (!nv_cache->chunks) { 124 FTL_ERRLOG(dev, "Failed to initialize NV cache chunks\n"); 125 return -1; 126 } 127 128 TAILQ_INIT(&nv_cache->chunk_free_list); 129 TAILQ_INIT(&nv_cache->chunk_open_list); 130 TAILQ_INIT(&nv_cache->chunk_full_list); 131 TAILQ_INIT(&nv_cache->chunk_comp_list); 132 TAILQ_INIT(&nv_cache->needs_free_persist_list); 133 134 /* First chunk metadata */ 135 md = ftl_md_get_buffer(nv_cache->md); 136 if (!md) { 137 FTL_ERRLOG(dev, "No NV cache metadata\n"); 138 return -1; 139 } 140 141 nv_cache->chunk_free_count = nv_cache->chunk_count; 142 143 chunk = nv_cache->chunks; 144 offset = nvc_data_offset(nv_cache); 145 for (i = 0; i < nv_cache->chunk_count; i++, chunk++, md++) { 146 chunk->nv_cache = nv_cache; 147 chunk->md = md; 148 nvc_validate_md(nv_cache, md); 149 chunk->offset = offset; 150 offset += nv_cache->chunk_blocks; 151 TAILQ_INSERT_TAIL(&nv_cache->chunk_free_list, chunk, entry); 152 } 153 assert(offset <= nvc_data_offset(nv_cache) + nvc_data_blocks(nv_cache)); 154 155 /* Start compaction when full chunks exceed given % of entire chunks */ 156 nv_cache->chunk_compaction_threshold = nv_cache->chunk_count * 157 dev->conf.nv_cache.chunk_compaction_threshold / 100; 158 TAILQ_INIT(&nv_cache->compactor_list); 159 for (i = 0; i < FTL_NV_CACHE_NUM_COMPACTORS; i++) { 160 compactor = compactor_alloc(dev); 161 162 if (!compactor) { 163 FTL_ERRLOG(dev, "Cannot allocate compaction process\n"); 164 return -1; 165 } 166 167 TAILQ_INSERT_TAIL(&nv_cache->compactor_list, compactor, entry); 168 } 169 170 #define FTL_MAX_OPEN_CHUNKS 2 171 nv_cache->p2l_pool = ftl_mempool_create(FTL_MAX_OPEN_CHUNKS, 172 nv_cache_p2l_map_pool_elem_size(nv_cache), 173 FTL_BLOCK_SIZE, 174 SPDK_ENV_SOCKET_ID_ANY); 175 if (!nv_cache->p2l_pool) { 176 return -ENOMEM; 177 } 178 179 /* One entry per open chunk */ 180 nv_cache->chunk_md_pool = ftl_mempool_create(FTL_MAX_OPEN_CHUNKS, 181 sizeof(struct ftl_nv_cache_chunk_md), 182 FTL_BLOCK_SIZE, 183 SPDK_ENV_SOCKET_ID_ANY); 184 if (!nv_cache->chunk_md_pool) { 185 return -ENOMEM; 186 } 187 188 /* Each compactor can be reading a different chunk which it needs to switch state to free to at the end, 189 * plus one backup each for high invalidity chunks processing (if there's a backlog of chunks with extremely 190 * small, even 0, validity then they can be processed by the compactors quickly and trigger a lot of updates 191 * to free state at once) */ 192 nv_cache->free_chunk_md_pool = ftl_mempool_create(2 * FTL_NV_CACHE_NUM_COMPACTORS, 193 sizeof(struct ftl_nv_cache_chunk_md), 194 FTL_BLOCK_SIZE, 195 SPDK_ENV_SOCKET_ID_ANY); 196 if (!nv_cache->free_chunk_md_pool) { 197 return -ENOMEM; 198 } 199 200 nv_cache->throttle.interval_tsc = FTL_NV_CACHE_THROTTLE_INTERVAL_MS * 201 (spdk_get_ticks_hz() / 1000); 202 nv_cache->chunk_free_target = spdk_divide_round_up(nv_cache->chunk_count * 203 dev->conf.nv_cache.chunk_free_target, 204 100); 205 return 0; 206 } 207 208 void 209 ftl_nv_cache_deinit(struct spdk_ftl_dev *dev) 210 { 211 struct ftl_nv_cache *nv_cache = &dev->nv_cache; 212 struct ftl_nv_cache_compactor *compactor; 213 214 while (!TAILQ_EMPTY(&nv_cache->compactor_list)) { 215 compactor = TAILQ_FIRST(&nv_cache->compactor_list); 216 TAILQ_REMOVE(&nv_cache->compactor_list, compactor, entry); 217 218 compactor_free(dev, compactor); 219 } 220 221 ftl_mempool_destroy(nv_cache->md_pool); 222 ftl_mempool_destroy(nv_cache->p2l_pool); 223 ftl_mempool_destroy(nv_cache->chunk_md_pool); 224 ftl_mempool_destroy(nv_cache->free_chunk_md_pool); 225 nv_cache->md_pool = NULL; 226 nv_cache->p2l_pool = NULL; 227 nv_cache->chunk_md_pool = NULL; 228 nv_cache->free_chunk_md_pool = NULL; 229 230 free(nv_cache->chunks); 231 nv_cache->chunks = NULL; 232 } 233 234 static uint64_t 235 chunk_get_free_space(struct ftl_nv_cache *nv_cache, 236 struct ftl_nv_cache_chunk *chunk) 237 { 238 assert(chunk->md->write_pointer + nv_cache->tail_md_chunk_blocks <= 239 nv_cache->chunk_blocks); 240 return nv_cache->chunk_blocks - chunk->md->write_pointer - 241 nv_cache->tail_md_chunk_blocks; 242 } 243 244 static bool 245 chunk_is_closed(struct ftl_nv_cache_chunk *chunk) 246 { 247 return chunk->md->write_pointer == chunk->nv_cache->chunk_blocks; 248 } 249 250 static void ftl_chunk_close(struct ftl_nv_cache_chunk *chunk); 251 252 static uint64_t 253 ftl_nv_cache_get_wr_buffer(struct ftl_nv_cache *nv_cache, struct ftl_io *io) 254 { 255 uint64_t address = FTL_LBA_INVALID; 256 uint64_t num_blocks = io->num_blocks; 257 uint64_t free_space; 258 struct ftl_nv_cache_chunk *chunk; 259 260 do { 261 chunk = nv_cache->chunk_current; 262 /* Chunk has been closed so pick new one */ 263 if (chunk && chunk_is_closed(chunk)) { 264 chunk = NULL; 265 } 266 267 if (!chunk) { 268 chunk = TAILQ_FIRST(&nv_cache->chunk_open_list); 269 if (chunk && chunk->md->state == FTL_CHUNK_STATE_OPEN) { 270 TAILQ_REMOVE(&nv_cache->chunk_open_list, chunk, entry); 271 nv_cache->chunk_current = chunk; 272 } else { 273 break; 274 } 275 } 276 277 free_space = chunk_get_free_space(nv_cache, chunk); 278 279 if (free_space >= num_blocks) { 280 /* Enough space in chunk */ 281 282 /* Calculate address in NV cache */ 283 address = chunk->offset + chunk->md->write_pointer; 284 285 /* Set chunk in IO */ 286 io->nv_cache_chunk = chunk; 287 288 /* Move write pointer */ 289 chunk->md->write_pointer += num_blocks; 290 break; 291 } 292 293 /* Not enough space in nv_cache_chunk */ 294 nv_cache->chunk_current = NULL; 295 296 if (0 == free_space) { 297 continue; 298 } 299 300 chunk->md->blocks_skipped = free_space; 301 chunk->md->blocks_written += free_space; 302 chunk->md->write_pointer += free_space; 303 304 if (chunk->md->blocks_written == chunk_tail_md_offset(nv_cache)) { 305 ftl_chunk_close(chunk); 306 } 307 } while (1); 308 309 return address; 310 } 311 312 void 313 ftl_nv_cache_fill_md(struct ftl_io *io) 314 { 315 struct ftl_nv_cache_chunk *chunk = io->nv_cache_chunk; 316 uint64_t i; 317 union ftl_md_vss *metadata = io->md; 318 uint64_t lba = ftl_io_get_lba(io, 0); 319 320 for (i = 0; i < io->num_blocks; ++i, lba++, metadata++) { 321 metadata->nv_cache.lba = lba; 322 metadata->nv_cache.seq_id = chunk->md->seq_id; 323 } 324 } 325 326 uint64_t 327 chunk_tail_md_offset(struct ftl_nv_cache *nv_cache) 328 { 329 return nv_cache->chunk_blocks - nv_cache->tail_md_chunk_blocks; 330 } 331 332 static void 333 chunk_advance_blocks(struct ftl_nv_cache *nv_cache, struct ftl_nv_cache_chunk *chunk, 334 uint64_t advanced_blocks) 335 { 336 chunk->md->blocks_written += advanced_blocks; 337 338 assert(chunk->md->blocks_written <= nv_cache->chunk_blocks); 339 340 if (chunk->md->blocks_written == chunk_tail_md_offset(nv_cache)) { 341 ftl_chunk_close(chunk); 342 } 343 } 344 345 static uint64_t 346 chunk_user_blocks_written(struct ftl_nv_cache_chunk *chunk) 347 { 348 return chunk->md->blocks_written - chunk->md->blocks_skipped - 349 chunk->nv_cache->tail_md_chunk_blocks; 350 } 351 352 static bool 353 is_chunk_compacted(struct ftl_nv_cache_chunk *chunk) 354 { 355 assert(chunk->md->blocks_written != 0); 356 357 if (chunk_user_blocks_written(chunk) == chunk->md->blocks_compacted) { 358 return true; 359 } 360 361 return false; 362 } 363 364 static int 365 ftl_chunk_alloc_md_entry(struct ftl_nv_cache_chunk *chunk) 366 { 367 struct ftl_nv_cache *nv_cache = chunk->nv_cache; 368 struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(nv_cache, struct spdk_ftl_dev, nv_cache); 369 struct ftl_p2l_map *p2l_map = &chunk->p2l_map; 370 struct ftl_layout_region *region = &dev->layout.region[FTL_LAYOUT_REGION_TYPE_NVC_MD]; 371 372 p2l_map->chunk_dma_md = ftl_mempool_get(nv_cache->chunk_md_pool); 373 374 if (!p2l_map->chunk_dma_md) { 375 return -ENOMEM; 376 } 377 378 memset(p2l_map->chunk_dma_md, 0, region->entry_size * FTL_BLOCK_SIZE); 379 return 0; 380 } 381 382 static void 383 ftl_chunk_free_md_entry(struct ftl_nv_cache_chunk *chunk) 384 { 385 struct ftl_p2l_map *p2l_map = &chunk->p2l_map; 386 387 ftl_mempool_put(chunk->nv_cache->chunk_md_pool, p2l_map->chunk_dma_md); 388 p2l_map->chunk_dma_md = NULL; 389 } 390 391 static void 392 ftl_chunk_free(struct ftl_nv_cache_chunk *chunk) 393 { 394 struct ftl_nv_cache *nv_cache = chunk->nv_cache; 395 396 /* Reset chunk */ 397 memset(chunk->md, 0, sizeof(*chunk->md)); 398 399 TAILQ_INSERT_TAIL(&nv_cache->needs_free_persist_list, chunk, entry); 400 nv_cache->chunk_free_persist_count++; 401 } 402 403 static int 404 ftl_chunk_alloc_chunk_free_entry(struct ftl_nv_cache_chunk *chunk) 405 { 406 struct ftl_nv_cache *nv_cache = chunk->nv_cache; 407 struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(nv_cache, struct spdk_ftl_dev, nv_cache); 408 struct ftl_p2l_map *p2l_map = &chunk->p2l_map; 409 struct ftl_layout_region *region = &dev->layout.region[FTL_LAYOUT_REGION_TYPE_NVC_MD]; 410 411 p2l_map->chunk_dma_md = ftl_mempool_get(nv_cache->free_chunk_md_pool); 412 413 if (!p2l_map->chunk_dma_md) { 414 return -ENOMEM; 415 } 416 417 memset(p2l_map->chunk_dma_md, 0, region->entry_size * FTL_BLOCK_SIZE); 418 return 0; 419 } 420 421 static void 422 ftl_chunk_free_chunk_free_entry(struct ftl_nv_cache_chunk *chunk) 423 { 424 struct ftl_p2l_map *p2l_map = &chunk->p2l_map; 425 426 ftl_mempool_put(chunk->nv_cache->free_chunk_md_pool, p2l_map->chunk_dma_md); 427 p2l_map->chunk_dma_md = NULL; 428 } 429 430 static void 431 chunk_free_cb(int status, void *ctx) 432 { 433 struct ftl_nv_cache_chunk *chunk = (struct ftl_nv_cache_chunk *)ctx; 434 435 if (spdk_likely(!status)) { 436 struct ftl_nv_cache *nv_cache = chunk->nv_cache; 437 438 nv_cache->chunk_free_persist_count--; 439 TAILQ_INSERT_TAIL(&nv_cache->chunk_free_list, chunk, entry); 440 nv_cache->chunk_free_count++; 441 nv_cache->chunk_full_count--; 442 chunk->md->state = FTL_CHUNK_STATE_FREE; 443 chunk->md->close_seq_id = 0; 444 ftl_chunk_free_chunk_free_entry(chunk); 445 } else { 446 #ifdef SPDK_FTL_RETRY_ON_ERROR 447 ftl_md_persist_entry_retry(&chunk->md_persist_entry_ctx); 448 #else 449 ftl_abort(); 450 #endif 451 } 452 } 453 454 static void 455 ftl_chunk_persist_free_state(struct ftl_nv_cache *nv_cache) 456 { 457 int rc; 458 struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(nv_cache, struct spdk_ftl_dev, nv_cache); 459 struct ftl_p2l_map *p2l_map; 460 struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_NVC_MD]; 461 struct ftl_layout_region *region = &dev->layout.region[FTL_LAYOUT_REGION_TYPE_NVC_MD]; 462 struct ftl_nv_cache_chunk *tchunk, *chunk = NULL; 463 464 TAILQ_FOREACH_SAFE(chunk, &nv_cache->needs_free_persist_list, entry, tchunk) { 465 p2l_map = &chunk->p2l_map; 466 rc = ftl_chunk_alloc_chunk_free_entry(chunk); 467 if (rc) { 468 break; 469 } 470 471 TAILQ_REMOVE(&nv_cache->needs_free_persist_list, chunk, entry); 472 473 memcpy(p2l_map->chunk_dma_md, chunk->md, region->entry_size * FTL_BLOCK_SIZE); 474 p2l_map->chunk_dma_md->state = FTL_CHUNK_STATE_FREE; 475 p2l_map->chunk_dma_md->close_seq_id = 0; 476 p2l_map->chunk_dma_md->p2l_map_checksum = 0; 477 478 ftl_md_persist_entry(md, get_chunk_idx(chunk), p2l_map->chunk_dma_md, NULL, 479 chunk_free_cb, chunk, &chunk->md_persist_entry_ctx); 480 } 481 } 482 483 static void 484 compaction_stats_update(struct ftl_nv_cache_chunk *chunk) 485 { 486 struct ftl_nv_cache *nv_cache = chunk->nv_cache; 487 struct compaction_bw_stats *compaction_bw = &nv_cache->compaction_recent_bw; 488 double *ptr; 489 490 if (spdk_unlikely(chunk->compaction_length_tsc == 0)) { 491 return; 492 } 493 494 if (spdk_likely(compaction_bw->count == FTL_NV_CACHE_COMPACTION_SMA_N)) { 495 ptr = compaction_bw->buf + compaction_bw->first; 496 compaction_bw->first++; 497 if (compaction_bw->first == FTL_NV_CACHE_COMPACTION_SMA_N) { 498 compaction_bw->first = 0; 499 } 500 compaction_bw->sum -= *ptr; 501 } else { 502 ptr = compaction_bw->buf + compaction_bw->count; 503 compaction_bw->count++; 504 } 505 506 *ptr = (double)chunk->md->blocks_compacted * FTL_BLOCK_SIZE / chunk->compaction_length_tsc; 507 chunk->compaction_length_tsc = 0; 508 509 compaction_bw->sum += *ptr; 510 nv_cache->compaction_sma = compaction_bw->sum / compaction_bw->count; 511 } 512 513 static void 514 chunk_compaction_advance(struct ftl_nv_cache_chunk *chunk, uint64_t num_blocks) 515 { 516 struct ftl_nv_cache *nv_cache = chunk->nv_cache; 517 uint64_t tsc = spdk_thread_get_last_tsc(spdk_get_thread()); 518 519 chunk->compaction_length_tsc += tsc - chunk->compaction_start_tsc; 520 chunk->compaction_start_tsc = tsc; 521 522 chunk->md->blocks_compacted += num_blocks; 523 if (!is_chunk_compacted(chunk)) { 524 return; 525 } 526 527 /* Remove chunk from compacted list */ 528 TAILQ_REMOVE(&nv_cache->chunk_comp_list, chunk, entry); 529 nv_cache->chunk_comp_count--; 530 531 compaction_stats_update(chunk); 532 533 ftl_chunk_free(chunk); 534 } 535 536 static bool 537 is_compaction_required(struct ftl_nv_cache *nv_cache) 538 { 539 uint64_t full; 540 541 if (spdk_unlikely(nv_cache->halt)) { 542 return false; 543 } 544 545 full = nv_cache->chunk_full_count - nv_cache->compaction_active_count; 546 if (full >= nv_cache->chunk_compaction_threshold) { 547 return true; 548 } 549 550 return false; 551 } 552 553 static void compaction_process_finish_read(struct ftl_nv_cache_compactor *compactor); 554 static void compaction_process_pin_lba(struct ftl_nv_cache_compactor *comp); 555 556 static void 557 _compaction_process_pin_lba(void *_comp) 558 { 559 struct ftl_nv_cache_compactor *comp = _comp; 560 561 compaction_process_pin_lba(comp); 562 } 563 564 static void 565 compaction_process_pin_lba_cb(struct spdk_ftl_dev *dev, int status, struct ftl_l2p_pin_ctx *pin_ctx) 566 { 567 struct ftl_nv_cache_compactor *comp = pin_ctx->cb_ctx; 568 struct ftl_rq *rq = comp->rd; 569 570 if (status) { 571 rq->iter.status = status; 572 pin_ctx->lba = FTL_LBA_INVALID; 573 } 574 575 if (--rq->iter.remaining == 0) { 576 if (rq->iter.status) { 577 /* unpin and try again */ 578 ftl_rq_unpin(rq); 579 spdk_thread_send_msg(spdk_get_thread(), _compaction_process_pin_lba, comp); 580 return; 581 } 582 583 compaction_process_finish_read(comp); 584 } 585 } 586 587 static void 588 compaction_process_pin_lba(struct ftl_nv_cache_compactor *comp) 589 { 590 union ftl_md_vss *md; 591 struct ftl_nv_cache_chunk *chunk = comp->rd->owner.priv; 592 struct spdk_ftl_dev *dev = comp->rd->dev; 593 uint64_t i; 594 uint32_t count = comp->rd->iter.count; 595 struct ftl_rq_entry *entry; 596 struct ftl_l2p_pin_ctx *pin_ctx; 597 598 assert(comp->rd->iter.idx == 0); 599 comp->rd->iter.remaining = count; 600 comp->rd->iter.status = 0; 601 602 for (i = 0; i < count; i++) { 603 entry = &comp->rd->entries[i]; 604 pin_ctx = &entry->l2p_pin_ctx; 605 md = entry->io_md; 606 if (md->nv_cache.lba == FTL_LBA_INVALID || md->nv_cache.seq_id != chunk->md->seq_id) { 607 ftl_l2p_pin_skip(dev, compaction_process_pin_lba_cb, comp, pin_ctx); 608 } else { 609 ftl_l2p_pin(dev, md->nv_cache.lba, 1, compaction_process_pin_lba_cb, comp, pin_ctx); 610 } 611 } 612 } 613 614 static int compaction_submit_read(struct ftl_nv_cache_compactor *compactor, ftl_addr addr, 615 uint64_t num_blocks); 616 617 static void 618 compaction_retry_read(void *_compactor) 619 { 620 struct ftl_nv_cache_compactor *compactor = _compactor; 621 struct ftl_rq *rq = compactor->rd; 622 struct spdk_bdev *bdev; 623 int ret; 624 625 ret = compaction_submit_read(compactor, rq->io.addr, rq->iter.count); 626 627 if (spdk_likely(!ret)) { 628 return; 629 } 630 631 if (ret == -ENOMEM) { 632 bdev = spdk_bdev_desc_get_bdev(compactor->nv_cache->bdev_desc); 633 compactor->bdev_io_wait.bdev = bdev; 634 compactor->bdev_io_wait.cb_fn = compaction_retry_read; 635 compactor->bdev_io_wait.cb_arg = compactor; 636 spdk_bdev_queue_io_wait(bdev, compactor->nv_cache->cache_ioch, &compactor->bdev_io_wait); 637 } else { 638 ftl_abort(); 639 } 640 } 641 642 static void 643 compaction_process_read_cb(struct spdk_bdev_io *bdev_io, 644 bool success, void *cb_arg) 645 { 646 struct ftl_nv_cache_compactor *compactor = cb_arg; 647 struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(compactor->nv_cache, struct spdk_ftl_dev, nv_cache); 648 649 ftl_stats_bdev_io_completed(dev, FTL_STATS_TYPE_CMP, bdev_io); 650 651 spdk_bdev_free_io(bdev_io); 652 653 if (!success) { 654 /* retry */ 655 spdk_thread_send_msg(spdk_get_thread(), compaction_retry_read, compactor); 656 return; 657 } 658 659 compaction_process_pin_lba(compactor); 660 } 661 662 static bool 663 is_chunk_to_read(struct ftl_nv_cache_chunk *chunk) 664 { 665 assert(chunk->md->blocks_written != 0); 666 667 if (chunk_user_blocks_written(chunk) == chunk->md->read_pointer) { 668 return false; 669 } 670 671 return true; 672 } 673 674 static struct ftl_nv_cache_chunk * 675 get_chunk_for_compaction(struct ftl_nv_cache *nv_cache) 676 { 677 struct ftl_nv_cache_chunk *chunk = NULL; 678 679 if (!TAILQ_EMPTY(&nv_cache->chunk_comp_list)) { 680 chunk = TAILQ_FIRST(&nv_cache->chunk_comp_list); 681 if (is_chunk_to_read(chunk)) { 682 return chunk; 683 } 684 } 685 686 if (!TAILQ_EMPTY(&nv_cache->chunk_full_list)) { 687 chunk = TAILQ_FIRST(&nv_cache->chunk_full_list); 688 TAILQ_REMOVE(&nv_cache->chunk_full_list, chunk, entry); 689 690 assert(chunk->md->write_pointer); 691 } else { 692 return NULL; 693 } 694 695 if (spdk_likely(chunk)) { 696 assert(chunk->md->write_pointer != 0); 697 TAILQ_INSERT_HEAD(&nv_cache->chunk_comp_list, chunk, entry); 698 nv_cache->chunk_comp_count++; 699 } 700 701 return chunk; 702 } 703 704 static uint64_t 705 chunk_blocks_to_read(struct ftl_nv_cache_chunk *chunk) 706 { 707 uint64_t blocks_written; 708 uint64_t blocks_to_read; 709 710 assert(chunk->md->blocks_written >= chunk->md->blocks_skipped); 711 blocks_written = chunk_user_blocks_written(chunk); 712 713 assert(blocks_written >= chunk->md->read_pointer); 714 blocks_to_read = blocks_written - chunk->md->read_pointer; 715 716 return blocks_to_read; 717 } 718 719 static void 720 compactor_deactivate(struct ftl_nv_cache_compactor *compactor) 721 { 722 struct ftl_nv_cache *nv_cache = compactor->nv_cache; 723 724 nv_cache->compaction_active_count--; 725 TAILQ_INSERT_TAIL(&nv_cache->compactor_list, compactor, entry); 726 } 727 728 static int 729 compaction_submit_read(struct ftl_nv_cache_compactor *compactor, ftl_addr addr, 730 uint64_t num_blocks) 731 { 732 struct ftl_nv_cache *nv_cache = compactor->nv_cache; 733 struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(nv_cache, struct spdk_ftl_dev, nv_cache); 734 735 return ftl_nv_cache_bdev_readv_blocks_with_md(dev, nv_cache->bdev_desc, 736 nv_cache->cache_ioch, 737 compactor->rd->io_vec, num_blocks, 738 compactor->rd->io_md, 739 ftl_addr_to_nvc_offset(dev, addr), num_blocks, 740 compaction_process_read_cb, 741 compactor); 742 } 743 744 static void 745 compaction_process_pad(struct ftl_nv_cache_compactor *compactor) 746 { 747 struct ftl_rq *wr = compactor->wr; 748 const uint64_t num_entries = wr->num_blocks; 749 struct ftl_rq_entry *iter; 750 751 iter = &wr->entries[wr->iter.idx]; 752 753 while (wr->iter.idx < num_entries) { 754 iter->addr = FTL_ADDR_INVALID; 755 iter->owner.priv = NULL; 756 iter->lba = FTL_LBA_INVALID; 757 iter->seq_id = 0; 758 iter++; 759 wr->iter.idx++; 760 } 761 } 762 763 static void 764 compaction_process(struct ftl_nv_cache_compactor *compactor) 765 { 766 struct ftl_nv_cache *nv_cache = compactor->nv_cache; 767 struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(nv_cache, 768 struct spdk_ftl_dev, nv_cache); 769 struct ftl_nv_cache_chunk *chunk; 770 uint64_t to_read, addr, begin, end, offset; 771 int rc; 772 773 /* Check if all read blocks done */ 774 assert(compactor->rd->iter.idx <= compactor->rd->iter.count); 775 if (compactor->rd->iter.idx < compactor->rd->iter.count) { 776 compaction_process_finish_read(compactor); 777 return; 778 } 779 780 /* 781 * Get currently handled chunk 782 */ 783 chunk = get_chunk_for_compaction(nv_cache); 784 if (!chunk) { 785 /* No chunks to compact, pad this request */ 786 compaction_process_pad(compactor); 787 ftl_writer_queue_rq(&dev->writer_user, compactor->wr); 788 return; 789 } 790 791 chunk->compaction_start_tsc = spdk_thread_get_last_tsc(spdk_get_thread()); 792 793 /* 794 * Get range of blocks to read 795 */ 796 to_read = chunk_blocks_to_read(chunk); 797 assert(to_read > 0); 798 799 addr = ftl_addr_from_nvc_offset(dev, chunk->offset + chunk->md->read_pointer); 800 begin = ftl_bitmap_find_first_set(dev->valid_map, addr, addr + to_read); 801 if (begin != UINT64_MAX) { 802 offset = spdk_min(begin - addr, to_read); 803 } else { 804 offset = to_read; 805 } 806 807 if (offset) { 808 chunk->md->read_pointer += offset; 809 chunk_compaction_advance(chunk, offset); 810 to_read -= offset; 811 if (!to_read) { 812 compactor_deactivate(compactor); 813 return; 814 } 815 } 816 817 end = ftl_bitmap_find_first_clear(dev->valid_map, begin + 1, begin + to_read); 818 if (end != UINT64_MAX) { 819 to_read = end - begin; 820 } 821 822 addr = begin; 823 to_read = spdk_min(to_read, compactor->rd->num_blocks); 824 825 /* Read data and metadata from NV cache */ 826 rc = compaction_submit_read(compactor, addr, to_read); 827 if (spdk_unlikely(rc)) { 828 /* An error occurred, inactivate this compactor, it will retry 829 * in next iteration 830 */ 831 compactor_deactivate(compactor); 832 return; 833 } 834 835 /* IO has started, initialize compaction */ 836 compactor->rd->owner.priv = chunk; 837 compactor->rd->iter.idx = 0; 838 compactor->rd->iter.count = to_read; 839 compactor->rd->io.addr = addr; 840 841 /* Move read pointer in the chunk */ 842 chunk->md->read_pointer += to_read; 843 } 844 845 static void 846 compaction_process_start(struct ftl_nv_cache_compactor *compactor) 847 { 848 compactor->nv_cache->compaction_active_count++; 849 compaction_process(compactor); 850 } 851 852 static void 853 compaction_process_ftl_done(struct ftl_rq *rq) 854 { 855 struct spdk_ftl_dev *dev = rq->dev; 856 struct ftl_nv_cache_compactor *compactor = rq->owner.priv; 857 struct ftl_nv_cache *nv_cache = &dev->nv_cache; 858 struct ftl_band *band = rq->io.band; 859 struct ftl_rq_entry *entry; 860 ftl_addr addr; 861 uint64_t i; 862 863 if (spdk_unlikely(false == rq->success)) { 864 /* IO error retry writing */ 865 #ifdef SPDK_FTL_RETRY_ON_ERROR 866 ftl_writer_queue_rq(&dev->writer_user, rq); 867 return; 868 #else 869 ftl_abort(); 870 #endif 871 } 872 873 /* Update L2P table */ 874 addr = rq->io.addr; 875 for (i = 0, entry = rq->entries; i < rq->num_blocks; i++, entry++) { 876 struct ftl_nv_cache_chunk *chunk = entry->owner.priv; 877 878 if (entry->lba == FTL_LBA_INVALID) { 879 assert(entry->addr == FTL_ADDR_INVALID); 880 addr = ftl_band_next_addr(band, addr, 1); 881 continue; 882 } 883 884 ftl_l2p_update_base(dev, entry->lba, addr, entry->addr); 885 ftl_l2p_unpin(dev, entry->lba, 1); 886 887 chunk_compaction_advance(chunk, 1); 888 addr = ftl_band_next_addr(band, addr, 1); 889 } 890 891 compactor->wr->iter.idx = 0; 892 893 if (is_compaction_required(nv_cache)) { 894 compaction_process(compactor); 895 } else { 896 compactor_deactivate(compactor); 897 } 898 } 899 900 static void 901 compaction_process_finish_read(struct ftl_nv_cache_compactor *compactor) 902 { 903 struct ftl_rq *wr = compactor->wr; 904 struct ftl_rq *rd = compactor->rd; 905 ftl_addr cache_addr = rd->io.addr; 906 struct ftl_nv_cache_chunk *chunk = rd->owner.priv; 907 struct spdk_ftl_dev *dev; 908 struct ftl_rq_entry *iter; 909 union ftl_md_vss *md; 910 ftl_addr current_addr; 911 const uint64_t num_entries = wr->num_blocks; 912 uint64_t tsc = spdk_thread_get_last_tsc(spdk_get_thread()); 913 914 chunk->compaction_length_tsc += tsc - chunk->compaction_start_tsc; 915 chunk->compaction_start_tsc = tsc; 916 917 dev = SPDK_CONTAINEROF(compactor->nv_cache, 918 struct spdk_ftl_dev, nv_cache); 919 920 assert(wr->iter.idx < num_entries); 921 assert(rd->iter.idx < rd->iter.count); 922 923 cache_addr += rd->iter.idx; 924 iter = &wr->entries[wr->iter.idx]; 925 926 while (wr->iter.idx < num_entries && rd->iter.idx < rd->iter.count) { 927 /* Get metadata */ 928 md = rd->entries[rd->iter.idx].io_md; 929 if (md->nv_cache.lba == FTL_LBA_INVALID || md->nv_cache.seq_id != chunk->md->seq_id) { 930 cache_addr++; 931 rd->iter.idx++; 932 chunk_compaction_advance(chunk, 1); 933 continue; 934 } 935 936 current_addr = ftl_l2p_get(dev, md->nv_cache.lba); 937 if (current_addr == cache_addr) { 938 /* Swap payload */ 939 ftl_rq_swap_payload(wr, wr->iter.idx, rd, rd->iter.idx); 940 941 /* 942 * Address still the same, we may continue to compact it 943 * back to FTL, set valid number of entries within 944 * this batch 945 */ 946 iter->addr = current_addr; 947 iter->owner.priv = chunk; 948 iter->lba = md->nv_cache.lba; 949 iter->seq_id = chunk->md->seq_id; 950 951 /* Advance within batch */ 952 iter++; 953 wr->iter.idx++; 954 } else { 955 /* This address already invalidated, just omit this block */ 956 chunk_compaction_advance(chunk, 1); 957 ftl_l2p_unpin(dev, md->nv_cache.lba, 1); 958 } 959 960 /* Advance within reader */ 961 rd->iter.idx++; 962 cache_addr++; 963 } 964 965 if (num_entries == wr->iter.idx) { 966 /* 967 * Request contains data to be placed on FTL, compact it 968 */ 969 ftl_writer_queue_rq(&dev->writer_user, wr); 970 } else { 971 if (is_compaction_required(compactor->nv_cache)) { 972 compaction_process(compactor); 973 } else { 974 compactor_deactivate(compactor); 975 } 976 } 977 } 978 979 static void 980 compactor_free(struct spdk_ftl_dev *dev, struct ftl_nv_cache_compactor *compactor) 981 { 982 if (!compactor) { 983 return; 984 } 985 986 ftl_rq_del(compactor->wr); 987 ftl_rq_del(compactor->rd); 988 free(compactor); 989 } 990 991 static struct ftl_nv_cache_compactor * 992 compactor_alloc(struct spdk_ftl_dev *dev) 993 { 994 struct ftl_nv_cache_compactor *compactor; 995 996 compactor = calloc(1, sizeof(*compactor)); 997 if (!compactor) { 998 goto error; 999 } 1000 1001 /* Allocate help request for writing */ 1002 compactor->wr = ftl_rq_new(dev, dev->md_size); 1003 if (!compactor->wr) { 1004 goto error; 1005 } 1006 1007 /* Allocate help request for reading */ 1008 compactor->rd = ftl_rq_new(dev, dev->nv_cache.md_size); 1009 if (!compactor->rd) { 1010 goto error; 1011 } 1012 1013 compactor->nv_cache = &dev->nv_cache; 1014 compactor->wr->owner.priv = compactor; 1015 compactor->wr->owner.cb = compaction_process_ftl_done; 1016 compactor->wr->owner.compaction = true; 1017 1018 return compactor; 1019 1020 error: 1021 compactor_free(dev, compactor); 1022 return NULL; 1023 } 1024 1025 static void 1026 ftl_nv_cache_submit_cb_done(struct ftl_io *io) 1027 { 1028 struct ftl_nv_cache *nv_cache = &io->dev->nv_cache; 1029 1030 chunk_advance_blocks(nv_cache, io->nv_cache_chunk, io->num_blocks); 1031 io->nv_cache_chunk = NULL; 1032 1033 ftl_mempool_put(nv_cache->md_pool, io->md); 1034 ftl_io_complete(io); 1035 } 1036 1037 static void 1038 ftl_nv_cache_l2p_update(struct ftl_io *io) 1039 { 1040 struct spdk_ftl_dev *dev = io->dev; 1041 ftl_addr next_addr = io->addr; 1042 size_t i; 1043 1044 for (i = 0; i < io->num_blocks; ++i, ++next_addr) { 1045 ftl_l2p_update_cache(dev, ftl_io_get_lba(io, i), next_addr, io->map[i]); 1046 } 1047 1048 ftl_l2p_unpin(dev, io->lba, io->num_blocks); 1049 ftl_nv_cache_submit_cb_done(io); 1050 } 1051 1052 static void 1053 ftl_nv_cache_submit_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 1054 { 1055 struct ftl_io *io = cb_arg; 1056 1057 ftl_stats_bdev_io_completed(io->dev, FTL_STATS_TYPE_USER, bdev_io); 1058 1059 spdk_bdev_free_io(bdev_io); 1060 1061 if (spdk_unlikely(!success)) { 1062 FTL_ERRLOG(io->dev, "Non-volatile cache write failed at %"PRIx64"\n", 1063 io->addr); 1064 io->status = -EIO; 1065 ftl_nv_cache_submit_cb_done(io); 1066 } else { 1067 ftl_nv_cache_l2p_update(io); 1068 } 1069 } 1070 1071 static void 1072 nv_cache_write(void *_io) 1073 { 1074 struct ftl_io *io = _io; 1075 struct spdk_ftl_dev *dev = io->dev; 1076 struct ftl_nv_cache *nv_cache = &dev->nv_cache; 1077 int rc; 1078 1079 rc = ftl_nv_cache_bdev_writev_blocks_with_md(dev, 1080 nv_cache->bdev_desc, nv_cache->cache_ioch, 1081 io->iov, io->iov_cnt, io->md, 1082 ftl_addr_to_nvc_offset(dev, io->addr), io->num_blocks, 1083 ftl_nv_cache_submit_cb, io); 1084 if (spdk_unlikely(rc)) { 1085 if (rc == -ENOMEM) { 1086 struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(nv_cache->bdev_desc); 1087 io->bdev_io_wait.bdev = bdev; 1088 io->bdev_io_wait.cb_fn = nv_cache_write; 1089 io->bdev_io_wait.cb_arg = io; 1090 spdk_bdev_queue_io_wait(bdev, nv_cache->cache_ioch, &io->bdev_io_wait); 1091 } else { 1092 ftl_abort(); 1093 } 1094 } 1095 } 1096 1097 static void 1098 ftl_nv_cache_pin_cb(struct spdk_ftl_dev *dev, int status, struct ftl_l2p_pin_ctx *pin_ctx) 1099 { 1100 struct ftl_io *io = pin_ctx->cb_ctx; 1101 size_t i; 1102 1103 if (spdk_unlikely(status != 0)) { 1104 /* Retry on the internal L2P fault */ 1105 FTL_ERRLOG(dev, "Cannot PIN LBA for NV cache write failed at %"PRIx64"\n", 1106 io->addr); 1107 io->status = -EAGAIN; 1108 ftl_nv_cache_submit_cb_done(io); 1109 return; 1110 } 1111 1112 /* Remember previous l2p mapping to resolve conflicts in case of outstanding write-after-write */ 1113 for (i = 0; i < io->num_blocks; ++i) { 1114 io->map[i] = ftl_l2p_get(dev, ftl_io_get_lba(io, i)); 1115 } 1116 1117 assert(io->iov_pos == 0); 1118 1119 ftl_trace_submission(io->dev, io, io->addr, io->num_blocks); 1120 1121 nv_cache_write(io); 1122 } 1123 1124 bool 1125 ftl_nv_cache_write(struct ftl_io *io) 1126 { 1127 struct spdk_ftl_dev *dev = io->dev; 1128 uint64_t cache_offset; 1129 1130 io->md = ftl_mempool_get(dev->nv_cache.md_pool); 1131 if (spdk_unlikely(!io->md)) { 1132 return false; 1133 } 1134 1135 /* Reserve area on the write buffer cache */ 1136 cache_offset = ftl_nv_cache_get_wr_buffer(&dev->nv_cache, io); 1137 if (cache_offset == FTL_LBA_INVALID) { 1138 /* No free space in NV cache, resubmit request */ 1139 ftl_mempool_put(dev->nv_cache.md_pool, io->md); 1140 return false; 1141 } 1142 io->addr = ftl_addr_from_nvc_offset(dev, cache_offset); 1143 io->nv_cache_chunk = dev->nv_cache.chunk_current; 1144 1145 ftl_nv_cache_fill_md(io); 1146 ftl_l2p_pin(io->dev, io->lba, io->num_blocks, 1147 ftl_nv_cache_pin_cb, io, 1148 &io->l2p_pin_ctx); 1149 1150 dev->nv_cache.throttle.blocks_submitted += io->num_blocks; 1151 1152 return true; 1153 } 1154 1155 int 1156 ftl_nv_cache_read(struct ftl_io *io, ftl_addr addr, uint32_t num_blocks, 1157 spdk_bdev_io_completion_cb cb, void *cb_arg) 1158 { 1159 int rc; 1160 struct ftl_nv_cache *nv_cache = &io->dev->nv_cache; 1161 1162 assert(ftl_addr_in_nvc(io->dev, addr)); 1163 1164 rc = ftl_nv_cache_bdev_read_blocks_with_md(io->dev, nv_cache->bdev_desc, nv_cache->cache_ioch, 1165 ftl_io_iovec_addr(io), NULL, ftl_addr_to_nvc_offset(io->dev, addr), 1166 num_blocks, cb, cb_arg); 1167 1168 return rc; 1169 } 1170 1171 bool 1172 ftl_nv_cache_is_halted(struct ftl_nv_cache *nv_cache) 1173 { 1174 struct ftl_nv_cache_compactor *compactor; 1175 1176 if (nv_cache->compaction_active_count) { 1177 return false; 1178 } 1179 1180 TAILQ_FOREACH(compactor, &nv_cache->compactor_list, entry) { 1181 if (compactor->rd->iter.idx != 0 || compactor->wr->iter.idx != 0) { 1182 return false; 1183 } 1184 } 1185 1186 if (nv_cache->chunk_open_count > 0) { 1187 return false; 1188 } 1189 1190 return true; 1191 } 1192 1193 static void 1194 ftl_nv_cache_compaction_reset(struct ftl_nv_cache_compactor *compactor) 1195 { 1196 struct ftl_rq *rd = compactor->rd; 1197 struct ftl_rq *wr = compactor->wr; 1198 uint64_t lba; 1199 uint64_t i; 1200 1201 for (i = rd->iter.idx; i < rd->iter.count; i++) { 1202 lba = ((union ftl_md_vss *)rd->entries[i].io_md)->nv_cache.lba; 1203 if (lba != FTL_LBA_INVALID) { 1204 ftl_l2p_unpin(rd->dev, lba, 1); 1205 } 1206 } 1207 1208 rd->iter.idx = 0; 1209 rd->iter.count = 0; 1210 1211 for (i = 0; i < wr->iter.idx; i++) { 1212 lba = wr->entries[i].lba; 1213 assert(lba != FTL_LBA_INVALID); 1214 ftl_l2p_unpin(wr->dev, lba, 1); 1215 } 1216 1217 wr->iter.idx = 0; 1218 } 1219 1220 void 1221 ftl_chunk_map_set_lba(struct ftl_nv_cache_chunk *chunk, 1222 uint64_t offset, uint64_t lba) 1223 { 1224 struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(chunk->nv_cache, struct spdk_ftl_dev, nv_cache); 1225 struct ftl_p2l_map *p2l_map = &chunk->p2l_map; 1226 1227 ftl_lba_store(dev, p2l_map->chunk_map, offset, lba); 1228 } 1229 1230 uint64_t 1231 ftl_chunk_map_get_lba(struct ftl_nv_cache_chunk *chunk, uint64_t offset) 1232 { 1233 struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(chunk->nv_cache, struct spdk_ftl_dev, nv_cache); 1234 struct ftl_p2l_map *p2l_map = &chunk->p2l_map; 1235 1236 return ftl_lba_load(dev, p2l_map->chunk_map, offset); 1237 } 1238 1239 static void 1240 ftl_chunk_set_addr(struct ftl_nv_cache_chunk *chunk, uint64_t lba, ftl_addr addr) 1241 { 1242 struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(chunk->nv_cache, struct spdk_ftl_dev, nv_cache); 1243 uint64_t cache_offset = ftl_addr_to_nvc_offset(dev, addr); 1244 uint64_t offset; 1245 1246 offset = (cache_offset - chunk->offset) % chunk->nv_cache->chunk_blocks; 1247 ftl_chunk_map_set_lba(chunk, offset, lba); 1248 } 1249 1250 struct ftl_nv_cache_chunk * 1251 ftl_nv_cache_get_chunk_from_addr(struct spdk_ftl_dev *dev, ftl_addr addr) 1252 { 1253 struct ftl_nv_cache_chunk *chunk = dev->nv_cache.chunks; 1254 uint64_t chunk_idx; 1255 uint64_t cache_offset = ftl_addr_to_nvc_offset(dev, addr); 1256 1257 assert(chunk != NULL); 1258 chunk_idx = (cache_offset - chunk->offset) / chunk->nv_cache->chunk_blocks; 1259 chunk += chunk_idx; 1260 1261 return chunk; 1262 } 1263 1264 void 1265 ftl_nv_cache_set_addr(struct spdk_ftl_dev *dev, uint64_t lba, ftl_addr addr) 1266 { 1267 struct ftl_nv_cache_chunk *chunk; 1268 1269 chunk = ftl_nv_cache_get_chunk_from_addr(dev, addr); 1270 1271 assert(lba != FTL_LBA_INVALID); 1272 1273 ftl_chunk_set_addr(chunk, lba, addr); 1274 ftl_bitmap_set(dev->valid_map, addr); 1275 } 1276 1277 static void 1278 ftl_nv_cache_throttle_update(struct ftl_nv_cache *nv_cache) 1279 { 1280 double err; 1281 double modifier; 1282 1283 err = ((double)nv_cache->chunk_free_count - nv_cache->chunk_free_target) / nv_cache->chunk_count; 1284 modifier = FTL_NV_CACHE_THROTTLE_MODIFIER_KP * err; 1285 1286 if (modifier < FTL_NV_CACHE_THROTTLE_MODIFIER_MIN) { 1287 modifier = FTL_NV_CACHE_THROTTLE_MODIFIER_MIN; 1288 } else if (modifier > FTL_NV_CACHE_THROTTLE_MODIFIER_MAX) { 1289 modifier = FTL_NV_CACHE_THROTTLE_MODIFIER_MAX; 1290 } 1291 1292 if (spdk_unlikely(nv_cache->compaction_sma == 0 || nv_cache->compaction_active_count == 0)) { 1293 nv_cache->throttle.blocks_submitted_limit = UINT64_MAX; 1294 } else { 1295 double blocks_per_interval = nv_cache->compaction_sma * nv_cache->throttle.interval_tsc / 1296 FTL_BLOCK_SIZE; 1297 nv_cache->throttle.blocks_submitted_limit = blocks_per_interval * (1.0 + modifier); 1298 } 1299 } 1300 1301 static void 1302 ftl_nv_cache_process_throttle(struct ftl_nv_cache *nv_cache) 1303 { 1304 uint64_t tsc = spdk_thread_get_last_tsc(spdk_get_thread()); 1305 1306 if (spdk_unlikely(!nv_cache->throttle.start_tsc)) { 1307 nv_cache->throttle.start_tsc = tsc; 1308 } else if (tsc - nv_cache->throttle.start_tsc >= nv_cache->throttle.interval_tsc) { 1309 ftl_nv_cache_throttle_update(nv_cache); 1310 nv_cache->throttle.start_tsc = tsc; 1311 nv_cache->throttle.blocks_submitted = 0; 1312 } 1313 } 1314 1315 static void ftl_chunk_open(struct ftl_nv_cache_chunk *chunk); 1316 1317 void 1318 ftl_nv_cache_process(struct spdk_ftl_dev *dev) 1319 { 1320 struct ftl_nv_cache *nv_cache = &dev->nv_cache; 1321 1322 assert(dev->nv_cache.bdev_desc); 1323 1324 if (nv_cache->chunk_open_count < FTL_MAX_OPEN_CHUNKS && spdk_likely(!nv_cache->halt) && 1325 !TAILQ_EMPTY(&nv_cache->chunk_free_list)) { 1326 struct ftl_nv_cache_chunk *chunk = TAILQ_FIRST(&nv_cache->chunk_free_list); 1327 TAILQ_REMOVE(&nv_cache->chunk_free_list, chunk, entry); 1328 TAILQ_INSERT_TAIL(&nv_cache->chunk_open_list, chunk, entry); 1329 nv_cache->chunk_free_count--; 1330 chunk->md->seq_id = ftl_get_next_seq_id(dev); 1331 ftl_chunk_open(chunk); 1332 ftl_add_io_activity(dev); 1333 } 1334 1335 if (is_compaction_required(nv_cache) && !TAILQ_EMPTY(&nv_cache->compactor_list)) { 1336 struct ftl_nv_cache_compactor *comp = 1337 TAILQ_FIRST(&nv_cache->compactor_list); 1338 1339 TAILQ_REMOVE(&nv_cache->compactor_list, comp, entry); 1340 1341 compaction_process_start(comp); 1342 ftl_add_io_activity(dev); 1343 } 1344 1345 ftl_chunk_persist_free_state(nv_cache); 1346 1347 if (spdk_unlikely(nv_cache->halt)) { 1348 struct ftl_nv_cache_compactor *compactor; 1349 1350 TAILQ_FOREACH(compactor, &nv_cache->compactor_list, entry) { 1351 ftl_nv_cache_compaction_reset(compactor); 1352 } 1353 } 1354 1355 ftl_nv_cache_process_throttle(nv_cache); 1356 } 1357 1358 static bool 1359 ftl_nv_cache_full(struct ftl_nv_cache *nv_cache) 1360 { 1361 if (0 == nv_cache->chunk_open_count && NULL == nv_cache->chunk_current) { 1362 return true; 1363 } else { 1364 return false; 1365 } 1366 } 1367 1368 bool 1369 ftl_nv_cache_throttle(struct spdk_ftl_dev *dev) 1370 { 1371 struct ftl_nv_cache *nv_cache = &dev->nv_cache; 1372 1373 if (dev->nv_cache.throttle.blocks_submitted >= nv_cache->throttle.blocks_submitted_limit || 1374 ftl_nv_cache_full(nv_cache)) { 1375 return true; 1376 } 1377 1378 return false; 1379 } 1380 1381 static void 1382 chunk_free_p2l_map(struct ftl_nv_cache_chunk *chunk) 1383 { 1384 1385 struct ftl_nv_cache *nv_cache = chunk->nv_cache; 1386 struct ftl_p2l_map *p2l_map = &chunk->p2l_map; 1387 1388 ftl_mempool_put(nv_cache->p2l_pool, p2l_map->chunk_map); 1389 p2l_map->chunk_map = NULL; 1390 1391 ftl_chunk_free_md_entry(chunk); 1392 } 1393 1394 int 1395 ftl_nv_cache_save_state(struct ftl_nv_cache *nv_cache) 1396 { 1397 struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(nv_cache, struct spdk_ftl_dev, nv_cache); 1398 struct ftl_nv_cache_chunk *chunk; 1399 int status = 0; 1400 uint64_t i; 1401 1402 assert(nv_cache->chunk_open_count == 0); 1403 1404 if (nv_cache->compaction_active_count) { 1405 FTL_ERRLOG(dev, "Cannot save NV cache state, compaction in progress\n"); 1406 return -EINVAL; 1407 } 1408 1409 chunk = nv_cache->chunks; 1410 if (!chunk) { 1411 FTL_ERRLOG(dev, "Cannot save NV cache state, no NV cache metadata\n"); 1412 return -ENOMEM; 1413 } 1414 1415 for (i = 0; i < nv_cache->chunk_count; i++, chunk++) { 1416 nvc_validate_md(nv_cache, chunk->md); 1417 1418 if (chunk->md->read_pointer) { 1419 /* Only full chunks can be compacted */ 1420 if (chunk->md->blocks_written != nv_cache->chunk_blocks) { 1421 assert(0); 1422 status = -EINVAL; 1423 break; 1424 } 1425 1426 /* 1427 * Chunk in the middle of compaction, start over after 1428 * load 1429 */ 1430 chunk->md->read_pointer = chunk->md->blocks_compacted = 0; 1431 } else if (chunk->md->blocks_written == nv_cache->chunk_blocks) { 1432 /* Full chunk */ 1433 } else if (0 == chunk->md->blocks_written) { 1434 /* Empty chunk */ 1435 } else { 1436 assert(0); 1437 status = -EINVAL; 1438 break; 1439 } 1440 } 1441 1442 if (status) { 1443 FTL_ERRLOG(dev, "Cannot save NV cache state, inconsistent NV cache" 1444 "metadata\n"); 1445 } 1446 1447 return status; 1448 } 1449 1450 static int 1451 sort_chunks_cmp(const void *a, const void *b) 1452 { 1453 struct ftl_nv_cache_chunk *a_chunk = *(struct ftl_nv_cache_chunk **)a; 1454 struct ftl_nv_cache_chunk *b_chunk = *(struct ftl_nv_cache_chunk **)b; 1455 1456 return a_chunk->md->seq_id - b_chunk->md->seq_id; 1457 } 1458 1459 static int 1460 sort_chunks(struct ftl_nv_cache *nv_cache) 1461 { 1462 struct ftl_nv_cache_chunk **chunks_list; 1463 struct ftl_nv_cache_chunk *chunk; 1464 uint32_t i; 1465 1466 if (TAILQ_EMPTY(&nv_cache->chunk_full_list)) { 1467 return 0; 1468 } 1469 1470 chunks_list = calloc(nv_cache->chunk_full_count, 1471 sizeof(chunks_list[0])); 1472 if (!chunks_list) { 1473 return -ENOMEM; 1474 } 1475 1476 i = 0; 1477 TAILQ_FOREACH(chunk, &nv_cache->chunk_full_list, entry) { 1478 chunks_list[i] = chunk; 1479 i++; 1480 } 1481 assert(i == nv_cache->chunk_full_count); 1482 1483 qsort(chunks_list, nv_cache->chunk_full_count, sizeof(chunks_list[0]), 1484 sort_chunks_cmp); 1485 1486 TAILQ_INIT(&nv_cache->chunk_full_list); 1487 for (i = 0; i < nv_cache->chunk_full_count; i++) { 1488 chunk = chunks_list[i]; 1489 TAILQ_INSERT_TAIL(&nv_cache->chunk_full_list, chunk, entry); 1490 } 1491 1492 free(chunks_list); 1493 return 0; 1494 } 1495 1496 static int 1497 chunk_alloc_p2l_map(struct ftl_nv_cache_chunk *chunk) 1498 { 1499 struct ftl_nv_cache *nv_cache = chunk->nv_cache; 1500 struct ftl_p2l_map *p2l_map = &chunk->p2l_map; 1501 1502 assert(p2l_map->ref_cnt == 0); 1503 assert(p2l_map->chunk_map == NULL); 1504 1505 p2l_map->chunk_map = ftl_mempool_get(nv_cache->p2l_pool); 1506 1507 if (!p2l_map->chunk_map) { 1508 return -ENOMEM; 1509 } 1510 1511 if (ftl_chunk_alloc_md_entry(chunk)) { 1512 ftl_mempool_put(nv_cache->p2l_pool, p2l_map->chunk_map); 1513 p2l_map->chunk_map = NULL; 1514 return -ENOMEM; 1515 } 1516 1517 /* Set the P2L to FTL_LBA_INVALID */ 1518 memset(p2l_map->chunk_map, -1, FTL_BLOCK_SIZE * nv_cache->tail_md_chunk_blocks); 1519 1520 return 0; 1521 } 1522 1523 int 1524 ftl_nv_cache_load_state(struct ftl_nv_cache *nv_cache) 1525 { 1526 struct ftl_nv_cache_chunk *chunk; 1527 uint64_t chunks_number, offset, i; 1528 int status = 0; 1529 struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(nv_cache, struct spdk_ftl_dev, nv_cache); 1530 1531 nv_cache->chunk_current = NULL; 1532 TAILQ_INIT(&nv_cache->chunk_free_list); 1533 TAILQ_INIT(&nv_cache->chunk_full_list); 1534 nv_cache->chunk_full_count = nv_cache->chunk_free_count = 0; 1535 1536 assert(nv_cache->chunk_open_count == 0); 1537 offset = nvc_data_offset(nv_cache); 1538 chunk = nv_cache->chunks; 1539 if (!chunk) { 1540 FTL_ERRLOG(dev, "No NV cache metadata\n"); 1541 return -1; 1542 } 1543 1544 for (i = 0; i < nv_cache->chunk_count; i++, chunk++) { 1545 chunk->nv_cache = nv_cache; 1546 nvc_validate_md(nv_cache, chunk->md); 1547 1548 if (offset != chunk->offset) { 1549 status = -EINVAL; 1550 goto error; 1551 } 1552 1553 if (chunk->md->blocks_written == nv_cache->chunk_blocks) { 1554 /* Chunk full, move it on full list */ 1555 TAILQ_INSERT_TAIL(&nv_cache->chunk_full_list, chunk, entry); 1556 nv_cache->chunk_full_count++; 1557 } else if (0 == chunk->md->blocks_written) { 1558 /* Chunk empty, move it on empty list */ 1559 TAILQ_INSERT_TAIL(&nv_cache->chunk_free_list, chunk, entry); 1560 nv_cache->chunk_free_count++; 1561 } else { 1562 status = -EINVAL; 1563 goto error; 1564 } 1565 1566 offset += nv_cache->chunk_blocks; 1567 } 1568 1569 chunks_number = nv_cache->chunk_free_count + nv_cache->chunk_full_count; 1570 assert(nv_cache->chunk_current == NULL); 1571 1572 if (chunks_number != nv_cache->chunk_count) { 1573 FTL_ERRLOG(dev, "Inconsistent NV cache metadata\n"); 1574 status = -EINVAL; 1575 goto error; 1576 } 1577 1578 status = sort_chunks(nv_cache); 1579 if (status) { 1580 FTL_ERRLOG(dev, "FTL NV Cache: sorting chunks ERROR\n"); 1581 } 1582 1583 FTL_NOTICELOG(dev, "FTL NV Cache: full chunks = %lu, empty chunks = %lu\n", 1584 nv_cache->chunk_full_count, nv_cache->chunk_free_count); 1585 1586 if (0 == status) { 1587 FTL_NOTICELOG(dev, "FTL NV Cache: state loaded successfully\n"); 1588 } else { 1589 FTL_ERRLOG(dev, "FTL NV Cache: loading state ERROR\n"); 1590 } 1591 1592 error: 1593 return status; 1594 } 1595 1596 void 1597 ftl_nv_cache_get_max_seq_id(struct ftl_nv_cache *nv_cache, uint64_t *open_seq_id, 1598 uint64_t *close_seq_id) 1599 { 1600 uint64_t i, o_seq_id = 0, c_seq_id = 0; 1601 struct ftl_nv_cache_chunk *chunk; 1602 1603 chunk = nv_cache->chunks; 1604 assert(chunk); 1605 1606 /* Iterate over chunks and get their max open and close seq id */ 1607 for (i = 0; i < nv_cache->chunk_count; i++, chunk++) { 1608 o_seq_id = spdk_max(o_seq_id, chunk->md->seq_id); 1609 c_seq_id = spdk_max(c_seq_id, chunk->md->close_seq_id); 1610 } 1611 1612 *open_seq_id = o_seq_id; 1613 *close_seq_id = c_seq_id; 1614 } 1615 1616 typedef void (*ftl_chunk_ops_cb)(struct ftl_nv_cache_chunk *chunk, void *cntx, bool status); 1617 1618 static void 1619 write_brq_end(struct spdk_bdev_io *bdev_io, bool success, void *arg) 1620 { 1621 struct ftl_basic_rq *brq = arg; 1622 struct ftl_nv_cache_chunk *chunk = brq->io.chunk; 1623 1624 ftl_stats_bdev_io_completed(brq->dev, FTL_STATS_TYPE_MD_NV_CACHE, bdev_io); 1625 1626 brq->success = success; 1627 if (spdk_likely(success)) { 1628 chunk_advance_blocks(chunk->nv_cache, chunk, brq->num_blocks); 1629 } 1630 1631 spdk_bdev_free_io(bdev_io); 1632 brq->owner.cb(brq); 1633 } 1634 1635 static void 1636 _ftl_chunk_basic_rq_write(void *_brq) 1637 { 1638 struct ftl_basic_rq *brq = _brq; 1639 struct ftl_nv_cache *nv_cache = brq->io.chunk->nv_cache; 1640 struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(nv_cache, struct spdk_ftl_dev, nv_cache); 1641 int rc; 1642 1643 rc = ftl_nv_cache_bdev_write_blocks_with_md(dev, nv_cache->bdev_desc, nv_cache->cache_ioch, 1644 brq->io_payload, NULL, brq->io.addr, 1645 brq->num_blocks, write_brq_end, brq); 1646 if (spdk_unlikely(rc)) { 1647 if (rc == -ENOMEM) { 1648 struct spdk_bdev *bdev = spdk_bdev_desc_get_bdev(nv_cache->bdev_desc); 1649 brq->io.bdev_io_wait.bdev = bdev; 1650 brq->io.bdev_io_wait.cb_fn = _ftl_chunk_basic_rq_write; 1651 brq->io.bdev_io_wait.cb_arg = brq; 1652 spdk_bdev_queue_io_wait(bdev, nv_cache->cache_ioch, &brq->io.bdev_io_wait); 1653 } else { 1654 ftl_abort(); 1655 } 1656 } 1657 } 1658 1659 static void 1660 ftl_chunk_basic_rq_write(struct ftl_nv_cache_chunk *chunk, struct ftl_basic_rq *brq) 1661 { 1662 struct ftl_nv_cache *nv_cache = chunk->nv_cache; 1663 struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(nv_cache, struct spdk_ftl_dev, nv_cache); 1664 1665 brq->io.chunk = chunk; 1666 brq->success = false; 1667 1668 _ftl_chunk_basic_rq_write(brq); 1669 1670 chunk->md->write_pointer += brq->num_blocks; 1671 dev->stats.io_activity_total += brq->num_blocks; 1672 } 1673 1674 static void 1675 read_brq_end(struct spdk_bdev_io *bdev_io, bool success, void *arg) 1676 { 1677 struct ftl_basic_rq *brq = arg; 1678 1679 ftl_stats_bdev_io_completed(brq->dev, FTL_STATS_TYPE_MD_NV_CACHE, bdev_io); 1680 1681 brq->success = success; 1682 1683 brq->owner.cb(brq); 1684 spdk_bdev_free_io(bdev_io); 1685 } 1686 1687 static int 1688 ftl_chunk_basic_rq_read(struct ftl_nv_cache_chunk *chunk, struct ftl_basic_rq *brq) 1689 { 1690 struct ftl_nv_cache *nv_cache = chunk->nv_cache; 1691 struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(nv_cache, struct spdk_ftl_dev, nv_cache); 1692 int rc; 1693 1694 brq->io.chunk = chunk; 1695 brq->success = false; 1696 1697 rc = ftl_nv_cache_bdev_read_blocks_with_md(dev, nv_cache->bdev_desc, nv_cache->cache_ioch, 1698 brq->io_payload, NULL, brq->io.addr, brq->num_blocks, read_brq_end, brq); 1699 1700 if (spdk_likely(!rc)) { 1701 dev->stats.io_activity_total += brq->num_blocks; 1702 } 1703 1704 return rc; 1705 } 1706 1707 static void 1708 chunk_open_cb(int status, void *ctx) 1709 { 1710 struct ftl_nv_cache_chunk *chunk = (struct ftl_nv_cache_chunk *)ctx; 1711 1712 if (spdk_unlikely(status)) { 1713 #ifdef SPDK_FTL_RETRY_ON_ERROR 1714 ftl_md_persist_entry_retry(&chunk->md_persist_entry_ctx); 1715 return; 1716 #else 1717 ftl_abort(); 1718 #endif 1719 } 1720 1721 chunk->md->state = FTL_CHUNK_STATE_OPEN; 1722 } 1723 1724 static void 1725 ftl_chunk_open(struct ftl_nv_cache_chunk *chunk) 1726 { 1727 struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(chunk->nv_cache, struct spdk_ftl_dev, nv_cache); 1728 struct ftl_p2l_map *p2l_map = &chunk->p2l_map; 1729 struct ftl_layout_region *region = &dev->layout.region[FTL_LAYOUT_REGION_TYPE_NVC_MD]; 1730 struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_NVC_MD]; 1731 1732 if (chunk_alloc_p2l_map(chunk)) { 1733 assert(0); 1734 /* 1735 * We control number of opening chunk and it shall be consistent with size of chunk 1736 * P2L map pool 1737 */ 1738 ftl_abort(); 1739 return; 1740 } 1741 1742 chunk->nv_cache->chunk_open_count++; 1743 1744 assert(chunk->md->write_pointer == 0); 1745 assert(chunk->md->blocks_written == 0); 1746 1747 memcpy(p2l_map->chunk_dma_md, chunk->md, region->entry_size * FTL_BLOCK_SIZE); 1748 p2l_map->chunk_dma_md->state = FTL_CHUNK_STATE_OPEN; 1749 p2l_map->chunk_dma_md->p2l_map_checksum = 0; 1750 1751 ftl_md_persist_entry(md, get_chunk_idx(chunk), p2l_map->chunk_dma_md, 1752 NULL, chunk_open_cb, chunk, 1753 &chunk->md_persist_entry_ctx); 1754 } 1755 1756 static void 1757 chunk_close_cb(int status, void *ctx) 1758 { 1759 struct ftl_nv_cache_chunk *chunk = (struct ftl_nv_cache_chunk *)ctx; 1760 1761 assert(chunk->md->write_pointer == chunk->nv_cache->chunk_blocks); 1762 1763 if (spdk_likely(!status)) { 1764 chunk->md->p2l_map_checksum = chunk->p2l_map.chunk_dma_md->p2l_map_checksum; 1765 chunk_free_p2l_map(chunk); 1766 1767 assert(chunk->nv_cache->chunk_open_count > 0); 1768 chunk->nv_cache->chunk_open_count--; 1769 1770 /* Chunk full move it on full list */ 1771 TAILQ_INSERT_TAIL(&chunk->nv_cache->chunk_full_list, chunk, entry); 1772 chunk->nv_cache->chunk_full_count++; 1773 1774 chunk->nv_cache->last_seq_id = chunk->md->close_seq_id; 1775 1776 chunk->md->state = FTL_CHUNK_STATE_CLOSED; 1777 } else { 1778 #ifdef SPDK_FTL_RETRY_ON_ERROR 1779 ftl_md_persist_entry_retry(&chunk->md_persist_entry_ctx); 1780 #else 1781 ftl_abort(); 1782 #endif 1783 } 1784 } 1785 1786 static void 1787 chunk_map_write_cb(struct ftl_basic_rq *brq) 1788 { 1789 struct ftl_nv_cache_chunk *chunk = brq->io.chunk; 1790 struct ftl_p2l_map *p2l_map = &chunk->p2l_map; 1791 struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(chunk->nv_cache, struct spdk_ftl_dev, nv_cache); 1792 struct ftl_layout_region *region = &dev->layout.region[FTL_LAYOUT_REGION_TYPE_NVC_MD]; 1793 struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_NVC_MD]; 1794 uint32_t chunk_map_crc; 1795 1796 if (spdk_likely(brq->success)) { 1797 chunk_map_crc = spdk_crc32c_update(p2l_map->chunk_map, 1798 chunk->nv_cache->tail_md_chunk_blocks * FTL_BLOCK_SIZE, 0); 1799 memcpy(p2l_map->chunk_dma_md, chunk->md, region->entry_size * FTL_BLOCK_SIZE); 1800 p2l_map->chunk_dma_md->state = FTL_CHUNK_STATE_CLOSED; 1801 p2l_map->chunk_dma_md->p2l_map_checksum = chunk_map_crc; 1802 ftl_md_persist_entry(md, get_chunk_idx(chunk), chunk->p2l_map.chunk_dma_md, 1803 NULL, chunk_close_cb, chunk, 1804 &chunk->md_persist_entry_ctx); 1805 } else { 1806 #ifdef SPDK_FTL_RETRY_ON_ERROR 1807 /* retry */ 1808 chunk->md->write_pointer -= brq->num_blocks; 1809 ftl_chunk_basic_rq_write(chunk, brq); 1810 #else 1811 ftl_abort(); 1812 #endif 1813 } 1814 } 1815 1816 static void 1817 ftl_chunk_close(struct ftl_nv_cache_chunk *chunk) 1818 { 1819 struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(chunk->nv_cache, struct spdk_ftl_dev, nv_cache); 1820 struct ftl_basic_rq *brq = &chunk->metadata_rq; 1821 void *metadata = chunk->p2l_map.chunk_map; 1822 1823 chunk->md->close_seq_id = ftl_get_next_seq_id(dev); 1824 ftl_basic_rq_init(dev, brq, metadata, chunk->nv_cache->tail_md_chunk_blocks); 1825 ftl_basic_rq_set_owner(brq, chunk_map_write_cb, chunk); 1826 1827 assert(chunk->md->write_pointer == chunk_tail_md_offset(chunk->nv_cache)); 1828 brq->io.addr = chunk->offset + chunk->md->write_pointer; 1829 1830 ftl_chunk_basic_rq_write(chunk, brq); 1831 } 1832 1833 static int ftl_chunk_read_tail_md(struct ftl_nv_cache_chunk *chunk, struct ftl_basic_rq *brq, 1834 void (*cb)(struct ftl_basic_rq *brq), void *cb_ctx); 1835 static void read_tail_md_cb(struct ftl_basic_rq *brq); 1836 static void recover_open_chunk_cb(struct ftl_basic_rq *brq); 1837 1838 static void 1839 restore_chunk_close_cb(int status, void *ctx) 1840 { 1841 struct ftl_basic_rq *parent = (struct ftl_basic_rq *)ctx; 1842 struct ftl_nv_cache_chunk *chunk = parent->io.chunk; 1843 struct ftl_p2l_map *p2l_map = &chunk->p2l_map; 1844 1845 if (spdk_unlikely(status)) { 1846 parent->success = false; 1847 } else { 1848 chunk->md->p2l_map_checksum = p2l_map->chunk_dma_md->p2l_map_checksum; 1849 chunk->md->state = FTL_CHUNK_STATE_CLOSED; 1850 } 1851 1852 read_tail_md_cb(parent); 1853 } 1854 1855 static void 1856 restore_fill_p2l_map_cb(struct ftl_basic_rq *parent) 1857 { 1858 struct ftl_nv_cache_chunk *chunk = parent->io.chunk; 1859 struct ftl_p2l_map *p2l_map = &chunk->p2l_map; 1860 struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(chunk->nv_cache, struct spdk_ftl_dev, nv_cache); 1861 struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_NVC_MD]; 1862 struct ftl_layout_region *region = &dev->layout.region[FTL_LAYOUT_REGION_TYPE_NVC_MD]; 1863 uint32_t chunk_map_crc; 1864 1865 /* Set original callback */ 1866 ftl_basic_rq_set_owner(parent, recover_open_chunk_cb, parent->owner.priv); 1867 1868 if (spdk_unlikely(!parent->success)) { 1869 read_tail_md_cb(parent); 1870 return; 1871 } 1872 1873 chunk_map_crc = spdk_crc32c_update(p2l_map->chunk_map, 1874 chunk->nv_cache->tail_md_chunk_blocks * FTL_BLOCK_SIZE, 0); 1875 memcpy(p2l_map->chunk_dma_md, chunk->md, region->entry_size * FTL_BLOCK_SIZE); 1876 p2l_map->chunk_dma_md->state = FTL_CHUNK_STATE_CLOSED; 1877 p2l_map->chunk_dma_md->write_pointer = chunk->nv_cache->chunk_blocks; 1878 p2l_map->chunk_dma_md->blocks_written = chunk->nv_cache->chunk_blocks; 1879 p2l_map->chunk_dma_md->p2l_map_checksum = chunk_map_crc; 1880 1881 ftl_md_persist_entry(md, get_chunk_idx(chunk), p2l_map->chunk_dma_md, NULL, 1882 restore_chunk_close_cb, parent, &chunk->md_persist_entry_ctx); 1883 } 1884 1885 static void 1886 restore_fill_tail_md(struct ftl_basic_rq *parent, struct ftl_nv_cache_chunk *chunk) 1887 { 1888 struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(chunk->nv_cache, struct spdk_ftl_dev, nv_cache); 1889 void *metadata; 1890 1891 chunk->md->close_seq_id = ftl_get_next_seq_id(dev); 1892 1893 metadata = chunk->p2l_map.chunk_map; 1894 ftl_basic_rq_init(dev, parent, metadata, chunk->nv_cache->tail_md_chunk_blocks); 1895 ftl_basic_rq_set_owner(parent, restore_fill_p2l_map_cb, parent->owner.priv); 1896 1897 parent->io.addr = chunk->offset + chunk_tail_md_offset(chunk->nv_cache); 1898 parent->io.chunk = chunk; 1899 1900 ftl_chunk_basic_rq_write(chunk, parent); 1901 } 1902 1903 static void 1904 read_open_chunk_cb(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) 1905 { 1906 struct ftl_rq *rq = (struct ftl_rq *)cb_arg; 1907 struct ftl_basic_rq *parent = (struct ftl_basic_rq *)rq->owner.priv; 1908 struct ftl_nv_cache_chunk *chunk = parent->io.chunk; 1909 struct ftl_nv_cache *nv_cache = chunk->nv_cache; 1910 struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(chunk->nv_cache, struct spdk_ftl_dev, nv_cache); 1911 union ftl_md_vss *md; 1912 uint64_t cache_offset = bdev_io->u.bdev.offset_blocks; 1913 uint64_t len = bdev_io->u.bdev.num_blocks; 1914 ftl_addr addr = ftl_addr_from_nvc_offset(dev, cache_offset); 1915 int rc; 1916 1917 ftl_stats_bdev_io_completed(dev, FTL_STATS_TYPE_USER, bdev_io); 1918 1919 spdk_bdev_free_io(bdev_io); 1920 1921 if (!success) { 1922 parent->success = false; 1923 read_tail_md_cb(parent); 1924 return; 1925 } 1926 1927 while (rq->iter.idx < rq->iter.count) { 1928 /* Get metadata */ 1929 md = rq->entries[rq->iter.idx].io_md; 1930 if (md->nv_cache.seq_id != chunk->md->seq_id) { 1931 md->nv_cache.lba = FTL_LBA_INVALID; 1932 } 1933 /* 1934 * The p2l map contains effectively random data at this point (since it contains arbitrary 1935 * blocks from potentially not even filled tail md), so even LBA_INVALID needs to be set explicitly 1936 */ 1937 1938 ftl_chunk_set_addr(chunk, md->nv_cache.lba, addr + rq->iter.idx); 1939 rq->iter.idx++; 1940 } 1941 1942 if (cache_offset + len < chunk->offset + chunk_tail_md_offset(nv_cache)) { 1943 cache_offset += len; 1944 len = spdk_min(dev->xfer_size, chunk->offset + chunk_tail_md_offset(nv_cache) - cache_offset); 1945 rq->iter.idx = 0; 1946 rq->iter.count = len; 1947 1948 rc = ftl_nv_cache_bdev_readv_blocks_with_md(dev, nv_cache->bdev_desc, 1949 nv_cache->cache_ioch, 1950 rq->io_vec, len, 1951 rq->io_md, 1952 cache_offset, len, 1953 read_open_chunk_cb, 1954 rq); 1955 1956 if (rc) { 1957 ftl_rq_del(rq); 1958 parent->success = false; 1959 read_tail_md_cb(parent); 1960 return; 1961 } 1962 } else { 1963 ftl_rq_del(rq); 1964 restore_fill_tail_md(parent, chunk); 1965 } 1966 } 1967 1968 static void 1969 restore_open_chunk(struct ftl_nv_cache_chunk *chunk, struct ftl_basic_rq *parent) 1970 { 1971 struct ftl_nv_cache *nv_cache = chunk->nv_cache; 1972 struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(nv_cache, struct spdk_ftl_dev, nv_cache); 1973 struct ftl_rq *rq; 1974 uint64_t addr; 1975 uint64_t len = dev->xfer_size; 1976 int rc; 1977 1978 /* 1979 * We've just read the p2l map, prefill it with INVALID LBA 1980 * TODO we need to do this because tail md blocks (p2l map) are also represented in the p2l map, instead of just user data region 1981 */ 1982 memset(chunk->p2l_map.chunk_map, -1, FTL_BLOCK_SIZE * nv_cache->tail_md_chunk_blocks); 1983 1984 /* Need to read user data, recalculate chunk's P2L and write tail md with it */ 1985 rq = ftl_rq_new(dev, dev->nv_cache.md_size); 1986 if (!rq) { 1987 parent->success = false; 1988 read_tail_md_cb(parent); 1989 return; 1990 } 1991 1992 rq->owner.priv = parent; 1993 rq->iter.idx = 0; 1994 rq->iter.count = len; 1995 1996 addr = chunk->offset; 1997 1998 len = spdk_min(dev->xfer_size, chunk->offset + chunk_tail_md_offset(nv_cache) - addr); 1999 2000 rc = ftl_nv_cache_bdev_readv_blocks_with_md(dev, nv_cache->bdev_desc, 2001 nv_cache->cache_ioch, 2002 rq->io_vec, len, 2003 rq->io_md, 2004 addr, len, 2005 read_open_chunk_cb, 2006 rq); 2007 2008 if (rc) { 2009 ftl_rq_del(rq); 2010 parent->success = false; 2011 read_tail_md_cb(parent); 2012 } 2013 } 2014 2015 static void 2016 read_tail_md_cb(struct ftl_basic_rq *brq) 2017 { 2018 brq->owner.cb(brq); 2019 } 2020 2021 static int 2022 ftl_chunk_read_tail_md(struct ftl_nv_cache_chunk *chunk, struct ftl_basic_rq *brq, 2023 void (*cb)(struct ftl_basic_rq *brq), void *cb_ctx) 2024 { 2025 struct spdk_ftl_dev *dev = SPDK_CONTAINEROF(chunk->nv_cache, struct spdk_ftl_dev, nv_cache); 2026 void *metadata; 2027 int rc; 2028 2029 metadata = chunk->p2l_map.chunk_map; 2030 ftl_basic_rq_init(dev, brq, metadata, chunk->nv_cache->tail_md_chunk_blocks); 2031 ftl_basic_rq_set_owner(brq, cb, cb_ctx); 2032 2033 brq->io.addr = chunk->offset + chunk_tail_md_offset(chunk->nv_cache); 2034 rc = ftl_chunk_basic_rq_read(chunk, brq); 2035 2036 return rc; 2037 } 2038 2039 struct restore_chunk_md_ctx { 2040 ftl_chunk_md_cb cb; 2041 void *cb_ctx; 2042 int status; 2043 uint64_t qd; 2044 uint64_t id; 2045 }; 2046 2047 static inline bool 2048 is_chunk_count_valid(struct ftl_nv_cache *nv_cache) 2049 { 2050 uint64_t chunk_count = 0; 2051 2052 chunk_count += nv_cache->chunk_open_count; 2053 chunk_count += nv_cache->chunk_free_count; 2054 chunk_count += nv_cache->chunk_full_count; 2055 chunk_count += nv_cache->chunk_comp_count; 2056 2057 return chunk_count == nv_cache->chunk_count; 2058 } 2059 2060 static void 2061 walk_tail_md_cb(struct ftl_basic_rq *brq) 2062 { 2063 struct ftl_mngt_process *mngt = brq->owner.priv; 2064 struct ftl_nv_cache_chunk *chunk = brq->io.chunk; 2065 struct restore_chunk_md_ctx *ctx = ftl_mngt_get_step_ctx(mngt); 2066 int rc = 0; 2067 2068 if (brq->success) { 2069 rc = ctx->cb(chunk, ctx->cb_ctx); 2070 } else { 2071 rc = -EIO; 2072 } 2073 2074 if (rc) { 2075 ctx->status = rc; 2076 } 2077 ctx->qd--; 2078 chunk_free_p2l_map(chunk); 2079 ftl_mngt_continue_step(mngt); 2080 } 2081 2082 static void 2083 ftl_mngt_nv_cache_walk_tail_md(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt, 2084 uint64_t seq_id, ftl_chunk_md_cb cb, void *cb_ctx) 2085 { 2086 struct ftl_nv_cache *nvc = &dev->nv_cache; 2087 struct restore_chunk_md_ctx *ctx; 2088 2089 ctx = ftl_mngt_get_step_ctx(mngt); 2090 if (!ctx) { 2091 if (ftl_mngt_alloc_step_ctx(mngt, sizeof(*ctx))) { 2092 ftl_mngt_fail_step(mngt); 2093 return; 2094 } 2095 ctx = ftl_mngt_get_step_ctx(mngt); 2096 assert(ctx); 2097 2098 ctx->cb = cb; 2099 ctx->cb_ctx = cb_ctx; 2100 } 2101 2102 /* 2103 * This function generates a high queue depth and will utilize ftl_mngt_continue_step during completions to make sure all chunks 2104 * are processed before returning an error (if any were found) or continuing on. 2105 */ 2106 if (0 == ctx->qd && ctx->id == nvc->chunk_count) { 2107 if (!is_chunk_count_valid(nvc)) { 2108 FTL_ERRLOG(dev, "Recovery ERROR, invalid number of chunk\n"); 2109 assert(false); 2110 ctx->status = -EINVAL; 2111 } 2112 2113 if (ctx->status) { 2114 ftl_mngt_fail_step(mngt); 2115 } else { 2116 ftl_mngt_next_step(mngt); 2117 } 2118 return; 2119 } 2120 2121 while (ctx->id < nvc->chunk_count) { 2122 struct ftl_nv_cache_chunk *chunk = &nvc->chunks[ctx->id]; 2123 int rc; 2124 2125 if (!chunk->recovery) { 2126 /* This chunk is empty and not used in recovery */ 2127 ctx->id++; 2128 continue; 2129 } 2130 2131 if (seq_id && (chunk->md->close_seq_id <= seq_id)) { 2132 ctx->id++; 2133 continue; 2134 } 2135 2136 if (chunk_alloc_p2l_map(chunk)) { 2137 /* No more free P2L map, break and continue later */ 2138 break; 2139 } 2140 ctx->id++; 2141 2142 rc = ftl_chunk_read_tail_md(chunk, &chunk->metadata_rq, walk_tail_md_cb, mngt); 2143 2144 if (0 == rc) { 2145 ctx->qd++; 2146 } else { 2147 chunk_free_p2l_map(chunk); 2148 ctx->status = rc; 2149 } 2150 } 2151 2152 if (0 == ctx->qd) { 2153 /* 2154 * No QD could happen due to all leftover chunks being in free state. 2155 * Additionally ftl_chunk_read_tail_md could fail starting with the first IO in a given patch. 2156 * For streamlining of all potential error handling (since many chunks are reading P2L at the same time), 2157 * we're using ftl_mngt_continue_step to arrive at the same spot of checking for mngt step end (see beginning of function). 2158 */ 2159 ftl_mngt_continue_step(mngt); 2160 } 2161 2162 } 2163 2164 void 2165 ftl_mngt_nv_cache_restore_l2p(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt, 2166 ftl_chunk_md_cb cb, void *cb_ctx) 2167 { 2168 ftl_mngt_nv_cache_walk_tail_md(dev, mngt, dev->sb->ckpt_seq_id, cb, cb_ctx); 2169 } 2170 2171 static void 2172 restore_chunk_state_cb(struct spdk_ftl_dev *dev, struct ftl_md *md, int status) 2173 { 2174 struct ftl_mngt_process *mngt = md->owner.cb_ctx; 2175 struct ftl_nv_cache *nvc = &dev->nv_cache; 2176 struct ftl_nv_cache_chunk *chunk; 2177 uint64_t i; 2178 2179 if (status) { 2180 /* Restore error, end step */ 2181 ftl_mngt_fail_step(mngt); 2182 return; 2183 } 2184 2185 for (i = 0; i < nvc->chunk_count; i++) { 2186 chunk = &nvc->chunks[i]; 2187 2188 switch (chunk->md->state) { 2189 case FTL_CHUNK_STATE_FREE: 2190 break; 2191 case FTL_CHUNK_STATE_OPEN: 2192 TAILQ_REMOVE(&nvc->chunk_free_list, chunk, entry); 2193 nvc->chunk_free_count--; 2194 2195 TAILQ_INSERT_TAIL(&nvc->chunk_open_list, chunk, entry); 2196 nvc->chunk_open_count++; 2197 2198 /* Chunk is not empty, mark it to be recovered */ 2199 chunk->recovery = true; 2200 break; 2201 case FTL_CHUNK_STATE_CLOSED: 2202 TAILQ_REMOVE(&nvc->chunk_free_list, chunk, entry); 2203 nvc->chunk_free_count--; 2204 2205 TAILQ_INSERT_TAIL(&nvc->chunk_full_list, chunk, entry); 2206 nvc->chunk_full_count++; 2207 2208 /* Chunk is not empty, mark it to be recovered */ 2209 chunk->recovery = true; 2210 break; 2211 default: 2212 status = -EINVAL; 2213 } 2214 } 2215 2216 if (status) { 2217 ftl_mngt_fail_step(mngt); 2218 } else { 2219 ftl_mngt_next_step(mngt); 2220 } 2221 } 2222 2223 void 2224 ftl_mngt_nv_cache_restore_chunk_state(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 2225 { 2226 struct ftl_md *md = dev->layout.md[FTL_LAYOUT_REGION_TYPE_NVC_MD]; 2227 2228 md->owner.cb_ctx = mngt; 2229 md->cb = restore_chunk_state_cb; 2230 ftl_md_restore(md); 2231 } 2232 2233 static void 2234 recover_open_chunk_cb(struct ftl_basic_rq *brq) 2235 { 2236 struct ftl_mngt_process *mngt = brq->owner.priv; 2237 struct ftl_nv_cache_chunk *chunk = brq->io.chunk; 2238 struct ftl_nv_cache *nvc = chunk->nv_cache; 2239 struct spdk_ftl_dev *dev = ftl_mngt_get_dev(mngt); 2240 2241 chunk_free_p2l_map(chunk); 2242 2243 if (!brq->success) { 2244 FTL_ERRLOG(dev, "Recovery chunk ERROR, offset = %"PRIu64", seq id %"PRIu64"\n", chunk->offset, 2245 chunk->md->seq_id); 2246 ftl_mngt_fail_step(mngt); 2247 return; 2248 } 2249 2250 FTL_NOTICELOG(dev, "Recovered chunk, offset = %"PRIu64", seq id %"PRIu64"\n", chunk->offset, 2251 chunk->md->seq_id); 2252 2253 TAILQ_REMOVE(&nvc->chunk_open_list, chunk, entry); 2254 nvc->chunk_open_count--; 2255 2256 TAILQ_INSERT_TAIL(&nvc->chunk_full_list, chunk, entry); 2257 nvc->chunk_full_count++; 2258 2259 /* This is closed chunk */ 2260 chunk->md->write_pointer = nvc->chunk_blocks; 2261 chunk->md->blocks_written = nvc->chunk_blocks; 2262 2263 ftl_mngt_continue_step(mngt); 2264 } 2265 2266 void 2267 ftl_mngt_nv_cache_recover_open_chunk(struct spdk_ftl_dev *dev, struct ftl_mngt_process *mngt) 2268 { 2269 struct ftl_nv_cache *nvc = &dev->nv_cache; 2270 struct ftl_nv_cache_chunk *chunk; 2271 struct ftl_basic_rq *brq = ftl_mngt_get_step_ctx(mngt); 2272 2273 if (!brq) { 2274 if (TAILQ_EMPTY(&nvc->chunk_open_list)) { 2275 FTL_NOTICELOG(dev, "No open chunks to recover P2L\n"); 2276 ftl_mngt_next_step(mngt); 2277 return; 2278 } 2279 2280 if (ftl_mngt_alloc_step_ctx(mngt, sizeof(*brq))) { 2281 ftl_mngt_fail_step(mngt); 2282 return; 2283 } 2284 brq = ftl_mngt_get_step_ctx(mngt); 2285 ftl_basic_rq_set_owner(brq, recover_open_chunk_cb, mngt); 2286 } 2287 2288 if (TAILQ_EMPTY(&nvc->chunk_open_list)) { 2289 if (!is_chunk_count_valid(nvc)) { 2290 FTL_ERRLOG(dev, "Recovery ERROR, invalid number of chunk\n"); 2291 ftl_mngt_fail_step(mngt); 2292 return; 2293 } 2294 2295 /* 2296 * Now all chunks loaded and closed, do final step of restoring 2297 * chunks state 2298 */ 2299 if (ftl_nv_cache_load_state(nvc)) { 2300 ftl_mngt_fail_step(mngt); 2301 } else { 2302 ftl_mngt_next_step(mngt); 2303 } 2304 } else { 2305 chunk = TAILQ_FIRST(&nvc->chunk_open_list); 2306 if (chunk_alloc_p2l_map(chunk)) { 2307 ftl_mngt_fail_step(mngt); 2308 return; 2309 } 2310 2311 brq->io.chunk = chunk; 2312 2313 FTL_NOTICELOG(dev, "Start recovery open chunk, offset = %"PRIu64", seq id %"PRIu64"\n", 2314 chunk->offset, chunk->md->seq_id); 2315 restore_open_chunk(chunk, brq); 2316 } 2317 } 2318 2319 int 2320 ftl_nv_cache_chunks_busy(struct ftl_nv_cache *nv_cache) 2321 { 2322 /* chunk_current is migrating to closed status when closing, any others should already be 2323 * moved to free chunk list. Also need to wait for free md requests */ 2324 return nv_cache->chunk_open_count == 0 && nv_cache->chunk_free_persist_count == 0; 2325 } 2326 2327 void 2328 ftl_nv_cache_halt(struct ftl_nv_cache *nv_cache) 2329 { 2330 struct ftl_nv_cache_chunk *chunk; 2331 uint64_t free_space; 2332 2333 nv_cache->halt = true; 2334 2335 /* Set chunks on open list back to free state since no user data has been written to it */ 2336 while (!TAILQ_EMPTY(&nv_cache->chunk_open_list)) { 2337 chunk = TAILQ_FIRST(&nv_cache->chunk_open_list); 2338 2339 /* Chunks are moved between lists on metadata update submission, but state is changed 2340 * on completion. Breaking early in such a case to make sure all the necessary resources 2341 * will be freed (during next pass(es) of ftl_nv_cache_halt). 2342 */ 2343 if (chunk->md->state != FTL_CHUNK_STATE_OPEN) { 2344 break; 2345 } 2346 2347 TAILQ_REMOVE(&nv_cache->chunk_open_list, chunk, entry); 2348 chunk_free_p2l_map(chunk); 2349 memset(chunk->md, 0, sizeof(*chunk->md)); 2350 assert(nv_cache->chunk_open_count > 0); 2351 nv_cache->chunk_open_count--; 2352 } 2353 2354 /* Close current chunk by skipping all not written blocks */ 2355 chunk = nv_cache->chunk_current; 2356 if (chunk != NULL) { 2357 nv_cache->chunk_current = NULL; 2358 if (chunk_is_closed(chunk)) { 2359 return; 2360 } 2361 2362 free_space = chunk_get_free_space(nv_cache, chunk); 2363 chunk->md->blocks_skipped = free_space; 2364 chunk->md->blocks_written += free_space; 2365 chunk->md->write_pointer += free_space; 2366 ftl_chunk_close(chunk); 2367 } 2368 } 2369 2370 uint64_t 2371 ftl_nv_cache_acquire_trim_seq_id(struct ftl_nv_cache *nv_cache) 2372 { 2373 struct ftl_nv_cache_chunk *chunk = nv_cache->chunk_current; 2374 uint64_t seq_id, free_space; 2375 2376 if (!chunk) { 2377 chunk = TAILQ_FIRST(&nv_cache->chunk_open_list); 2378 if (chunk && chunk->md->state == FTL_CHUNK_STATE_OPEN) { 2379 return chunk->md->seq_id; 2380 } else { 2381 return 0; 2382 } 2383 } 2384 2385 if (chunk_is_closed(chunk)) { 2386 return 0; 2387 } 2388 2389 seq_id = nv_cache->chunk_current->md->seq_id; 2390 free_space = chunk_get_free_space(nv_cache, chunk); 2391 2392 chunk->md->blocks_skipped = free_space; 2393 chunk->md->blocks_written += free_space; 2394 chunk->md->write_pointer += free_space; 2395 if (chunk->md->blocks_written == chunk_tail_md_offset(nv_cache)) { 2396 ftl_chunk_close(chunk); 2397 } 2398 nv_cache->chunk_current = NULL; 2399 2400 seq_id++; 2401 return seq_id; 2402 } 2403