Lines Matching defs:cache
25 struct ftl_l2p_cache *cache;
138 /* MD layout cache: Offset on a device in FTL_BLOCK_SIZE unit */
141 /* MD layout cache: Device of region */
144 /* MD layout cache: IO channel of region */
148 typedef void (*ftl_l2p_cache_clear_cb)(struct ftl_l2p_cache *cache, int status, void *ctx_page);
149 typedef void (*ftl_l2p_cache_persist_cb)(struct ftl_l2p_cache *cache, int status, void *ctx_page);
154 static void page_set_end(struct spdk_ftl_dev *dev, struct ftl_l2p_cache *cache,
173 ftl_l2p_cache_get_lbas_in_page(struct ftl_l2p_cache *cache)
175 return cache->lbas_in_page;
185 ftl_l2p_cache_lru_remove_page(struct ftl_l2p_cache *cache, struct ftl_l2p_page *page)
190 TAILQ_REMOVE(&cache->lru_list, page, list_entry);
195 ftl_l2p_cache_lru_add_page(struct ftl_l2p_cache *cache, struct ftl_l2p_page *page)
200 TAILQ_INSERT_HEAD(&cache->lru_list, page, list_entry);
206 ftl_l2p_cache_lru_promote_page(struct ftl_l2p_cache *cache, struct ftl_l2p_page *page)
212 ftl_l2p_cache_lru_remove_page(cache, page);
213 ftl_l2p_cache_lru_add_page(cache, page);
217 ftl_l2p_cache_page_insert(struct ftl_l2p_cache *cache, struct ftl_l2p_page *page)
219 struct ftl_l2p_l1_map_entry *me = cache->l2_mapping;
227 ftl_l2p_cache_page_remove(struct ftl_l2p_cache *cache, struct ftl_l2p_page *page)
229 struct ftl_l2p_l1_map_entry *me = cache->l2_mapping;
235 cache->l2_pgs_avail++;
236 ftl_mempool_put(cache->l2_ctx_pool, page);
240 ftl_l2p_cache_get_coldest_page(struct ftl_l2p_cache *cache)
242 return TAILQ_LAST(&cache->lru_list, l2p_lru_list);
252 ftl_l2p_cache_page_get_bdev_offset(struct ftl_l2p_cache *cache,
255 return cache->cache_layout_offset + page->page_no;
259 ftl_l2p_cache_get_bdev_desc(struct ftl_l2p_cache *cache)
261 return cache->cache_layout_bdev_desc;
265 ftl_l2p_cache_get_bdev_iochannel(struct ftl_l2p_cache *cache)
267 return cache->cache_layout_ioch;
271 ftl_l2p_cache_page_alloc(struct ftl_l2p_cache *cache, size_t page_no)
273 struct ftl_l2p_page *page = ftl_mempool_get(cache->l2_ctx_pool);
276 cache->l2_pgs_avail--;
280 page->obj_id = ftl_mempool_get_df_obj_id(cache->l2_ctx_pool, page);
282 page->page_buffer = (char *)ftl_md_get_buffer(cache->l1_md) + ftl_mempool_get_df_obj_index(
283 cache->l2_ctx_pool, page) * FTL_BLOCK_SIZE;
303 struct ftl_l2p_cache *cache, struct ftl_l2p_page *page, uint64_t lba)
305 return ftl_addr_load(dev, page->page_buffer, lba % cache->lbas_in_page);
309 ftl_l2p_cache_set_addr(struct spdk_ftl_dev *dev, struct ftl_l2p_cache *cache,
312 ftl_addr_store(dev, page->page_buffer, lba % cache->lbas_in_page, addr);
319 struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)dev->l2p;
324 naddr = ftl_l2p_cache_get_lbas_in_page(cache);
332 ftl_l2p_cache_set_addr(dev, cache, page, i, FTL_ADDR_INVALID);
337 ftl_l2p_cache_page_pin(struct ftl_l2p_cache *cache, struct ftl_l2p_page *page)
342 ftl_l2p_cache_lru_remove_page(cache, page);
347 ftl_l2p_cache_page_unpin(struct ftl_l2p_cache *cache, struct ftl_l2p_page *page)
361 ftl_l2p_cache_lru_add_page(cache, page);
375 ftl_l2p_cache_evict_continue(struct ftl_l2p_cache *cache)
377 return cache->l2_pgs_avail + cache->l2_pgs_evicting < cache->evict_keep;
383 struct ftl_l2p_cache *cache;
387 cache = calloc(1, sizeof(struct ftl_l2p_cache));
388 if (cache == NULL) {
391 cache->dev = dev;
393 cache->l2_md = ftl_md_create(dev,
398 if (cache->l2_md == NULL) {
401 cache->l2_mapping = ftl_md_get_buffer(cache->l2_md);
403 cache->lbas_in_page = dev->layout.l2p.lbas_in_page;
404 cache->num_pages = l2_pages;
406 return cache;
408 free(cache);
413 get_l2p_page_by_df_id(struct ftl_l2p_cache *cache, size_t page_no)
415 struct ftl_l2p_l1_map_entry *me = cache->l2_mapping;
419 return ftl_mempool_get_df_ptr(cache->l2_ctx_pool, obj_id);
429 struct ftl_l2p_cache *cache;
440 cache = (struct ftl_l2p_cache *)dev->l2p;
441 cache->page_sets_pool = ftl_mempool_create(page_sets_pool_size,
444 if (!cache->page_sets_pool) {
451 if (max_resident_pgs > cache->num_pages) {
453 max_resident_pgs = cache->num_pages;
461 TAILQ_INIT(&cache->deferred_page_set_list);
462 TAILQ_INIT(&cache->lru_list);
464 cache->l2_ctx_md = ftl_md_create(dev,
468 if (cache->l2_ctx_md == NULL) {
472 cache->l2_pgs_resident_max = max_resident_pgs;
473 cache->l2_pgs_avail = max_resident_pgs;
474 cache->l2_pgs_evicting = 0;
475 cache->l2_ctx_pool = ftl_mempool_create_ext(ftl_md_get_buffer(cache->l2_ctx_md),
478 if (cache->l2_ctx_pool == NULL) {
484 cache->evict_keep = spdk_divide_round_up(cache->num_pages * FTL_L2P_CACHE_PAGE_AVAIL_RATIO, 100);
485 cache->evict_keep = spdk_min(FTL_L2P_CACHE_PAGE_AVAIL_MAX, cache->evict_keep);
488 memset(cache->l2_mapping, (int)FTL_DF_OBJ_ID_INVALID, ftl_md_get_buffer_size(cache->l2_md));
489 ftl_mempool_initialize_ext(cache->l2_ctx_pool);
492 cache->l1_md = ftl_md_create(dev,
497 if (cache->l1_md == NULL) {
503 cache->cache_layout_offset = reg->current.offset;
504 cache->cache_layout_bdev_desc = reg->bdev_desc;
505 cache->cache_layout_ioch = reg->ioch;
511 ftl_l2p_cache_deinit_l2(struct spdk_ftl_dev *dev, struct ftl_l2p_cache *cache)
513 ftl_md_destroy(cache->l2_ctx_md, ftl_md_destroy_shm_flags(dev));
514 cache->l2_ctx_md = NULL;
516 ftl_mempool_destroy_ext(cache->l2_ctx_pool);
517 cache->l2_ctx_pool = NULL;
519 ftl_md_destroy(cache->l1_md, ftl_md_destroy_shm_flags(dev));
520 cache->l1_md = NULL;
522 ftl_mempool_destroy(cache->page_sets_pool);
523 cache->page_sets_pool = NULL;
529 struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)dev->l2p;
531 ftl_l2p_cache_deinit_l2(dev, cache);
532 ftl_md_destroy(cache->l2_md, ftl_md_destroy_shm_flags(dev));
533 free(cache);
539 struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)dev->l2p;
541 if (!cache) {
544 assert(cache->state == L2P_CACHE_SHUTDOWN_DONE || cache->state == L2P_CACHE_INIT);
551 process_init_ctx(struct spdk_ftl_dev *dev, struct ftl_l2p_cache *cache,
554 struct ftl_l2p_cache_process_ctx *ctx = &cache->mctx;
557 assert(0 == cache->l2_pgs_evicting);
566 process_finish(struct ftl_l2p_cache *cache)
568 struct ftl_l2p_cache_process_ctx ctx = cache->mctx;
570 assert(cache->l2_pgs_avail == cache->l2_pgs_resident_max);
573 memset(&cache->mctx, 0, sizeof(cache->mctx));
574 ctx.cb(cache->dev, ctx.status, ctx.cb_ctx);
578 static void process_persist(struct ftl_l2p_cache *cache);
583 struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)page->ctx.cache;
588 rc = ftl_nv_cache_bdev_read_blocks_with_md(cache->dev, ftl_l2p_cache_get_bdev_desc(cache),
589 ftl_l2p_cache_get_bdev_iochannel(cache),
590 page->page_buffer, NULL, ftl_l2p_cache_page_get_bdev_offset(cache, page),
602 struct ftl_l2p_cache *cache = page->ctx.cache;
603 struct spdk_ftl_dev *dev = cache->dev;
604 struct ftl_l2p_cache_process_ctx *ctx = &cache->mctx;
621 ftl_l2p_cache_page_remove(cache, page);
624 process_persist(cache);
632 struct ftl_l2p_cache *cache = page->ctx.cache;
633 struct spdk_ftl_dev *dev = cache->dev;
638 rc = ftl_nv_cache_bdev_write_blocks_with_md(dev, ftl_l2p_cache_get_bdev_desc(cache),
639 ftl_l2p_cache_get_bdev_iochannel(cache),
640 page->page_buffer, NULL, ftl_l2p_cache_page_get_bdev_offset(cache, page),
648 bdev = spdk_bdev_desc_get_bdev(ftl_l2p_cache_get_bdev_desc(cache));
655 rc = spdk_bdev_queue_io_wait(bdev, ftl_l2p_cache_get_bdev_iochannel(cache), bdev_io_wait);
670 static void process_trim(struct ftl_l2p_cache *cache);
676 struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)page->ctx.cache;
677 struct spdk_ftl_dev *dev = cache->dev;
678 struct ftl_l2p_cache_process_ctx *ctx = &cache->mctx;
691 ftl_l2p_cache_page_remove(cache, page);
694 process_trim(cache);
701 struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)page->ctx.cache;
702 struct spdk_ftl_dev *dev = cache->dev;
703 struct ftl_l2p_cache_process_ctx *ctx = &cache->mctx;
716 process_trim(cache);
721 process_trim(struct ftl_l2p_cache *cache)
723 struct ftl_l2p_cache_process_ctx *ctx = &cache->mctx;
725 while (ctx->idx < cache->num_pages && ctx->qd < 64) {
728 if (!ftl_bitmap_get(cache->dev->trim_map, ctx->idx)) {
735 assert(get_l2p_page_by_df_id(cache, ctx->idx) == NULL);
738 page = ftl_l2p_cache_page_alloc(cache, ctx->idx);
746 page->ctx.cache = cache;
748 ftl_l2p_cache_page_insert(cache, page);
756 process_finish(cache);
763 struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)dev->l2p;
765 process_init_ctx(dev, cache, cb, cb_ctx);
766 process_trim(cache);
794 struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)dev->l2p;
795 struct ftl_l2p_l1_map_entry *me = cache->l2_mapping;
800 for (page_no = 0; page_no < cache->num_pages; ++page_no) {
806 page = ftl_mempool_claim_df(cache->l2_ctx_pool, obj_id);
808 assert(page->obj_id == ftl_mempool_get_df_obj_id(cache->l2_ctx_pool, page));
812 assert(cache->l2_pgs_avail > 0);
813 cache->l2_pgs_avail--;
815 page->page_buffer = (char *)ftl_md_get_buffer(cache->l1_md) + ftl_mempool_get_df_obj_index(
816 cache->l2_ctx_pool, page) * FTL_BLOCK_SIZE;
824 ftl_l2p_cache_lru_add_page(cache, page);
827 ftl_mempool_initialize_ext(cache->l2_ctx_pool);
833 struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)dev->l2p;
834 struct ftl_l2p_l1_map_entry *me = cache->l2_mapping;
839 for (page_no = 0; page_no < cache->num_pages; ++page_no) {
845 page = ftl_mempool_claim_df(cache->l2_ctx_pool, obj_id);
847 assert(page->obj_id == ftl_mempool_get_df_obj_id(cache->l2_ctx_pool, page));
850 assert(cache->l2_pgs_avail > 0);
851 cache->l2_pgs_avail--;
855 cache->l2_pgs_avail++;
856 ftl_mempool_release_df(cache->l2_ctx_pool, obj_id);
863 page->page_buffer = (char *)ftl_md_get_buffer(cache->l1_md) + ftl_mempool_get_df_obj_index(
864 cache->l2_ctx_pool, page) * FTL_BLOCK_SIZE;
872 ftl_l2p_cache_lru_add_page(cache, page);
875 ftl_mempool_initialize_ext(cache->l2_ctx_pool);
893 process_persist(struct ftl_l2p_cache *cache)
895 struct ftl_l2p_cache_process_ctx *ctx = &cache->mctx;
896 struct spdk_ftl_dev *dev = cache->dev;
898 while (ctx->idx < cache->num_pages && ctx->qd < 64) {
899 struct ftl_l2p_page *page = get_l2p_page_by_df_id(cache, ctx->idx);
912 ftl_l2p_cache_lru_remove_page(cache, page);
918 page->ctx.cache = cache;
922 ftl_l2p_cache_page_remove(cache, page);
927 process_finish(cache);
934 struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)dev->l2p;
936 process_init_ctx(dev, cache, cb, cb_ctx);
937 process_persist(cache);
943 struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)dev->l2p;
945 return cache->state == L2P_CACHE_SHUTDOWN_DONE;
951 struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)dev->l2p;
953 if (cache->state != L2P_CACHE_SHUTDOWN_DONE) {
954 cache->state = L2P_CACHE_IN_SHUTDOWN;
955 if (!cache->ios_in_flight && !cache->l2_pgs_evicting) {
956 cache->state = L2P_CACHE_SHUTDOWN_DONE;
964 struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)dev->l2p;
966 assert(cache->state == L2P_CACHE_INIT);
967 cache->state = L2P_CACHE_RUNNING;
971 get_page(struct ftl_l2p_cache *cache, uint64_t lba)
973 return get_l2p_page_by_df_id(cache, lba / cache->lbas_in_page);
988 ftl_l2p_cache_running(struct ftl_l2p_cache *cache)
990 return cache->state == L2P_CACHE_RUNNING;
1003 struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)dev->l2p;
1008 uint64_t start = pin_ctx->lba / cache->lbas_in_page;
1009 uint64_t end = (pin_ctx->lba + pin_ctx->count - 1) / cache->lbas_in_page;
1019 assert(ftl_l2p_cache_running(cache));
1020 page_set = ftl_mempool_get(cache->page_sets_pool);
1038 page = get_l2p_page_by_df_id(cache, i);
1045 ftl_l2p_cache_page_pin(cache, page);
1053 /* The page is not in the cache, queue the page_set to page in */
1060 page_set_end(dev, cache, page_set);
1062 TAILQ_INSERT_TAIL(&cache->deferred_page_set_list, page_set, list_entry);
1071 struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)dev->l2p;
1073 uint64_t start = lba / cache->lbas_in_page;
1074 uint64_t end = (lba + count - 1) / cache->lbas_in_page;
1078 assert(start < cache->num_pages);
1079 assert(end < cache->num_pages);
1082 page = get_l2p_page_by_df_id(cache, i);
1084 ftl_l2p_cache_page_unpin(cache, page);
1092 struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)dev->l2p;
1093 struct ftl_l2p_page *page = get_page(cache, lba);
1097 assert(ftl_l2p_cache_running(cache));
1105 ftl_l2p_cache_lru_promote_page(cache, page);
1106 addr = ftl_l2p_cache_get_addr(dev, cache, page, lba);
1115 struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)dev->l2p;
1116 struct ftl_l2p_page *page = get_page(cache, lba);
1119 assert(ftl_l2p_cache_running(cache));
1128 ftl_l2p_cache_lru_promote_page(cache, page);
1129 ftl_l2p_cache_set_addr(dev, cache, page, lba, addr);
1133 page_allocate(struct ftl_l2p_cache *cache, uint64_t page_no)
1135 struct ftl_l2p_page *page = ftl_l2p_cache_page_alloc(cache, page_no);
1136 ftl_l2p_cache_page_insert(cache, page);
1153 page_set_unpin(struct ftl_l2p_cache *cache, struct ftl_l2p_page_set *page_set)
1165 pinned_page = get_l2p_page_by_df_id(cache, pentry->pg_no);
1168 ftl_l2p_cache_page_unpin(cache, pinned_page);
1173 page_set_end(struct spdk_ftl_dev *dev, struct ftl_l2p_cache *cache,
1179 page_set_unpin(cache, page_set);
1184 TAILQ_REMOVE(&cache->deferred_page_set_list, page_set, list_entry);
1188 ftl_mempool_put(cache->page_sets_pool, page_set);
1192 page_in_io_complete(struct spdk_ftl_dev *dev, struct ftl_l2p_cache *cache,
1198 cache->ios_in_flight--;
1216 ftl_l2p_cache_page_pin(cache, page);
1225 page_set_end(dev, cache, page_set);
1231 ftl_l2p_cache_page_remove(cache, page);
1239 struct ftl_l2p_cache *cache = page->ctx.cache;
1240 struct spdk_ftl_dev *dev = cache->dev;
1244 page_in_io_complete(dev, cache, page, success);
1248 page_in_io(struct spdk_ftl_dev *dev, struct ftl_l2p_cache *cache, struct ftl_l2p_page *page)
1254 page->ctx.cache = cache;
1256 rc = ftl_nv_cache_bdev_read_blocks_with_md(cache->dev, ftl_l2p_cache_get_bdev_desc(cache),
1257 ftl_l2p_cache_get_bdev_iochannel(cache),
1258 page->page_buffer, NULL, ftl_l2p_cache_page_get_bdev_offset(cache, page),
1260 cache->ios_in_flight++;
1266 ioch = ftl_l2p_cache_get_bdev_iochannel(cache);
1267 bdev = spdk_bdev_desc_get_bdev(ftl_l2p_cache_get_bdev_desc(cache));
1284 struct ftl_l2p_cache *cache = page->ctx.cache;
1285 struct spdk_ftl_dev *dev = cache->dev;
1287 cache->ios_in_flight--;
1288 page_in_io(dev, cache, page);
1292 page_in(struct spdk_ftl_dev *dev, struct ftl_l2p_cache *cache,
1299 page = get_l2p_page_by_df_id(cache, pentry->pg_no);
1302 page = page_allocate(cache, pentry->pg_no);
1307 ftl_l2p_cache_page_pin(cache, page);
1317 page_in_io(dev, cache, page);
1322 ftl_l2p_cache_process_page_sets(struct spdk_ftl_dev *dev, struct ftl_l2p_cache *cache)
1328 page_set = TAILQ_FIRST(&cache->deferred_page_set_list);
1334 if (page_set->to_pin_cnt > cache->l2_pgs_avail) {
1338 if (cache->ios_in_flight > 512) {
1345 TAILQ_REMOVE(&cache->deferred_page_set_list, page_set, list_entry);
1353 page_in(dev, cache, page_set, pentry);
1361 page_set_end(dev, cache, page_set);
1368 eviction_get_page(struct spdk_ftl_dev *dev, struct ftl_l2p_cache *cache)
1371 struct ftl_l2p_page *page = ftl_l2p_cache_get_coldest_page(cache);
1378 ftl_l2p_cache_lru_remove_page(cache, page);
1395 page_out_io_complete(struct spdk_ftl_dev *dev, struct ftl_l2p_cache *cache,
1398 cache->l2_pgs_evicting--;
1409 ftl_l2p_cache_page_remove(cache, page);
1412 ftl_l2p_cache_lru_add_page(cache, page);
1422 struct ftl_l2p_cache *cache = page->ctx.cache;
1423 struct spdk_ftl_dev *dev = cache->dev;
1427 page_out_io_complete(dev, cache, page, success);
1431 page_out_io(struct spdk_ftl_dev *dev, struct ftl_l2p_cache *cache,
1439 page->ctx.cache = cache;
1441 rc = ftl_nv_cache_bdev_write_blocks_with_md(dev, ftl_l2p_cache_get_bdev_desc(cache),
1442 ftl_l2p_cache_get_bdev_iochannel(cache),
1443 page->page_buffer, NULL, ftl_l2p_cache_page_get_bdev_offset(cache, page),
1446 cache->l2_pgs_evicting++;
1452 ioch = ftl_l2p_cache_get_bdev_iochannel(cache);
1453 bdev = spdk_bdev_desc_get_bdev(ftl_l2p_cache_get_bdev_desc(cache));
1470 struct ftl_l2p_cache *cache = page->ctx.cache;
1471 struct spdk_ftl_dev *dev = cache->dev;
1473 cache->l2_pgs_evicting--;
1474 page_out_io(dev, cache, page);
1478 ftl_l2p_cache_process_eviction(struct spdk_ftl_dev *dev, struct ftl_l2p_cache *cache)
1482 if (!ftl_l2p_cache_evict_continue(cache)) {
1486 if (cache->l2_pgs_evicting > 512) {
1492 page = eviction_get_page(dev, cache);
1500 page_out_io(dev, cache, page);
1503 ftl_l2p_cache_page_remove(cache, page);
1510 struct ftl_l2p_cache *cache = dev->l2p;
1512 cache->lazy_trim.qd--;
1519 if (ftl_l2p_cache_running(cache)) {
1529 struct ftl_l2p_cache *cache = dev->l2p;
1537 if (cache->lazy_trim.qd == FTL_L2P_MAX_LAZY_TRIM_QD) {
1541 page_no = ftl_bitmap_find_first_set(dev->trim_map, cache->lazy_trim.page_no, UINT64_MAX);
1543 cache->lazy_trim.page_no = 0;
1546 page_no = ftl_bitmap_find_first_set(dev->trim_map, cache->lazy_trim.page_no, UINT64_MAX);
1553 cache->lazy_trim.page_no = page_no;
1555 pin_ctx = &cache->lazy_trim.pin_ctx;
1557 cache->lazy_trim.qd++;
1558 assert(cache->lazy_trim.qd <= FTL_L2P_MAX_LAZY_TRIM_QD);
1559 assert(page_no < cache->num_pages);
1561 pin_ctx->lba = page_no * cache->lbas_in_page;
1572 struct ftl_l2p_cache *cache = dev->l2p;
1575 if (spdk_unlikely(cache->state != L2P_CACHE_RUNNING)) {
1580 if (ftl_l2p_cache_process_page_sets(dev, cache)) {
1585 ftl_l2p_cache_process_eviction(dev, cache);