Lines Matching defs:page
32 L2P_CACHE_PAGE_INIT, /* Page in memory not initialized from disk page */
41 uint64_t updates; /* Number of times an L2P entry was updated in the page since it was last persisted */
64 /* A L2P page contains 1024 4B entries (or 512 8B ones for big drives).
122 TAILQ_HEAD(, ftl_l2p_page_set) deferred_page_set_list; /* for deferred page sets */
129 /* Currently processed page */
131 /* Context for page pinning */
150 typedef void (*ftl_l2p_cache_sync_cb)(struct spdk_ftl_dev *dev, int status, void *page,
160 ftl_l2p_page_queue_wait_ctx(struct ftl_l2p_page *page,
163 TAILQ_INSERT_TAIL(&page->ppe_list, ppe, list_entry);
185 ftl_l2p_cache_lru_remove_page(struct ftl_l2p_cache *cache, struct ftl_l2p_page *page)
187 assert(page);
188 assert(page->on_lru_list);
190 TAILQ_REMOVE(&cache->lru_list, page, list_entry);
191 page->on_lru_list = false;
195 ftl_l2p_cache_lru_add_page(struct ftl_l2p_cache *cache, struct ftl_l2p_page *page)
197 assert(page);
198 assert(!page->on_lru_list);
200 TAILQ_INSERT_HEAD(&cache->lru_list, page, list_entry);
202 page->on_lru_list = true;
206 ftl_l2p_cache_lru_promote_page(struct ftl_l2p_cache *cache, struct ftl_l2p_page *page)
208 if (!page->on_lru_list) {
212 ftl_l2p_cache_lru_remove_page(cache, page);
213 ftl_l2p_cache_lru_add_page(cache, page);
217 ftl_l2p_cache_page_insert(struct ftl_l2p_cache *cache, struct ftl_l2p_page *page)
222 assert(me[page->page_no].page_obj_id == FTL_DF_OBJ_ID_INVALID);
223 me[page->page_no].page_obj_id = page->obj_id;
227 ftl_l2p_cache_page_remove(struct ftl_l2p_cache *cache, struct ftl_l2p_page *page)
231 assert(me[page->page_no].page_obj_id != FTL_DF_OBJ_ID_INVALID);
232 assert(TAILQ_EMPTY(&page->ppe_list));
234 me[page->page_no].page_obj_id = FTL_DF_OBJ_ID_INVALID;
236 ftl_mempool_put(cache->l2_ctx_pool, page);
246 ftl_l2p_cache_get_hotter_page(struct ftl_l2p_page *page)
248 return TAILQ_PREV(page, l2p_lru_list, list_entry);
253 struct ftl_l2p_page *page)
255 return cache->cache_layout_offset + page->page_no;
273 struct ftl_l2p_page *page = ftl_mempool_get(cache->l2_ctx_pool);
274 ftl_bug(!page);
278 memset(page, 0, sizeof(*page));
280 page->obj_id = ftl_mempool_get_df_obj_id(cache->l2_ctx_pool, page);
282 page->page_buffer = (char *)ftl_md_get_buffer(cache->l1_md) + ftl_mempool_get_df_obj_index(
283 cache->l2_ctx_pool, page) * FTL_BLOCK_SIZE;
285 TAILQ_INIT(&page->ppe_list);
287 page->page_no = page_no;
288 page->state = L2P_CACHE_PAGE_INIT;
290 return page;
294 ftl_l2p_cache_page_can_remove(struct ftl_l2p_page *page)
296 return (!page->updates &&
297 page->state != L2P_CACHE_PAGE_INIT &&
298 !page->pin_ref_cnt);
303 struct ftl_l2p_cache *cache, struct ftl_l2p_page *page, uint64_t lba)
305 return ftl_addr_load(dev, page->page_buffer, lba % cache->lbas_in_page);
310 struct ftl_l2p_page *page, uint64_t lba, ftl_addr addr)
312 ftl_addr_store(dev, page->page_buffer, lba % cache->lbas_in_page, addr);
316 ftl_l2p_page_set_invalid(struct spdk_ftl_dev *dev, struct ftl_l2p_page *page)
322 page->updates++;
326 addr = ftl_addr_load(dev, page->page_buffer, i);
332 ftl_l2p_cache_set_addr(dev, cache, page, i, FTL_ADDR_INVALID);
337 ftl_l2p_cache_page_pin(struct ftl_l2p_cache *cache, struct ftl_l2p_page *page)
339 page->pin_ref_cnt++;
341 if (page->on_lru_list) {
342 ftl_l2p_cache_lru_remove_page(cache, page);
347 ftl_l2p_cache_page_unpin(struct ftl_l2p_cache *cache, struct ftl_l2p_page *page)
349 page->pin_ref_cnt--;
350 if (!page->pin_ref_cnt && !page->on_lru_list && page->state != L2P_CACHE_PAGE_FLUSHING) {
351 /* L2P_CACHE_PAGE_FLUSHING: the page is currently being evicted.
352 * In such a case, the page can't be returned to the rank list, because
354 * Moreover, the page could make it to the top of the rank list and be
357 * Depending on the page updates tracker, the page will be evicted
361 ftl_l2p_cache_lru_add_page(cache, page);
366 ftl_l2p_cache_page_can_evict(struct ftl_l2p_page *page)
368 return (page->state == L2P_CACHE_PAGE_FLUSHING ||
369 page->state == L2P_CACHE_PAGE_PERSISTING ||
370 page->state == L2P_CACHE_PAGE_INIT ||
371 page->pin_ref_cnt) ? false : true;
581 process_page_in(struct ftl_l2p_page *page, spdk_bdev_io_completion_cb cb)
583 struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)page->ctx.cache;
586 assert(page->page_buffer);
590 page->page_buffer, NULL, ftl_l2p_cache_page_get_bdev_offset(cache, page),
591 1, cb, page);
594 cb(NULL, false, page);
601 struct ftl_l2p_page *page = arg;
602 struct ftl_l2p_cache *cache = page->ctx.cache;
619 ftl_bitmap_clear(dev->trim_map, page->page_no);
621 ftl_l2p_cache_page_remove(cache, page);
628 process_page_out(struct ftl_l2p_page *page, spdk_bdev_io_completion_cb cb)
632 struct ftl_l2p_cache *cache = page->ctx.cache;
636 assert(page->page_buffer);
640 page->page_buffer, NULL, ftl_l2p_cache_page_get_bdev_offset(cache, page),
641 1, cb, page);
649 bdev_io_wait = &page->ctx.bdev_io_wait;
652 bdev_io_wait->cb_arg = page;
653 page->ctx.cb = cb;
665 struct ftl_l2p_page *page = _page;
667 process_page_out(page, page->ctx.cb);
675 struct ftl_l2p_page *page = (struct ftl_l2p_page *)ctx_page;
676 struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)page->ctx.cache;
688 assert(!page->on_lru_list);
689 assert(ftl_bitmap_get(dev->trim_map, page->page_no));
690 ftl_bitmap_clear(dev->trim_map, page->page_no);
691 ftl_l2p_cache_page_remove(cache, page);
700 struct ftl_l2p_page *page = (struct ftl_l2p_page *)ctx_page;
701 struct ftl_l2p_cache *cache = (struct ftl_l2p_cache *)page->ctx.cache;
710 assert(ftl_bitmap_get(dev->trim_map, page->page_no));
711 ftl_l2p_page_set_invalid(dev, page);
712 process_page_out(page, process_trim_page_out_cb);
726 struct ftl_l2p_page *page;
737 /* Allocate page to invalidate it */
738 page = ftl_l2p_cache_page_alloc(cache, ctx->idx);
739 if (!page) {
745 page->state = L2P_CACHE_PAGE_CLEARING;
746 page->ctx.cache = cache;
748 ftl_l2p_cache_page_insert(cache, page);
749 process_page_in(page, process_trim_page_in_cb);
796 struct ftl_l2p_page *page;
806 page = ftl_mempool_claim_df(cache->l2_ctx_pool, obj_id);
807 assert(page);
808 assert(page->obj_id == ftl_mempool_get_df_obj_id(cache->l2_ctx_pool, page));
809 assert(page->page_no == page_no);
810 assert(page->state != L2P_CACHE_PAGE_INIT);
811 assert(page->state != L2P_CACHE_PAGE_CLEARING);
815 page->page_buffer = (char *)ftl_md_get_buffer(cache->l1_md) + ftl_mempool_get_df_obj_index(
816 cache->l2_ctx_pool, page) * FTL_BLOCK_SIZE;
818 TAILQ_INIT(&page->ppe_list);
820 page->pin_ref_cnt = 0;
821 page->on_lru_list = 0;
822 memset(&page->ctx, 0, sizeof(page->ctx));
824 ftl_l2p_cache_lru_add_page(cache, page);
835 struct ftl_l2p_page *page;
845 page = ftl_mempool_claim_df(cache->l2_ctx_pool, obj_id);
846 assert(page);
847 assert(page->obj_id == ftl_mempool_get_df_obj_id(cache->l2_ctx_pool, page));
848 assert(page->page_no == page_no);
849 assert(page->state != L2P_CACHE_PAGE_CLEARING);
853 if (page->state == L2P_CACHE_PAGE_INIT) {
860 page->state = L2P_CACHE_PAGE_READY;
861 /* Assume page is dirty after crash */
862 page->updates = 1;
863 page->page_buffer = (char *)ftl_md_get_buffer(cache->l1_md) + ftl_mempool_get_df_obj_index(
864 cache->l2_ctx_pool, page) * FTL_BLOCK_SIZE;
866 TAILQ_INIT(&page->ppe_list);
868 page->pin_ref_cnt = 0;
869 page->on_lru_list = 0;
870 memset(&page->ctx, 0, sizeof(page->ctx));
872 ftl_l2p_cache_lru_add_page(cache, page);
899 struct ftl_l2p_page *page = get_l2p_page_by_df_id(cache, ctx->idx);
902 if (!page) {
906 /* Finish trim if the page was marked */
908 ftl_l2p_page_set_invalid(dev, page);
911 if (page->on_lru_list) {
912 ftl_l2p_cache_lru_remove_page(cache, page);
915 if (page->updates) {
916 /* Need to persist the page */
917 page->state = L2P_CACHE_PAGE_PERSISTING;
918 page->ctx.cache = cache;
920 process_page_out(page, process_persist_page_out_cb);
922 ftl_l2p_cache_page_remove(cache, page);
994 ftl_l2p_cache_page_is_pinnable(struct ftl_l2p_page *page)
996 return page->state != L2P_CACHE_PAGE_INIT;
1007 /* Calculate first and last page to pin, count of them */
1018 /* Get and initialize page sets */
1029 struct ftl_l2p_page *page;
1037 /* Try get page and pin */
1038 page = get_l2p_page_by_df_id(cache, i);
1039 if (page) {
1040 if (ftl_l2p_cache_page_is_pinnable(page)) {
1045 ftl_l2p_cache_page_pin(cache, page);
1047 /* The page is being loaded */
1048 /* Queue the page pin entry to be executed on page in */
1049 ftl_l2p_page_queue_wait_ctx(page, entry);
1053 /* The page is not in the cache, queue the page_set to page in */
1058 /* Check if page set is done */
1072 struct ftl_l2p_page *page;
1082 page = get_l2p_page_by_df_id(cache, i);
1083 ftl_bug(!page);
1084 ftl_l2p_cache_page_unpin(cache, page);
1093 struct ftl_l2p_page *page = get_page(cache, lba);
1096 ftl_bug(!page);
1098 assert(page->pin_ref_cnt);
1100 if (ftl_bitmap_get(dev->trim_map, page->page_no)) {
1101 ftl_l2p_page_set_invalid(dev, page);
1102 ftl_bitmap_clear(dev->trim_map, page->page_no);
1105 ftl_l2p_cache_lru_promote_page(cache, page);
1106 addr = ftl_l2p_cache_get_addr(dev, cache, page, lba);
1116 struct ftl_l2p_page *page = get_page(cache, lba);
1118 ftl_bug(!page);
1120 assert(page->pin_ref_cnt);
1122 if (ftl_bitmap_get(dev->trim_map, page->page_no)) {
1123 ftl_l2p_page_set_invalid(dev, page);
1124 ftl_bitmap_clear(dev->trim_map, page->page_no);
1127 page->updates++;
1128 ftl_l2p_cache_lru_promote_page(cache, page);
1129 ftl_l2p_cache_set_addr(dev, cache, page, lba, addr);
1135 struct ftl_l2p_page *page = ftl_l2p_cache_page_alloc(cache, page_no);
1136 ftl_l2p_cache_page_insert(cache, page);
1138 return page;
1193 struct ftl_l2p_page *page, bool success)
1200 assert(0 == page->pin_ref_cnt);
1201 assert(L2P_CACHE_PAGE_INIT == page->state);
1202 assert(false == page->on_lru_list);
1205 page->state = L2P_CACHE_PAGE_READY;
1208 while ((pentry = TAILQ_FIRST(&page->ppe_list))) {
1209 TAILQ_REMOVE(&page->ppe_list, pentry, list_entry);
1216 ftl_l2p_cache_page_pin(cache, page);
1230 ftl_bug(page->on_lru_list);
1231 ftl_l2p_cache_page_remove(cache, page);
1238 struct ftl_l2p_page *page = cb_arg;
1239 struct ftl_l2p_cache *cache = page->ctx.cache;
1244 page_in_io_complete(dev, cache, page, success);
1248 page_in_io(struct spdk_ftl_dev *dev, struct ftl_l2p_cache *cache, struct ftl_l2p_page *page)
1254 page->ctx.cache = cache;
1258 page->page_buffer, NULL, ftl_l2p_cache_page_get_bdev_offset(cache, page),
1259 1, page_in_io_cb, page);
1268 bdev_io_wait = &page->ctx.bdev_io_wait;
1271 bdev_io_wait->cb_arg = page;
1283 struct ftl_l2p_page *page = arg;
1284 struct ftl_l2p_cache *cache = page->ctx.cache;
1288 page_in_io(dev, cache, page);
1295 struct ftl_l2p_page *page;
1298 /* Get page */
1299 page = get_l2p_page_by_df_id(cache, pentry->pg_no);
1300 if (!page) {
1302 page = page_allocate(cache, pentry->pg_no);
1306 if (ftl_l2p_cache_page_is_pinnable(page)) {
1307 ftl_l2p_cache_page_pin(cache, page);
1313 ftl_l2p_page_queue_wait_ctx(page, pentry);
1317 page_in_io(dev, cache, page);
1335 /* No enough page to pin, wait */
1371 struct ftl_l2p_page *page = ftl_l2p_cache_get_coldest_page(cache);
1373 while (page) {
1374 ftl_bug(L2P_CACHE_PAGE_READY != page->state);
1375 ftl_bug(page->pin_ref_cnt);
1377 if (ftl_l2p_cache_page_can_evict(page)) {
1378 ftl_l2p_cache_lru_remove_page(cache, page);
1379 return page;
1383 * Practically only one iteration is needed to find a page. It is because
1388 page = ftl_l2p_cache_get_hotter_page(page);
1396 struct ftl_l2p_page *page, bool success)
1400 ftl_bug(page->ctx.updates > page->updates);
1401 ftl_bug(!TAILQ_EMPTY(&page->ppe_list));
1402 ftl_bug(page->on_lru_list);
1405 page->updates -= page->ctx.updates;
1408 if (success && ftl_l2p_cache_page_can_remove(page)) {
1409 ftl_l2p_cache_page_remove(cache, page);
1411 if (!page->pin_ref_cnt) {
1412 ftl_l2p_cache_lru_add_page(cache, page);
1414 page->state = L2P_CACHE_PAGE_READY;
1421 struct ftl_l2p_page *page = cb_arg;
1422 struct ftl_l2p_cache *cache = page->ctx.cache;
1427 page_out_io_complete(dev, cache, page, success);
1432 struct ftl_l2p_page *page)
1439 page->ctx.cache = cache;
1443 page->page_buffer, NULL, ftl_l2p_cache_page_get_bdev_offset(cache, page),
1444 1, page_out_io_cb, page);
1454 bdev_io_wait = &page->ctx.bdev_io_wait;
1457 bdev_io_wait->cb_arg = page;
1469 struct ftl_l2p_page *page = arg;
1470 struct ftl_l2p_cache *cache = page->ctx.cache;
1474 page_out_io(dev, cache, page);
1480 struct ftl_l2p_page *page;
1492 page = eviction_get_page(dev, cache);
1493 if (spdk_unlikely(!page)) {
1497 if (page->updates) {
1498 page->state = L2P_CACHE_PAGE_FLUSHING;
1499 page->ctx.updates = page->updates;
1500 page_out_io(dev, cache, page);
1503 ftl_l2p_cache_page_remove(cache, page);