1789Sahrens /* 2789Sahrens * CDDL HEADER START 3789Sahrens * 4789Sahrens * The contents of this file are subject to the terms of the 51484Sek110237 * Common Development and Distribution License (the "License"). 61484Sek110237 * You may not use this file except in compliance with the License. 7789Sahrens * 8789Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9789Sahrens * or http://www.opensolaris.org/os/licensing. 10789Sahrens * See the License for the specific language governing permissions 11789Sahrens * and limitations under the License. 12789Sahrens * 13789Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14789Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15789Sahrens * If applicable, add the following below this CDDL HEADER, with the 16789Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17789Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18789Sahrens * 19789Sahrens * CDDL HEADER END 20789Sahrens */ 21789Sahrens /* 223403Sbmc * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23789Sahrens * Use is subject to license terms. 24789Sahrens */ 25789Sahrens 26789Sahrens #pragma ident "%Z%%M% %I% %E% SMI" 27789Sahrens 28789Sahrens /* 293403Sbmc * DVA-based Adjustable Replacement Cache 30789Sahrens * 311544Seschrock * While much of the theory of operation used here is 321544Seschrock * based on the self-tuning, low overhead replacement cache 33789Sahrens * presented by Megiddo and Modha at FAST 2003, there are some 34789Sahrens * significant differences: 35789Sahrens * 36789Sahrens * 1. The Megiddo and Modha model assumes any page is evictable. 37789Sahrens * Pages in its cache cannot be "locked" into memory. This makes 38789Sahrens * the eviction algorithm simple: evict the last page in the list. 39789Sahrens * This also make the performance characteristics easy to reason 40789Sahrens * about. Our cache is not so simple. At any given moment, some 41789Sahrens * subset of the blocks in the cache are un-evictable because we 42789Sahrens * have handed out a reference to them. Blocks are only evictable 43789Sahrens * when there are no external references active. This makes 44789Sahrens * eviction far more problematic: we choose to evict the evictable 45789Sahrens * blocks that are the "lowest" in the list. 46789Sahrens * 47789Sahrens * There are times when it is not possible to evict the requested 48789Sahrens * space. In these circumstances we are unable to adjust the cache 49789Sahrens * size. To prevent the cache growing unbounded at these times we 50789Sahrens * implement a "cache throttle" that slowes the flow of new data 51789Sahrens * into the cache until we can make space avaiable. 52789Sahrens * 53789Sahrens * 2. The Megiddo and Modha model assumes a fixed cache size. 54789Sahrens * Pages are evicted when the cache is full and there is a cache 55789Sahrens * miss. Our model has a variable sized cache. It grows with 56789Sahrens * high use, but also tries to react to memory preasure from the 57789Sahrens * operating system: decreasing its size when system memory is 58789Sahrens * tight. 59789Sahrens * 60789Sahrens * 3. The Megiddo and Modha model assumes a fixed page size. All 61789Sahrens * elements of the cache are therefor exactly the same size. So 62789Sahrens * when adjusting the cache size following a cache miss, its simply 63789Sahrens * a matter of choosing a single page to evict. In our model, we 64789Sahrens * have variable sized cache blocks (rangeing from 512 bytes to 65789Sahrens * 128K bytes). We therefor choose a set of blocks to evict to make 66789Sahrens * space for a cache miss that approximates as closely as possible 67789Sahrens * the space used by the new block. 68789Sahrens * 69789Sahrens * See also: "ARC: A Self-Tuning, Low Overhead Replacement Cache" 70789Sahrens * by N. Megiddo & D. Modha, FAST 2003 71789Sahrens */ 72789Sahrens 73789Sahrens /* 74789Sahrens * The locking model: 75789Sahrens * 76789Sahrens * A new reference to a cache buffer can be obtained in two 77789Sahrens * ways: 1) via a hash table lookup using the DVA as a key, 78789Sahrens * or 2) via one of the ARC lists. The arc_read() inerface 79789Sahrens * uses method 1, while the internal arc algorithms for 80789Sahrens * adjusting the cache use method 2. We therefor provide two 81789Sahrens * types of locks: 1) the hash table lock array, and 2) the 82789Sahrens * arc list locks. 83789Sahrens * 84789Sahrens * Buffers do not have their own mutexs, rather they rely on the 85789Sahrens * hash table mutexs for the bulk of their protection (i.e. most 86789Sahrens * fields in the arc_buf_hdr_t are protected by these mutexs). 87789Sahrens * 88789Sahrens * buf_hash_find() returns the appropriate mutex (held) when it 89789Sahrens * locates the requested buffer in the hash table. It returns 90789Sahrens * NULL for the mutex if the buffer was not in the table. 91789Sahrens * 92789Sahrens * buf_hash_remove() expects the appropriate hash mutex to be 93789Sahrens * already held before it is invoked. 94789Sahrens * 95789Sahrens * Each arc state also has a mutex which is used to protect the 96789Sahrens * buffer list associated with the state. When attempting to 97789Sahrens * obtain a hash table lock while holding an arc list lock you 98789Sahrens * must use: mutex_tryenter() to avoid deadlock. Also note that 992688Smaybee * the active state mutex must be held before the ghost state mutex. 100789Sahrens * 1011544Seschrock * Arc buffers may have an associated eviction callback function. 1021544Seschrock * This function will be invoked prior to removing the buffer (e.g. 1031544Seschrock * in arc_do_user_evicts()). Note however that the data associated 1041544Seschrock * with the buffer may be evicted prior to the callback. The callback 1051544Seschrock * must be made with *no locks held* (to prevent deadlock). Additionally, 1061544Seschrock * the users of callbacks must ensure that their private data is 1071544Seschrock * protected from simultaneous callbacks from arc_buf_evict() 1081544Seschrock * and arc_do_user_evicts(). 1091544Seschrock * 110789Sahrens * Note that the majority of the performance stats are manipulated 111789Sahrens * with atomic operations. 112789Sahrens */ 113789Sahrens 114789Sahrens #include <sys/spa.h> 115789Sahrens #include <sys/zio.h> 1163093Sahrens #include <sys/zio_checksum.h> 117789Sahrens #include <sys/zfs_context.h> 118789Sahrens #include <sys/arc.h> 119789Sahrens #include <sys/refcount.h> 120789Sahrens #ifdef _KERNEL 121789Sahrens #include <sys/vmsystm.h> 122789Sahrens #include <vm/anon.h> 123789Sahrens #include <sys/fs/swapnode.h> 1241484Sek110237 #include <sys/dnlc.h> 125789Sahrens #endif 126789Sahrens #include <sys/callb.h> 1273403Sbmc #include <sys/kstat.h> 128789Sahrens 129789Sahrens static kmutex_t arc_reclaim_thr_lock; 130789Sahrens static kcondvar_t arc_reclaim_thr_cv; /* used to signal reclaim thr */ 131789Sahrens static uint8_t arc_thread_exit; 132789Sahrens 1331484Sek110237 #define ARC_REDUCE_DNLC_PERCENT 3 1341484Sek110237 uint_t arc_reduce_dnlc_percent = ARC_REDUCE_DNLC_PERCENT; 1351484Sek110237 136789Sahrens typedef enum arc_reclaim_strategy { 137789Sahrens ARC_RECLAIM_AGGR, /* Aggressive reclaim strategy */ 138789Sahrens ARC_RECLAIM_CONS /* Conservative reclaim strategy */ 139789Sahrens } arc_reclaim_strategy_t; 140789Sahrens 141789Sahrens /* number of seconds before growing cache again */ 142789Sahrens static int arc_grow_retry = 60; 143789Sahrens 1442391Smaybee /* 1452638Sperrin * minimum lifespan of a prefetch block in clock ticks 1462638Sperrin * (initialized in arc_init()) 1472391Smaybee */ 1482638Sperrin static int arc_min_prefetch_lifespan; 1492391Smaybee 150789Sahrens static int arc_dead; 151789Sahrens 152789Sahrens /* 1532885Sahrens * These tunables are for performance analysis. 1542885Sahrens */ 1552885Sahrens uint64_t zfs_arc_max; 1562885Sahrens uint64_t zfs_arc_min; 1574645Sek110237 uint64_t zfs_arc_meta_limit = 0; 1582885Sahrens 1592885Sahrens /* 1604309Smaybee * Note that buffers can be in one of 5 states: 161789Sahrens * ARC_anon - anonymous (discussed below) 1621544Seschrock * ARC_mru - recently used, currently cached 1631544Seschrock * ARC_mru_ghost - recentely used, no longer in cache 1641544Seschrock * ARC_mfu - frequently used, currently cached 1651544Seschrock * ARC_mfu_ghost - frequently used, no longer in cache 1664309Smaybee * When there are no active references to the buffer, they are 1674309Smaybee * are linked onto a list in one of these arc states. These are 1684309Smaybee * the only buffers that can be evicted or deleted. Within each 1694309Smaybee * state there are multiple lists, one for meta-data and one for 1704309Smaybee * non-meta-data. Meta-data (indirect blocks, blocks of dnodes, 1714309Smaybee * etc.) is tracked separately so that it can be managed more 1724309Smaybee * explicitly: favored over data, limited explicitely. 173789Sahrens * 174789Sahrens * Anonymous buffers are buffers that are not associated with 175789Sahrens * a DVA. These are buffers that hold dirty block copies 176789Sahrens * before they are written to stable storage. By definition, 1771544Seschrock * they are "ref'd" and are considered part of arc_mru 178789Sahrens * that cannot be freed. Generally, they will aquire a DVA 1791544Seschrock * as they are written and migrate onto the arc_mru list. 180789Sahrens */ 181789Sahrens 182789Sahrens typedef struct arc_state { 1834309Smaybee list_t arcs_list[ARC_BUFC_NUMTYPES]; /* list of evictable buffers */ 1844309Smaybee uint64_t arcs_lsize[ARC_BUFC_NUMTYPES]; /* amount of evictable data */ 1854309Smaybee uint64_t arcs_size; /* total amount of data in this state */ 1863403Sbmc kmutex_t arcs_mtx; 187789Sahrens } arc_state_t; 188789Sahrens 189789Sahrens /* The 5 states: */ 190789Sahrens static arc_state_t ARC_anon; 1911544Seschrock static arc_state_t ARC_mru; 1921544Seschrock static arc_state_t ARC_mru_ghost; 1931544Seschrock static arc_state_t ARC_mfu; 1941544Seschrock static arc_state_t ARC_mfu_ghost; 195789Sahrens 1963403Sbmc typedef struct arc_stats { 1973403Sbmc kstat_named_t arcstat_hits; 1983403Sbmc kstat_named_t arcstat_misses; 1993403Sbmc kstat_named_t arcstat_demand_data_hits; 2003403Sbmc kstat_named_t arcstat_demand_data_misses; 2013403Sbmc kstat_named_t arcstat_demand_metadata_hits; 2023403Sbmc kstat_named_t arcstat_demand_metadata_misses; 2033403Sbmc kstat_named_t arcstat_prefetch_data_hits; 2043403Sbmc kstat_named_t arcstat_prefetch_data_misses; 2053403Sbmc kstat_named_t arcstat_prefetch_metadata_hits; 2063403Sbmc kstat_named_t arcstat_prefetch_metadata_misses; 2073403Sbmc kstat_named_t arcstat_mru_hits; 2083403Sbmc kstat_named_t arcstat_mru_ghost_hits; 2093403Sbmc kstat_named_t arcstat_mfu_hits; 2103403Sbmc kstat_named_t arcstat_mfu_ghost_hits; 2113403Sbmc kstat_named_t arcstat_deleted; 2123403Sbmc kstat_named_t arcstat_recycle_miss; 2133403Sbmc kstat_named_t arcstat_mutex_miss; 2143403Sbmc kstat_named_t arcstat_evict_skip; 2153403Sbmc kstat_named_t arcstat_hash_elements; 2163403Sbmc kstat_named_t arcstat_hash_elements_max; 2173403Sbmc kstat_named_t arcstat_hash_collisions; 2183403Sbmc kstat_named_t arcstat_hash_chains; 2193403Sbmc kstat_named_t arcstat_hash_chain_max; 2203403Sbmc kstat_named_t arcstat_p; 2213403Sbmc kstat_named_t arcstat_c; 2223403Sbmc kstat_named_t arcstat_c_min; 2233403Sbmc kstat_named_t arcstat_c_max; 2243403Sbmc kstat_named_t arcstat_size; 2253403Sbmc } arc_stats_t; 2263403Sbmc 2273403Sbmc static arc_stats_t arc_stats = { 2283403Sbmc { "hits", KSTAT_DATA_UINT64 }, 2293403Sbmc { "misses", KSTAT_DATA_UINT64 }, 2303403Sbmc { "demand_data_hits", KSTAT_DATA_UINT64 }, 2313403Sbmc { "demand_data_misses", KSTAT_DATA_UINT64 }, 2323403Sbmc { "demand_metadata_hits", KSTAT_DATA_UINT64 }, 2333403Sbmc { "demand_metadata_misses", KSTAT_DATA_UINT64 }, 2343403Sbmc { "prefetch_data_hits", KSTAT_DATA_UINT64 }, 2353403Sbmc { "prefetch_data_misses", KSTAT_DATA_UINT64 }, 2363403Sbmc { "prefetch_metadata_hits", KSTAT_DATA_UINT64 }, 2373403Sbmc { "prefetch_metadata_misses", KSTAT_DATA_UINT64 }, 2383403Sbmc { "mru_hits", KSTAT_DATA_UINT64 }, 2393403Sbmc { "mru_ghost_hits", KSTAT_DATA_UINT64 }, 2403403Sbmc { "mfu_hits", KSTAT_DATA_UINT64 }, 2413403Sbmc { "mfu_ghost_hits", KSTAT_DATA_UINT64 }, 2423403Sbmc { "deleted", KSTAT_DATA_UINT64 }, 2433403Sbmc { "recycle_miss", KSTAT_DATA_UINT64 }, 2443403Sbmc { "mutex_miss", KSTAT_DATA_UINT64 }, 2453403Sbmc { "evict_skip", KSTAT_DATA_UINT64 }, 2463403Sbmc { "hash_elements", KSTAT_DATA_UINT64 }, 2473403Sbmc { "hash_elements_max", KSTAT_DATA_UINT64 }, 2483403Sbmc { "hash_collisions", KSTAT_DATA_UINT64 }, 2493403Sbmc { "hash_chains", KSTAT_DATA_UINT64 }, 2503403Sbmc { "hash_chain_max", KSTAT_DATA_UINT64 }, 2513403Sbmc { "p", KSTAT_DATA_UINT64 }, 2523403Sbmc { "c", KSTAT_DATA_UINT64 }, 2533403Sbmc { "c_min", KSTAT_DATA_UINT64 }, 2543403Sbmc { "c_max", KSTAT_DATA_UINT64 }, 2553403Sbmc { "size", KSTAT_DATA_UINT64 } 2563403Sbmc }; 257789Sahrens 2583403Sbmc #define ARCSTAT(stat) (arc_stats.stat.value.ui64) 2593403Sbmc 2603403Sbmc #define ARCSTAT_INCR(stat, val) \ 2613403Sbmc atomic_add_64(&arc_stats.stat.value.ui64, (val)); 2623403Sbmc 2633403Sbmc #define ARCSTAT_BUMP(stat) ARCSTAT_INCR(stat, 1) 2643403Sbmc #define ARCSTAT_BUMPDOWN(stat) ARCSTAT_INCR(stat, -1) 2653403Sbmc 2663403Sbmc #define ARCSTAT_MAX(stat, val) { \ 2673403Sbmc uint64_t m; \ 2683403Sbmc while ((val) > (m = arc_stats.stat.value.ui64) && \ 2693403Sbmc (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val)))) \ 2703403Sbmc continue; \ 2713403Sbmc } 2723403Sbmc 2733403Sbmc #define ARCSTAT_MAXSTAT(stat) \ 2743403Sbmc ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64) 275789Sahrens 2763403Sbmc /* 2773403Sbmc * We define a macro to allow ARC hits/misses to be easily broken down by 2783403Sbmc * two separate conditions, giving a total of four different subtypes for 2793403Sbmc * each of hits and misses (so eight statistics total). 2803403Sbmc */ 2813403Sbmc #define ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \ 2823403Sbmc if (cond1) { \ 2833403Sbmc if (cond2) { \ 2843403Sbmc ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \ 2853403Sbmc } else { \ 2863403Sbmc ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \ 2873403Sbmc } \ 2883403Sbmc } else { \ 2893403Sbmc if (cond2) { \ 2903403Sbmc ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \ 2913403Sbmc } else { \ 2923403Sbmc ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\ 2933403Sbmc } \ 2943403Sbmc } 295789Sahrens 2963403Sbmc kstat_t *arc_ksp; 2973403Sbmc static arc_state_t *arc_anon; 2983403Sbmc static arc_state_t *arc_mru; 2993403Sbmc static arc_state_t *arc_mru_ghost; 3003403Sbmc static arc_state_t *arc_mfu; 3013403Sbmc static arc_state_t *arc_mfu_ghost; 3023403Sbmc 3033403Sbmc /* 3043403Sbmc * There are several ARC variables that are critical to export as kstats -- 3053403Sbmc * but we don't want to have to grovel around in the kstat whenever we wish to 3063403Sbmc * manipulate them. For these variables, we therefore define them to be in 3073403Sbmc * terms of the statistic variable. This assures that we are not introducing 3083403Sbmc * the possibility of inconsistency by having shadow copies of the variables, 3093403Sbmc * while still allowing the code to be readable. 3103403Sbmc */ 3113403Sbmc #define arc_size ARCSTAT(arcstat_size) /* actual total arc size */ 3123403Sbmc #define arc_p ARCSTAT(arcstat_p) /* target size of MRU */ 3133403Sbmc #define arc_c ARCSTAT(arcstat_c) /* target size of cache */ 3143403Sbmc #define arc_c_min ARCSTAT(arcstat_c_min) /* min target cache size */ 3153403Sbmc #define arc_c_max ARCSTAT(arcstat_c_max) /* max target cache size */ 3163403Sbmc 3173403Sbmc static int arc_no_grow; /* Don't try to grow cache size */ 3183403Sbmc static uint64_t arc_tempreserve; 3194309Smaybee static uint64_t arc_meta_used; 3204309Smaybee static uint64_t arc_meta_limit; 3214309Smaybee static uint64_t arc_meta_max = 0; 322789Sahrens 323789Sahrens typedef struct arc_callback arc_callback_t; 324789Sahrens 325789Sahrens struct arc_callback { 3263547Smaybee void *acb_private; 327789Sahrens arc_done_func_t *acb_done; 328789Sahrens arc_byteswap_func_t *acb_byteswap; 329789Sahrens arc_buf_t *acb_buf; 330789Sahrens zio_t *acb_zio_dummy; 331789Sahrens arc_callback_t *acb_next; 332789Sahrens }; 333789Sahrens 3343547Smaybee typedef struct arc_write_callback arc_write_callback_t; 3353547Smaybee 3363547Smaybee struct arc_write_callback { 3373547Smaybee void *awcb_private; 3383547Smaybee arc_done_func_t *awcb_ready; 3393547Smaybee arc_done_func_t *awcb_done; 3403547Smaybee arc_buf_t *awcb_buf; 3413547Smaybee }; 3423547Smaybee 343789Sahrens struct arc_buf_hdr { 344789Sahrens /* protected by hash lock */ 345789Sahrens dva_t b_dva; 346789Sahrens uint64_t b_birth; 347789Sahrens uint64_t b_cksum0; 348789Sahrens 3493093Sahrens kmutex_t b_freeze_lock; 3503093Sahrens zio_cksum_t *b_freeze_cksum; 3513093Sahrens 352789Sahrens arc_buf_hdr_t *b_hash_next; 353789Sahrens arc_buf_t *b_buf; 354789Sahrens uint32_t b_flags; 3551544Seschrock uint32_t b_datacnt; 356789Sahrens 3573290Sjohansen arc_callback_t *b_acb; 358789Sahrens kcondvar_t b_cv; 3593290Sjohansen 3603290Sjohansen /* immutable */ 3613290Sjohansen arc_buf_contents_t b_type; 3623290Sjohansen uint64_t b_size; 3633290Sjohansen spa_t *b_spa; 364789Sahrens 365789Sahrens /* protected by arc state mutex */ 366789Sahrens arc_state_t *b_state; 367789Sahrens list_node_t b_arc_node; 368789Sahrens 369789Sahrens /* updated atomically */ 370789Sahrens clock_t b_arc_access; 371789Sahrens 372789Sahrens /* self protecting */ 373789Sahrens refcount_t b_refcnt; 374789Sahrens }; 375789Sahrens 3761544Seschrock static arc_buf_t *arc_eviction_list; 3771544Seschrock static kmutex_t arc_eviction_mtx; 3782887Smaybee static arc_buf_hdr_t arc_eviction_hdr; 3792688Smaybee static void arc_get_data_buf(arc_buf_t *buf); 3802688Smaybee static void arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock); 3814309Smaybee static int arc_evict_needed(arc_buf_contents_t type); 382*4709Smaybee static void arc_evict_ghost(arc_state_t *state, int64_t bytes); 3831544Seschrock 3841544Seschrock #define GHOST_STATE(state) \ 3853403Sbmc ((state) == arc_mru_ghost || (state) == arc_mfu_ghost) 3861544Seschrock 387789Sahrens /* 388789Sahrens * Private ARC flags. These flags are private ARC only flags that will show up 389789Sahrens * in b_flags in the arc_hdr_buf_t. Some flags are publicly declared, and can 390789Sahrens * be passed in as arc_flags in things like arc_read. However, these flags 391789Sahrens * should never be passed and should only be set by ARC code. When adding new 392789Sahrens * public flags, make sure not to smash the private ones. 393789Sahrens */ 394789Sahrens 3951544Seschrock #define ARC_IN_HASH_TABLE (1 << 9) /* this buffer is hashed */ 396789Sahrens #define ARC_IO_IN_PROGRESS (1 << 10) /* I/O in progress for buf */ 397789Sahrens #define ARC_IO_ERROR (1 << 11) /* I/O failed for buf */ 398789Sahrens #define ARC_FREED_IN_READ (1 << 12) /* buf freed while in read */ 3991544Seschrock #define ARC_BUF_AVAILABLE (1 << 13) /* block not in active use */ 4002391Smaybee #define ARC_INDIRECT (1 << 14) /* this is an indirect block */ 401789Sahrens 4021544Seschrock #define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_IN_HASH_TABLE) 403789Sahrens #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_IO_IN_PROGRESS) 404789Sahrens #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_IO_ERROR) 405789Sahrens #define HDR_FREED_IN_READ(hdr) ((hdr)->b_flags & ARC_FREED_IN_READ) 4061544Seschrock #define HDR_BUF_AVAILABLE(hdr) ((hdr)->b_flags & ARC_BUF_AVAILABLE) 407789Sahrens 408789Sahrens /* 409789Sahrens * Hash table routines 410789Sahrens */ 411789Sahrens 412789Sahrens #define HT_LOCK_PAD 64 413789Sahrens 414789Sahrens struct ht_lock { 415789Sahrens kmutex_t ht_lock; 416789Sahrens #ifdef _KERNEL 417789Sahrens unsigned char pad[(HT_LOCK_PAD - sizeof (kmutex_t))]; 418789Sahrens #endif 419789Sahrens }; 420789Sahrens 421789Sahrens #define BUF_LOCKS 256 422789Sahrens typedef struct buf_hash_table { 423789Sahrens uint64_t ht_mask; 424789Sahrens arc_buf_hdr_t **ht_table; 425789Sahrens struct ht_lock ht_locks[BUF_LOCKS]; 426789Sahrens } buf_hash_table_t; 427789Sahrens 428789Sahrens static buf_hash_table_t buf_hash_table; 429789Sahrens 430789Sahrens #define BUF_HASH_INDEX(spa, dva, birth) \ 431789Sahrens (buf_hash(spa, dva, birth) & buf_hash_table.ht_mask) 432789Sahrens #define BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)]) 433789Sahrens #define BUF_HASH_LOCK(idx) (&(BUF_HASH_LOCK_NTRY(idx).ht_lock)) 434789Sahrens #define HDR_LOCK(buf) \ 435789Sahrens (BUF_HASH_LOCK(BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth))) 436789Sahrens 437789Sahrens uint64_t zfs_crc64_table[256]; 438789Sahrens 439789Sahrens static uint64_t 440789Sahrens buf_hash(spa_t *spa, dva_t *dva, uint64_t birth) 441789Sahrens { 442789Sahrens uintptr_t spav = (uintptr_t)spa; 443789Sahrens uint8_t *vdva = (uint8_t *)dva; 444789Sahrens uint64_t crc = -1ULL; 445789Sahrens int i; 446789Sahrens 447789Sahrens ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 448789Sahrens 449789Sahrens for (i = 0; i < sizeof (dva_t); i++) 450789Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF]; 451789Sahrens 452789Sahrens crc ^= (spav>>8) ^ birth; 453789Sahrens 454789Sahrens return (crc); 455789Sahrens } 456789Sahrens 457789Sahrens #define BUF_EMPTY(buf) \ 458789Sahrens ((buf)->b_dva.dva_word[0] == 0 && \ 459789Sahrens (buf)->b_dva.dva_word[1] == 0 && \ 460789Sahrens (buf)->b_birth == 0) 461789Sahrens 462789Sahrens #define BUF_EQUAL(spa, dva, birth, buf) \ 463789Sahrens ((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \ 464789Sahrens ((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \ 465789Sahrens ((buf)->b_birth == birth) && ((buf)->b_spa == spa) 466789Sahrens 467789Sahrens static arc_buf_hdr_t * 468789Sahrens buf_hash_find(spa_t *spa, dva_t *dva, uint64_t birth, kmutex_t **lockp) 469789Sahrens { 470789Sahrens uint64_t idx = BUF_HASH_INDEX(spa, dva, birth); 471789Sahrens kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 472789Sahrens arc_buf_hdr_t *buf; 473789Sahrens 474789Sahrens mutex_enter(hash_lock); 475789Sahrens for (buf = buf_hash_table.ht_table[idx]; buf != NULL; 476789Sahrens buf = buf->b_hash_next) { 477789Sahrens if (BUF_EQUAL(spa, dva, birth, buf)) { 478789Sahrens *lockp = hash_lock; 479789Sahrens return (buf); 480789Sahrens } 481789Sahrens } 482789Sahrens mutex_exit(hash_lock); 483789Sahrens *lockp = NULL; 484789Sahrens return (NULL); 485789Sahrens } 486789Sahrens 487789Sahrens /* 488789Sahrens * Insert an entry into the hash table. If there is already an element 489789Sahrens * equal to elem in the hash table, then the already existing element 490789Sahrens * will be returned and the new element will not be inserted. 491789Sahrens * Otherwise returns NULL. 492789Sahrens */ 493789Sahrens static arc_buf_hdr_t * 494789Sahrens buf_hash_insert(arc_buf_hdr_t *buf, kmutex_t **lockp) 495789Sahrens { 496789Sahrens uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth); 497789Sahrens kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 498789Sahrens arc_buf_hdr_t *fbuf; 4993403Sbmc uint32_t i; 500789Sahrens 5011544Seschrock ASSERT(!HDR_IN_HASH_TABLE(buf)); 502789Sahrens *lockp = hash_lock; 503789Sahrens mutex_enter(hash_lock); 504789Sahrens for (fbuf = buf_hash_table.ht_table[idx], i = 0; fbuf != NULL; 505789Sahrens fbuf = fbuf->b_hash_next, i++) { 506789Sahrens if (BUF_EQUAL(buf->b_spa, &buf->b_dva, buf->b_birth, fbuf)) 507789Sahrens return (fbuf); 508789Sahrens } 509789Sahrens 510789Sahrens buf->b_hash_next = buf_hash_table.ht_table[idx]; 511789Sahrens buf_hash_table.ht_table[idx] = buf; 5121544Seschrock buf->b_flags |= ARC_IN_HASH_TABLE; 513789Sahrens 514789Sahrens /* collect some hash table performance data */ 515789Sahrens if (i > 0) { 5163403Sbmc ARCSTAT_BUMP(arcstat_hash_collisions); 517789Sahrens if (i == 1) 5183403Sbmc ARCSTAT_BUMP(arcstat_hash_chains); 5193403Sbmc 5203403Sbmc ARCSTAT_MAX(arcstat_hash_chain_max, i); 521789Sahrens } 5223403Sbmc 5233403Sbmc ARCSTAT_BUMP(arcstat_hash_elements); 5243403Sbmc ARCSTAT_MAXSTAT(arcstat_hash_elements); 525789Sahrens 526789Sahrens return (NULL); 527789Sahrens } 528789Sahrens 529789Sahrens static void 530789Sahrens buf_hash_remove(arc_buf_hdr_t *buf) 531789Sahrens { 532789Sahrens arc_buf_hdr_t *fbuf, **bufp; 533789Sahrens uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth); 534789Sahrens 535789Sahrens ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx))); 5361544Seschrock ASSERT(HDR_IN_HASH_TABLE(buf)); 537789Sahrens 538789Sahrens bufp = &buf_hash_table.ht_table[idx]; 539789Sahrens while ((fbuf = *bufp) != buf) { 540789Sahrens ASSERT(fbuf != NULL); 541789Sahrens bufp = &fbuf->b_hash_next; 542789Sahrens } 543789Sahrens *bufp = buf->b_hash_next; 544789Sahrens buf->b_hash_next = NULL; 5451544Seschrock buf->b_flags &= ~ARC_IN_HASH_TABLE; 546789Sahrens 547789Sahrens /* collect some hash table performance data */ 5483403Sbmc ARCSTAT_BUMPDOWN(arcstat_hash_elements); 5493403Sbmc 550789Sahrens if (buf_hash_table.ht_table[idx] && 551789Sahrens buf_hash_table.ht_table[idx]->b_hash_next == NULL) 5523403Sbmc ARCSTAT_BUMPDOWN(arcstat_hash_chains); 553789Sahrens } 554789Sahrens 555789Sahrens /* 556789Sahrens * Global data structures and functions for the buf kmem cache. 557789Sahrens */ 558789Sahrens static kmem_cache_t *hdr_cache; 559789Sahrens static kmem_cache_t *buf_cache; 560789Sahrens 561789Sahrens static void 562789Sahrens buf_fini(void) 563789Sahrens { 564789Sahrens int i; 565789Sahrens 566789Sahrens kmem_free(buf_hash_table.ht_table, 567789Sahrens (buf_hash_table.ht_mask + 1) * sizeof (void *)); 568789Sahrens for (i = 0; i < BUF_LOCKS; i++) 569789Sahrens mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock); 570789Sahrens kmem_cache_destroy(hdr_cache); 571789Sahrens kmem_cache_destroy(buf_cache); 572789Sahrens } 573789Sahrens 574789Sahrens /* 575789Sahrens * Constructor callback - called when the cache is empty 576789Sahrens * and a new buf is requested. 577789Sahrens */ 578789Sahrens /* ARGSUSED */ 579789Sahrens static int 580789Sahrens hdr_cons(void *vbuf, void *unused, int kmflag) 581789Sahrens { 582789Sahrens arc_buf_hdr_t *buf = vbuf; 583789Sahrens 584789Sahrens bzero(buf, sizeof (arc_buf_hdr_t)); 585789Sahrens refcount_create(&buf->b_refcnt); 586789Sahrens cv_init(&buf->b_cv, NULL, CV_DEFAULT, NULL); 587789Sahrens return (0); 588789Sahrens } 589789Sahrens 590789Sahrens /* 591789Sahrens * Destructor callback - called when a cached buf is 592789Sahrens * no longer required. 593789Sahrens */ 594789Sahrens /* ARGSUSED */ 595789Sahrens static void 596789Sahrens hdr_dest(void *vbuf, void *unused) 597789Sahrens { 598789Sahrens arc_buf_hdr_t *buf = vbuf; 599789Sahrens 600789Sahrens refcount_destroy(&buf->b_refcnt); 601789Sahrens cv_destroy(&buf->b_cv); 602789Sahrens } 603789Sahrens 604789Sahrens /* 605789Sahrens * Reclaim callback -- invoked when memory is low. 606789Sahrens */ 607789Sahrens /* ARGSUSED */ 608789Sahrens static void 609789Sahrens hdr_recl(void *unused) 610789Sahrens { 611789Sahrens dprintf("hdr_recl called\n"); 6123158Smaybee /* 6133158Smaybee * umem calls the reclaim func when we destroy the buf cache, 6143158Smaybee * which is after we do arc_fini(). 6153158Smaybee */ 6163158Smaybee if (!arc_dead) 6173158Smaybee cv_signal(&arc_reclaim_thr_cv); 618789Sahrens } 619789Sahrens 620789Sahrens static void 621789Sahrens buf_init(void) 622789Sahrens { 623789Sahrens uint64_t *ct; 6241544Seschrock uint64_t hsize = 1ULL << 12; 625789Sahrens int i, j; 626789Sahrens 627789Sahrens /* 628789Sahrens * The hash table is big enough to fill all of physical memory 6291544Seschrock * with an average 64K block size. The table will take up 6301544Seschrock * totalmem*sizeof(void*)/64K (eg. 128KB/GB with 8-byte pointers). 631789Sahrens */ 6321544Seschrock while (hsize * 65536 < physmem * PAGESIZE) 633789Sahrens hsize <<= 1; 6341544Seschrock retry: 635789Sahrens buf_hash_table.ht_mask = hsize - 1; 6361544Seschrock buf_hash_table.ht_table = 6371544Seschrock kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP); 6381544Seschrock if (buf_hash_table.ht_table == NULL) { 6391544Seschrock ASSERT(hsize > (1ULL << 8)); 6401544Seschrock hsize >>= 1; 6411544Seschrock goto retry; 6421544Seschrock } 643789Sahrens 644789Sahrens hdr_cache = kmem_cache_create("arc_buf_hdr_t", sizeof (arc_buf_hdr_t), 645789Sahrens 0, hdr_cons, hdr_dest, hdr_recl, NULL, NULL, 0); 646789Sahrens buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t), 647789Sahrens 0, NULL, NULL, NULL, NULL, NULL, 0); 648789Sahrens 649789Sahrens for (i = 0; i < 256; i++) 650789Sahrens for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--) 651789Sahrens *ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY); 652789Sahrens 653789Sahrens for (i = 0; i < BUF_LOCKS; i++) { 654789Sahrens mutex_init(&buf_hash_table.ht_locks[i].ht_lock, 655789Sahrens NULL, MUTEX_DEFAULT, NULL); 656789Sahrens } 657789Sahrens } 658789Sahrens 659789Sahrens #define ARC_MINTIME (hz>>4) /* 62 ms */ 660789Sahrens 661789Sahrens static void 6623093Sahrens arc_cksum_verify(arc_buf_t *buf) 6633093Sahrens { 6643093Sahrens zio_cksum_t zc; 6653093Sahrens 6663312Sahrens if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 6673093Sahrens return; 6683093Sahrens 6693093Sahrens mutex_enter(&buf->b_hdr->b_freeze_lock); 6703265Sahrens if (buf->b_hdr->b_freeze_cksum == NULL || 6713265Sahrens (buf->b_hdr->b_flags & ARC_IO_ERROR)) { 6723093Sahrens mutex_exit(&buf->b_hdr->b_freeze_lock); 6733093Sahrens return; 6743093Sahrens } 6753093Sahrens fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc); 6763093Sahrens if (!ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc)) 6773093Sahrens panic("buffer modified while frozen!"); 6783093Sahrens mutex_exit(&buf->b_hdr->b_freeze_lock); 6793093Sahrens } 6803093Sahrens 6813093Sahrens static void 6823093Sahrens arc_cksum_compute(arc_buf_t *buf) 6833093Sahrens { 6843312Sahrens if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 6853093Sahrens return; 6863093Sahrens 6873093Sahrens mutex_enter(&buf->b_hdr->b_freeze_lock); 6883093Sahrens if (buf->b_hdr->b_freeze_cksum != NULL) { 6893093Sahrens mutex_exit(&buf->b_hdr->b_freeze_lock); 6903093Sahrens return; 6913093Sahrens } 6923093Sahrens buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP); 6933093Sahrens fletcher_2_native(buf->b_data, buf->b_hdr->b_size, 6943093Sahrens buf->b_hdr->b_freeze_cksum); 6953093Sahrens mutex_exit(&buf->b_hdr->b_freeze_lock); 6963093Sahrens } 6973093Sahrens 6983093Sahrens void 6993093Sahrens arc_buf_thaw(arc_buf_t *buf) 7003093Sahrens { 7013312Sahrens if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 7023093Sahrens return; 7033093Sahrens 7043403Sbmc if (buf->b_hdr->b_state != arc_anon) 7053093Sahrens panic("modifying non-anon buffer!"); 7063093Sahrens if (buf->b_hdr->b_flags & ARC_IO_IN_PROGRESS) 7073093Sahrens panic("modifying buffer while i/o in progress!"); 7083093Sahrens arc_cksum_verify(buf); 7093093Sahrens mutex_enter(&buf->b_hdr->b_freeze_lock); 7103093Sahrens if (buf->b_hdr->b_freeze_cksum != NULL) { 7113093Sahrens kmem_free(buf->b_hdr->b_freeze_cksum, sizeof (zio_cksum_t)); 7123093Sahrens buf->b_hdr->b_freeze_cksum = NULL; 7133093Sahrens } 7143093Sahrens mutex_exit(&buf->b_hdr->b_freeze_lock); 7153093Sahrens } 7163093Sahrens 7173093Sahrens void 7183093Sahrens arc_buf_freeze(arc_buf_t *buf) 7193093Sahrens { 7203312Sahrens if (!(zfs_flags & ZFS_DEBUG_MODIFY)) 7213312Sahrens return; 7223312Sahrens 7233093Sahrens ASSERT(buf->b_hdr->b_freeze_cksum != NULL || 7243403Sbmc buf->b_hdr->b_state == arc_anon); 7253093Sahrens arc_cksum_compute(buf); 7263093Sahrens } 7273093Sahrens 7283093Sahrens static void 729789Sahrens add_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag) 730789Sahrens { 731789Sahrens ASSERT(MUTEX_HELD(hash_lock)); 732789Sahrens 733789Sahrens if ((refcount_add(&ab->b_refcnt, tag) == 1) && 7343403Sbmc (ab->b_state != arc_anon)) { 7353700Sek110237 uint64_t delta = ab->b_size * ab->b_datacnt; 7364309Smaybee list_t *list = &ab->b_state->arcs_list[ab->b_type]; 7374309Smaybee uint64_t *size = &ab->b_state->arcs_lsize[ab->b_type]; 738789Sahrens 7393403Sbmc ASSERT(!MUTEX_HELD(&ab->b_state->arcs_mtx)); 7403403Sbmc mutex_enter(&ab->b_state->arcs_mtx); 741789Sahrens ASSERT(list_link_active(&ab->b_arc_node)); 7424309Smaybee list_remove(list, ab); 7431544Seschrock if (GHOST_STATE(ab->b_state)) { 7441544Seschrock ASSERT3U(ab->b_datacnt, ==, 0); 7451544Seschrock ASSERT3P(ab->b_buf, ==, NULL); 7461544Seschrock delta = ab->b_size; 7471544Seschrock } 7481544Seschrock ASSERT(delta > 0); 7494309Smaybee ASSERT3U(*size, >=, delta); 7504309Smaybee atomic_add_64(size, -delta); 7513403Sbmc mutex_exit(&ab->b_state->arcs_mtx); 7522391Smaybee /* remove the prefetch flag is we get a reference */ 7532391Smaybee if (ab->b_flags & ARC_PREFETCH) 7542391Smaybee ab->b_flags &= ~ARC_PREFETCH; 755789Sahrens } 756789Sahrens } 757789Sahrens 758789Sahrens static int 759789Sahrens remove_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag) 760789Sahrens { 761789Sahrens int cnt; 7623403Sbmc arc_state_t *state = ab->b_state; 763789Sahrens 7643403Sbmc ASSERT(state == arc_anon || MUTEX_HELD(hash_lock)); 7653403Sbmc ASSERT(!GHOST_STATE(state)); 766789Sahrens 767789Sahrens if (((cnt = refcount_remove(&ab->b_refcnt, tag)) == 0) && 7683403Sbmc (state != arc_anon)) { 7694309Smaybee uint64_t *size = &state->arcs_lsize[ab->b_type]; 7704309Smaybee 7713403Sbmc ASSERT(!MUTEX_HELD(&state->arcs_mtx)); 7723403Sbmc mutex_enter(&state->arcs_mtx); 773789Sahrens ASSERT(!list_link_active(&ab->b_arc_node)); 7744309Smaybee list_insert_head(&state->arcs_list[ab->b_type], ab); 7751544Seschrock ASSERT(ab->b_datacnt > 0); 7764309Smaybee atomic_add_64(size, ab->b_size * ab->b_datacnt); 7773403Sbmc mutex_exit(&state->arcs_mtx); 778789Sahrens } 779789Sahrens return (cnt); 780789Sahrens } 781789Sahrens 782789Sahrens /* 783789Sahrens * Move the supplied buffer to the indicated state. The mutex 784789Sahrens * for the buffer must be held by the caller. 785789Sahrens */ 786789Sahrens static void 7871544Seschrock arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *ab, kmutex_t *hash_lock) 788789Sahrens { 7891544Seschrock arc_state_t *old_state = ab->b_state; 7903700Sek110237 int64_t refcnt = refcount_count(&ab->b_refcnt); 7913700Sek110237 uint64_t from_delta, to_delta; 792789Sahrens 793789Sahrens ASSERT(MUTEX_HELD(hash_lock)); 7941544Seschrock ASSERT(new_state != old_state); 7951544Seschrock ASSERT(refcnt == 0 || ab->b_datacnt > 0); 7961544Seschrock ASSERT(ab->b_datacnt == 0 || !GHOST_STATE(new_state)); 7971544Seschrock 7981544Seschrock from_delta = to_delta = ab->b_datacnt * ab->b_size; 799789Sahrens 800789Sahrens /* 801789Sahrens * If this buffer is evictable, transfer it from the 802789Sahrens * old state list to the new state list. 803789Sahrens */ 8041544Seschrock if (refcnt == 0) { 8053403Sbmc if (old_state != arc_anon) { 8063403Sbmc int use_mutex = !MUTEX_HELD(&old_state->arcs_mtx); 8074309Smaybee uint64_t *size = &old_state->arcs_lsize[ab->b_type]; 8081544Seschrock 8091544Seschrock if (use_mutex) 8103403Sbmc mutex_enter(&old_state->arcs_mtx); 8111544Seschrock 8121544Seschrock ASSERT(list_link_active(&ab->b_arc_node)); 8134309Smaybee list_remove(&old_state->arcs_list[ab->b_type], ab); 814789Sahrens 8152391Smaybee /* 8162391Smaybee * If prefetching out of the ghost cache, 8172391Smaybee * we will have a non-null datacnt. 8182391Smaybee */ 8192391Smaybee if (GHOST_STATE(old_state) && ab->b_datacnt == 0) { 8202391Smaybee /* ghost elements have a ghost size */ 8211544Seschrock ASSERT(ab->b_buf == NULL); 8221544Seschrock from_delta = ab->b_size; 823789Sahrens } 8244309Smaybee ASSERT3U(*size, >=, from_delta); 8254309Smaybee atomic_add_64(size, -from_delta); 8261544Seschrock 8271544Seschrock if (use_mutex) 8283403Sbmc mutex_exit(&old_state->arcs_mtx); 829789Sahrens } 8303403Sbmc if (new_state != arc_anon) { 8313403Sbmc int use_mutex = !MUTEX_HELD(&new_state->arcs_mtx); 8324309Smaybee uint64_t *size = &new_state->arcs_lsize[ab->b_type]; 833789Sahrens 8341544Seschrock if (use_mutex) 8353403Sbmc mutex_enter(&new_state->arcs_mtx); 8361544Seschrock 8374309Smaybee list_insert_head(&new_state->arcs_list[ab->b_type], ab); 8381544Seschrock 8391544Seschrock /* ghost elements have a ghost size */ 8401544Seschrock if (GHOST_STATE(new_state)) { 8411544Seschrock ASSERT(ab->b_datacnt == 0); 8421544Seschrock ASSERT(ab->b_buf == NULL); 8431544Seschrock to_delta = ab->b_size; 8441544Seschrock } 8454309Smaybee atomic_add_64(size, to_delta); 8461544Seschrock 8471544Seschrock if (use_mutex) 8483403Sbmc mutex_exit(&new_state->arcs_mtx); 849789Sahrens } 850789Sahrens } 851789Sahrens 852789Sahrens ASSERT(!BUF_EMPTY(ab)); 8533403Sbmc if (new_state == arc_anon && old_state != arc_anon) { 854789Sahrens buf_hash_remove(ab); 855789Sahrens } 856789Sahrens 8571544Seschrock /* adjust state sizes */ 8581544Seschrock if (to_delta) 8593403Sbmc atomic_add_64(&new_state->arcs_size, to_delta); 8601544Seschrock if (from_delta) { 8613403Sbmc ASSERT3U(old_state->arcs_size, >=, from_delta); 8623403Sbmc atomic_add_64(&old_state->arcs_size, -from_delta); 863789Sahrens } 864789Sahrens ab->b_state = new_state; 865789Sahrens } 866789Sahrens 8674309Smaybee void 8684309Smaybee arc_space_consume(uint64_t space) 8694309Smaybee { 8704309Smaybee atomic_add_64(&arc_meta_used, space); 8714309Smaybee atomic_add_64(&arc_size, space); 8724309Smaybee } 8734309Smaybee 8744309Smaybee void 8754309Smaybee arc_space_return(uint64_t space) 8764309Smaybee { 8774309Smaybee ASSERT(arc_meta_used >= space); 8784309Smaybee if (arc_meta_max < arc_meta_used) 8794309Smaybee arc_meta_max = arc_meta_used; 8804309Smaybee atomic_add_64(&arc_meta_used, -space); 8814309Smaybee ASSERT(arc_size >= space); 8824309Smaybee atomic_add_64(&arc_size, -space); 8834309Smaybee } 8844309Smaybee 8854309Smaybee void * 8864309Smaybee arc_data_buf_alloc(uint64_t size) 8874309Smaybee { 8884309Smaybee if (arc_evict_needed(ARC_BUFC_DATA)) 8894309Smaybee cv_signal(&arc_reclaim_thr_cv); 8904309Smaybee atomic_add_64(&arc_size, size); 8914309Smaybee return (zio_data_buf_alloc(size)); 8924309Smaybee } 8934309Smaybee 8944309Smaybee void 8954309Smaybee arc_data_buf_free(void *buf, uint64_t size) 8964309Smaybee { 8974309Smaybee zio_data_buf_free(buf, size); 8984309Smaybee ASSERT(arc_size >= size); 8994309Smaybee atomic_add_64(&arc_size, -size); 9004309Smaybee } 9014309Smaybee 902789Sahrens arc_buf_t * 9033290Sjohansen arc_buf_alloc(spa_t *spa, int size, void *tag, arc_buf_contents_t type) 904789Sahrens { 905789Sahrens arc_buf_hdr_t *hdr; 906789Sahrens arc_buf_t *buf; 907789Sahrens 908789Sahrens ASSERT3U(size, >, 0); 909789Sahrens hdr = kmem_cache_alloc(hdr_cache, KM_SLEEP); 910789Sahrens ASSERT(BUF_EMPTY(hdr)); 911789Sahrens hdr->b_size = size; 9123290Sjohansen hdr->b_type = type; 913789Sahrens hdr->b_spa = spa; 9143403Sbmc hdr->b_state = arc_anon; 915789Sahrens hdr->b_arc_access = 0; 916789Sahrens buf = kmem_cache_alloc(buf_cache, KM_SLEEP); 917789Sahrens buf->b_hdr = hdr; 9182688Smaybee buf->b_data = NULL; 9191544Seschrock buf->b_efunc = NULL; 9201544Seschrock buf->b_private = NULL; 921789Sahrens buf->b_next = NULL; 922789Sahrens hdr->b_buf = buf; 9232688Smaybee arc_get_data_buf(buf); 9241544Seschrock hdr->b_datacnt = 1; 925789Sahrens hdr->b_flags = 0; 926789Sahrens ASSERT(refcount_is_zero(&hdr->b_refcnt)); 927789Sahrens (void) refcount_add(&hdr->b_refcnt, tag); 928789Sahrens 929789Sahrens return (buf); 930789Sahrens } 931789Sahrens 9322688Smaybee static arc_buf_t * 9332688Smaybee arc_buf_clone(arc_buf_t *from) 9341544Seschrock { 9352688Smaybee arc_buf_t *buf; 9362688Smaybee arc_buf_hdr_t *hdr = from->b_hdr; 9372688Smaybee uint64_t size = hdr->b_size; 9381544Seschrock 9392688Smaybee buf = kmem_cache_alloc(buf_cache, KM_SLEEP); 9402688Smaybee buf->b_hdr = hdr; 9412688Smaybee buf->b_data = NULL; 9422688Smaybee buf->b_efunc = NULL; 9432688Smaybee buf->b_private = NULL; 9442688Smaybee buf->b_next = hdr->b_buf; 9452688Smaybee hdr->b_buf = buf; 9462688Smaybee arc_get_data_buf(buf); 9472688Smaybee bcopy(from->b_data, buf->b_data, size); 9482688Smaybee hdr->b_datacnt += 1; 9492688Smaybee return (buf); 9501544Seschrock } 9511544Seschrock 9521544Seschrock void 9531544Seschrock arc_buf_add_ref(arc_buf_t *buf, void* tag) 9541544Seschrock { 9552887Smaybee arc_buf_hdr_t *hdr; 9561544Seschrock kmutex_t *hash_lock; 9571544Seschrock 9582724Smaybee /* 9592724Smaybee * Check to see if this buffer is currently being evicted via 9602887Smaybee * arc_do_user_evicts(). 9612724Smaybee */ 9622887Smaybee mutex_enter(&arc_eviction_mtx); 9632887Smaybee hdr = buf->b_hdr; 9642887Smaybee if (hdr == NULL) { 9652887Smaybee mutex_exit(&arc_eviction_mtx); 9662724Smaybee return; 9672887Smaybee } 9682887Smaybee hash_lock = HDR_LOCK(hdr); 9692887Smaybee mutex_exit(&arc_eviction_mtx); 9702724Smaybee 9712724Smaybee mutex_enter(hash_lock); 9721544Seschrock if (buf->b_data == NULL) { 9731544Seschrock /* 9741544Seschrock * This buffer is evicted. 9751544Seschrock */ 9762724Smaybee mutex_exit(hash_lock); 9771544Seschrock return; 9781544Seschrock } 9791544Seschrock 9802724Smaybee ASSERT(buf->b_hdr == hdr); 9813403Sbmc ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 9821544Seschrock add_reference(hdr, hash_lock, tag); 9832688Smaybee arc_access(hdr, hash_lock); 9842688Smaybee mutex_exit(hash_lock); 9853403Sbmc ARCSTAT_BUMP(arcstat_hits); 9863403Sbmc ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 9873403Sbmc demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 9883403Sbmc data, metadata, hits); 9891544Seschrock } 9901544Seschrock 991789Sahrens static void 9922688Smaybee arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t all) 9931544Seschrock { 9941544Seschrock arc_buf_t **bufp; 9951544Seschrock 9961544Seschrock /* free up data associated with the buf */ 9971544Seschrock if (buf->b_data) { 9981544Seschrock arc_state_t *state = buf->b_hdr->b_state; 9991544Seschrock uint64_t size = buf->b_hdr->b_size; 10003290Sjohansen arc_buf_contents_t type = buf->b_hdr->b_type; 10011544Seschrock 10023093Sahrens arc_cksum_verify(buf); 10032688Smaybee if (!recycle) { 10043290Sjohansen if (type == ARC_BUFC_METADATA) { 10053290Sjohansen zio_buf_free(buf->b_data, size); 10064309Smaybee arc_space_return(size); 10073290Sjohansen } else { 10083290Sjohansen ASSERT(type == ARC_BUFC_DATA); 10093290Sjohansen zio_data_buf_free(buf->b_data, size); 10104309Smaybee atomic_add_64(&arc_size, -size); 10113290Sjohansen } 10122688Smaybee } 10131544Seschrock if (list_link_active(&buf->b_hdr->b_arc_node)) { 10144309Smaybee uint64_t *cnt = &state->arcs_lsize[type]; 10154309Smaybee 10161544Seschrock ASSERT(refcount_is_zero(&buf->b_hdr->b_refcnt)); 10173403Sbmc ASSERT(state != arc_anon); 10184309Smaybee 10194309Smaybee ASSERT3U(*cnt, >=, size); 10204309Smaybee atomic_add_64(cnt, -size); 10211544Seschrock } 10223403Sbmc ASSERT3U(state->arcs_size, >=, size); 10233403Sbmc atomic_add_64(&state->arcs_size, -size); 10241544Seschrock buf->b_data = NULL; 10251544Seschrock ASSERT(buf->b_hdr->b_datacnt > 0); 10261544Seschrock buf->b_hdr->b_datacnt -= 1; 10271544Seschrock } 10281544Seschrock 10291544Seschrock /* only remove the buf if requested */ 10301544Seschrock if (!all) 10311544Seschrock return; 10321544Seschrock 10331544Seschrock /* remove the buf from the hdr list */ 10341544Seschrock for (bufp = &buf->b_hdr->b_buf; *bufp != buf; bufp = &(*bufp)->b_next) 10351544Seschrock continue; 10361544Seschrock *bufp = buf->b_next; 10371544Seschrock 10381544Seschrock ASSERT(buf->b_efunc == NULL); 10391544Seschrock 10401544Seschrock /* clean up the buf */ 10411544Seschrock buf->b_hdr = NULL; 10421544Seschrock kmem_cache_free(buf_cache, buf); 10431544Seschrock } 10441544Seschrock 10451544Seschrock static void 10461544Seschrock arc_hdr_destroy(arc_buf_hdr_t *hdr) 1047789Sahrens { 1048789Sahrens ASSERT(refcount_is_zero(&hdr->b_refcnt)); 10493403Sbmc ASSERT3P(hdr->b_state, ==, arc_anon); 10501544Seschrock ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 1051789Sahrens 1052789Sahrens if (!BUF_EMPTY(hdr)) { 10531544Seschrock ASSERT(!HDR_IN_HASH_TABLE(hdr)); 1054789Sahrens bzero(&hdr->b_dva, sizeof (dva_t)); 1055789Sahrens hdr->b_birth = 0; 1056789Sahrens hdr->b_cksum0 = 0; 1057789Sahrens } 10581544Seschrock while (hdr->b_buf) { 1059789Sahrens arc_buf_t *buf = hdr->b_buf; 1060789Sahrens 10611544Seschrock if (buf->b_efunc) { 10621544Seschrock mutex_enter(&arc_eviction_mtx); 10631544Seschrock ASSERT(buf->b_hdr != NULL); 10642688Smaybee arc_buf_destroy(hdr->b_buf, FALSE, FALSE); 10651544Seschrock hdr->b_buf = buf->b_next; 10662887Smaybee buf->b_hdr = &arc_eviction_hdr; 10671544Seschrock buf->b_next = arc_eviction_list; 10681544Seschrock arc_eviction_list = buf; 10691544Seschrock mutex_exit(&arc_eviction_mtx); 10701544Seschrock } else { 10712688Smaybee arc_buf_destroy(hdr->b_buf, FALSE, TRUE); 10721544Seschrock } 1073789Sahrens } 10743093Sahrens if (hdr->b_freeze_cksum != NULL) { 10753093Sahrens kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t)); 10763093Sahrens hdr->b_freeze_cksum = NULL; 10773093Sahrens } 10781544Seschrock 1079789Sahrens ASSERT(!list_link_active(&hdr->b_arc_node)); 1080789Sahrens ASSERT3P(hdr->b_hash_next, ==, NULL); 1081789Sahrens ASSERT3P(hdr->b_acb, ==, NULL); 1082789Sahrens kmem_cache_free(hdr_cache, hdr); 1083789Sahrens } 1084789Sahrens 1085789Sahrens void 1086789Sahrens arc_buf_free(arc_buf_t *buf, void *tag) 1087789Sahrens { 1088789Sahrens arc_buf_hdr_t *hdr = buf->b_hdr; 10893403Sbmc int hashed = hdr->b_state != arc_anon; 10901544Seschrock 10911544Seschrock ASSERT(buf->b_efunc == NULL); 10921544Seschrock ASSERT(buf->b_data != NULL); 10931544Seschrock 10941544Seschrock if (hashed) { 10951544Seschrock kmutex_t *hash_lock = HDR_LOCK(hdr); 10961544Seschrock 10971544Seschrock mutex_enter(hash_lock); 10981544Seschrock (void) remove_reference(hdr, hash_lock, tag); 10991544Seschrock if (hdr->b_datacnt > 1) 11002688Smaybee arc_buf_destroy(buf, FALSE, TRUE); 11011544Seschrock else 11021544Seschrock hdr->b_flags |= ARC_BUF_AVAILABLE; 11031544Seschrock mutex_exit(hash_lock); 11041544Seschrock } else if (HDR_IO_IN_PROGRESS(hdr)) { 11051544Seschrock int destroy_hdr; 11061544Seschrock /* 11071544Seschrock * We are in the middle of an async write. Don't destroy 11081544Seschrock * this buffer unless the write completes before we finish 11091544Seschrock * decrementing the reference count. 11101544Seschrock */ 11111544Seschrock mutex_enter(&arc_eviction_mtx); 11121544Seschrock (void) remove_reference(hdr, NULL, tag); 11131544Seschrock ASSERT(refcount_is_zero(&hdr->b_refcnt)); 11141544Seschrock destroy_hdr = !HDR_IO_IN_PROGRESS(hdr); 11151544Seschrock mutex_exit(&arc_eviction_mtx); 11161544Seschrock if (destroy_hdr) 11171544Seschrock arc_hdr_destroy(hdr); 11181544Seschrock } else { 11191544Seschrock if (remove_reference(hdr, NULL, tag) > 0) { 11201544Seschrock ASSERT(HDR_IO_ERROR(hdr)); 11212688Smaybee arc_buf_destroy(buf, FALSE, TRUE); 11221544Seschrock } else { 11231544Seschrock arc_hdr_destroy(hdr); 11241544Seschrock } 11251544Seschrock } 11261544Seschrock } 11271544Seschrock 11281544Seschrock int 11291544Seschrock arc_buf_remove_ref(arc_buf_t *buf, void* tag) 11301544Seschrock { 11311544Seschrock arc_buf_hdr_t *hdr = buf->b_hdr; 1132789Sahrens kmutex_t *hash_lock = HDR_LOCK(hdr); 11331544Seschrock int no_callback = (buf->b_efunc == NULL); 11341544Seschrock 11353403Sbmc if (hdr->b_state == arc_anon) { 11361544Seschrock arc_buf_free(buf, tag); 11371544Seschrock return (no_callback); 11381544Seschrock } 1139789Sahrens 1140789Sahrens mutex_enter(hash_lock); 11413403Sbmc ASSERT(hdr->b_state != arc_anon); 11421544Seschrock ASSERT(buf->b_data != NULL); 1143789Sahrens 11441544Seschrock (void) remove_reference(hdr, hash_lock, tag); 11451544Seschrock if (hdr->b_datacnt > 1) { 11461544Seschrock if (no_callback) 11472688Smaybee arc_buf_destroy(buf, FALSE, TRUE); 11481544Seschrock } else if (no_callback) { 11491544Seschrock ASSERT(hdr->b_buf == buf && buf->b_next == NULL); 11501544Seschrock hdr->b_flags |= ARC_BUF_AVAILABLE; 1151789Sahrens } 11521544Seschrock ASSERT(no_callback || hdr->b_datacnt > 1 || 11531544Seschrock refcount_is_zero(&hdr->b_refcnt)); 1154789Sahrens mutex_exit(hash_lock); 11551544Seschrock return (no_callback); 1156789Sahrens } 1157789Sahrens 1158789Sahrens int 1159789Sahrens arc_buf_size(arc_buf_t *buf) 1160789Sahrens { 1161789Sahrens return (buf->b_hdr->b_size); 1162789Sahrens } 1163789Sahrens 1164789Sahrens /* 1165789Sahrens * Evict buffers from list until we've removed the specified number of 1166789Sahrens * bytes. Move the removed buffers to the appropriate evict state. 11672688Smaybee * If the recycle flag is set, then attempt to "recycle" a buffer: 11682688Smaybee * - look for a buffer to evict that is `bytes' long. 11692688Smaybee * - return the data block from this buffer rather than freeing it. 11702688Smaybee * This flag is used by callers that are trying to make space for a 11712688Smaybee * new buffer in a full arc cache. 1172789Sahrens */ 11732688Smaybee static void * 11743290Sjohansen arc_evict(arc_state_t *state, int64_t bytes, boolean_t recycle, 11753290Sjohansen arc_buf_contents_t type) 1176789Sahrens { 1177789Sahrens arc_state_t *evicted_state; 11782688Smaybee uint64_t bytes_evicted = 0, skipped = 0, missed = 0; 11792918Smaybee arc_buf_hdr_t *ab, *ab_prev = NULL; 11804309Smaybee list_t *list = &state->arcs_list[type]; 1181789Sahrens kmutex_t *hash_lock; 11822688Smaybee boolean_t have_lock; 11832918Smaybee void *stolen = NULL; 1184789Sahrens 11853403Sbmc ASSERT(state == arc_mru || state == arc_mfu); 1186789Sahrens 11873403Sbmc evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost; 1188789Sahrens 11893403Sbmc mutex_enter(&state->arcs_mtx); 11903403Sbmc mutex_enter(&evicted_state->arcs_mtx); 1191789Sahrens 11924309Smaybee for (ab = list_tail(list); ab; ab = ab_prev) { 11934309Smaybee ab_prev = list_prev(list, ab); 11942391Smaybee /* prefetch buffers have a minimum lifespan */ 11952688Smaybee if (HDR_IO_IN_PROGRESS(ab) || 11962688Smaybee (ab->b_flags & (ARC_PREFETCH|ARC_INDIRECT) && 11972688Smaybee lbolt - ab->b_arc_access < arc_min_prefetch_lifespan)) { 11982391Smaybee skipped++; 11992391Smaybee continue; 12002391Smaybee } 12012918Smaybee /* "lookahead" for better eviction candidate */ 12022918Smaybee if (recycle && ab->b_size != bytes && 12032918Smaybee ab_prev && ab_prev->b_size == bytes) 12042688Smaybee continue; 1205789Sahrens hash_lock = HDR_LOCK(ab); 12062688Smaybee have_lock = MUTEX_HELD(hash_lock); 12072688Smaybee if (have_lock || mutex_tryenter(hash_lock)) { 1208789Sahrens ASSERT3U(refcount_count(&ab->b_refcnt), ==, 0); 12091544Seschrock ASSERT(ab->b_datacnt > 0); 12101544Seschrock while (ab->b_buf) { 12111544Seschrock arc_buf_t *buf = ab->b_buf; 12122688Smaybee if (buf->b_data) { 12131544Seschrock bytes_evicted += ab->b_size; 12143290Sjohansen if (recycle && ab->b_type == type && 12153290Sjohansen ab->b_size == bytes) { 12162918Smaybee stolen = buf->b_data; 12172918Smaybee recycle = FALSE; 12182918Smaybee } 12192688Smaybee } 12201544Seschrock if (buf->b_efunc) { 12211544Seschrock mutex_enter(&arc_eviction_mtx); 12222918Smaybee arc_buf_destroy(buf, 12232918Smaybee buf->b_data == stolen, FALSE); 12241544Seschrock ab->b_buf = buf->b_next; 12252887Smaybee buf->b_hdr = &arc_eviction_hdr; 12261544Seschrock buf->b_next = arc_eviction_list; 12271544Seschrock arc_eviction_list = buf; 12281544Seschrock mutex_exit(&arc_eviction_mtx); 12291544Seschrock } else { 12302918Smaybee arc_buf_destroy(buf, 12312918Smaybee buf->b_data == stolen, TRUE); 12321544Seschrock } 12331544Seschrock } 12341544Seschrock ASSERT(ab->b_datacnt == 0); 1235789Sahrens arc_change_state(evicted_state, ab, hash_lock); 12361544Seschrock ASSERT(HDR_IN_HASH_TABLE(ab)); 12371544Seschrock ab->b_flags = ARC_IN_HASH_TABLE; 1238789Sahrens DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, ab); 12392688Smaybee if (!have_lock) 12402688Smaybee mutex_exit(hash_lock); 12411544Seschrock if (bytes >= 0 && bytes_evicted >= bytes) 1242789Sahrens break; 1243789Sahrens } else { 12442688Smaybee missed += 1; 1245789Sahrens } 1246789Sahrens } 12473403Sbmc 12483403Sbmc mutex_exit(&evicted_state->arcs_mtx); 12493403Sbmc mutex_exit(&state->arcs_mtx); 1250789Sahrens 1251789Sahrens if (bytes_evicted < bytes) 1252789Sahrens dprintf("only evicted %lld bytes from %x", 1253789Sahrens (longlong_t)bytes_evicted, state); 1254789Sahrens 12552688Smaybee if (skipped) 12563403Sbmc ARCSTAT_INCR(arcstat_evict_skip, skipped); 12573403Sbmc 12582688Smaybee if (missed) 12593403Sbmc ARCSTAT_INCR(arcstat_mutex_miss, missed); 12603403Sbmc 1261*4709Smaybee /* 1262*4709Smaybee * We have just evicted some date into the ghost state, make 1263*4709Smaybee * sure we also adjust the ghost state size if necessary. 1264*4709Smaybee */ 1265*4709Smaybee if (arc_no_grow && 1266*4709Smaybee arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size > arc_c) { 1267*4709Smaybee int64_t mru_over = arc_anon->arcs_size + arc_mru->arcs_size + 1268*4709Smaybee arc_mru_ghost->arcs_size - arc_c; 1269*4709Smaybee 1270*4709Smaybee if (mru_over > 0 && arc_mru_ghost->arcs_lsize[type] > 0) { 1271*4709Smaybee int64_t todelete = 1272*4709Smaybee MIN(arc_mru_ghost->arcs_lsize[type], mru_over); 1273*4709Smaybee arc_evict_ghost(arc_mru_ghost, todelete); 1274*4709Smaybee } else if (arc_mfu_ghost->arcs_lsize[type] > 0) { 1275*4709Smaybee int64_t todelete = MIN(arc_mfu_ghost->arcs_lsize[type], 1276*4709Smaybee arc_mru_ghost->arcs_size + 1277*4709Smaybee arc_mfu_ghost->arcs_size - arc_c); 1278*4709Smaybee arc_evict_ghost(arc_mfu_ghost, todelete); 1279*4709Smaybee } 1280*4709Smaybee } 1281*4709Smaybee 12822918Smaybee return (stolen); 1283789Sahrens } 1284789Sahrens 1285789Sahrens /* 1286789Sahrens * Remove buffers from list until we've removed the specified number of 1287789Sahrens * bytes. Destroy the buffers that are removed. 1288789Sahrens */ 1289789Sahrens static void 12901544Seschrock arc_evict_ghost(arc_state_t *state, int64_t bytes) 1291789Sahrens { 1292789Sahrens arc_buf_hdr_t *ab, *ab_prev; 12934309Smaybee list_t *list = &state->arcs_list[ARC_BUFC_DATA]; 1294789Sahrens kmutex_t *hash_lock; 12951544Seschrock uint64_t bytes_deleted = 0; 12963700Sek110237 uint64_t bufs_skipped = 0; 1297789Sahrens 12981544Seschrock ASSERT(GHOST_STATE(state)); 1299789Sahrens top: 13003403Sbmc mutex_enter(&state->arcs_mtx); 13014309Smaybee for (ab = list_tail(list); ab; ab = ab_prev) { 13024309Smaybee ab_prev = list_prev(list, ab); 1303789Sahrens hash_lock = HDR_LOCK(ab); 1304789Sahrens if (mutex_tryenter(hash_lock)) { 13052391Smaybee ASSERT(!HDR_IO_IN_PROGRESS(ab)); 13061544Seschrock ASSERT(ab->b_buf == NULL); 13073403Sbmc arc_change_state(arc_anon, ab, hash_lock); 1308789Sahrens mutex_exit(hash_lock); 13093403Sbmc ARCSTAT_BUMP(arcstat_deleted); 13101544Seschrock bytes_deleted += ab->b_size; 13111544Seschrock arc_hdr_destroy(ab); 1312789Sahrens DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, ab); 1313789Sahrens if (bytes >= 0 && bytes_deleted >= bytes) 1314789Sahrens break; 1315789Sahrens } else { 1316789Sahrens if (bytes < 0) { 13173403Sbmc mutex_exit(&state->arcs_mtx); 1318789Sahrens mutex_enter(hash_lock); 1319789Sahrens mutex_exit(hash_lock); 1320789Sahrens goto top; 1321789Sahrens } 1322789Sahrens bufs_skipped += 1; 1323789Sahrens } 1324789Sahrens } 13253403Sbmc mutex_exit(&state->arcs_mtx); 1326789Sahrens 13274309Smaybee if (list == &state->arcs_list[ARC_BUFC_DATA] && 13284309Smaybee (bytes < 0 || bytes_deleted < bytes)) { 13294309Smaybee list = &state->arcs_list[ARC_BUFC_METADATA]; 13304309Smaybee goto top; 13314309Smaybee } 13324309Smaybee 1333789Sahrens if (bufs_skipped) { 13343403Sbmc ARCSTAT_INCR(arcstat_mutex_miss, bufs_skipped); 1335789Sahrens ASSERT(bytes >= 0); 1336789Sahrens } 1337789Sahrens 1338789Sahrens if (bytes_deleted < bytes) 1339789Sahrens dprintf("only deleted %lld bytes from %p", 1340789Sahrens (longlong_t)bytes_deleted, state); 1341789Sahrens } 1342789Sahrens 1343789Sahrens static void 1344789Sahrens arc_adjust(void) 1345789Sahrens { 13463403Sbmc int64_t top_sz, mru_over, arc_over, todelete; 1347789Sahrens 13483403Sbmc top_sz = arc_anon->arcs_size + arc_mru->arcs_size; 1349789Sahrens 13504309Smaybee if (top_sz > arc_p && arc_mru->arcs_lsize[ARC_BUFC_DATA] > 0) { 13514309Smaybee int64_t toevict = 13524309Smaybee MIN(arc_mru->arcs_lsize[ARC_BUFC_DATA], top_sz - arc_p); 13534309Smaybee (void) arc_evict(arc_mru, toevict, FALSE, ARC_BUFC_DATA); 13544309Smaybee top_sz = arc_anon->arcs_size + arc_mru->arcs_size; 13554309Smaybee } 13564309Smaybee 13574309Smaybee if (top_sz > arc_p && arc_mru->arcs_lsize[ARC_BUFC_METADATA] > 0) { 13584309Smaybee int64_t toevict = 13594309Smaybee MIN(arc_mru->arcs_lsize[ARC_BUFC_METADATA], top_sz - arc_p); 13604309Smaybee (void) arc_evict(arc_mru, toevict, FALSE, ARC_BUFC_METADATA); 13613403Sbmc top_sz = arc_anon->arcs_size + arc_mru->arcs_size; 1362789Sahrens } 1363789Sahrens 13643403Sbmc mru_over = top_sz + arc_mru_ghost->arcs_size - arc_c; 1365789Sahrens 1366789Sahrens if (mru_over > 0) { 13674309Smaybee if (arc_mru_ghost->arcs_size > 0) { 13684309Smaybee todelete = MIN(arc_mru_ghost->arcs_size, mru_over); 13693403Sbmc arc_evict_ghost(arc_mru_ghost, todelete); 1370789Sahrens } 1371789Sahrens } 1372789Sahrens 13733403Sbmc if ((arc_over = arc_size - arc_c) > 0) { 13741544Seschrock int64_t tbl_over; 1375789Sahrens 13764309Smaybee if (arc_mfu->arcs_lsize[ARC_BUFC_DATA] > 0) { 13774309Smaybee int64_t toevict = 13784309Smaybee MIN(arc_mfu->arcs_lsize[ARC_BUFC_DATA], arc_over); 13793403Sbmc (void) arc_evict(arc_mfu, toevict, FALSE, 13804309Smaybee ARC_BUFC_DATA); 13814309Smaybee arc_over = arc_size - arc_c; 1382789Sahrens } 1383789Sahrens 13844309Smaybee if (arc_over > 0 && 13854309Smaybee arc_mfu->arcs_lsize[ARC_BUFC_METADATA] > 0) { 13864309Smaybee int64_t toevict = 13874309Smaybee MIN(arc_mfu->arcs_lsize[ARC_BUFC_METADATA], 13884309Smaybee arc_over); 13894309Smaybee (void) arc_evict(arc_mfu, toevict, FALSE, 13904309Smaybee ARC_BUFC_METADATA); 13914309Smaybee } 13924309Smaybee 13934309Smaybee tbl_over = arc_size + arc_mru_ghost->arcs_size + 13944309Smaybee arc_mfu_ghost->arcs_size - arc_c * 2; 13954309Smaybee 13964309Smaybee if (tbl_over > 0 && arc_mfu_ghost->arcs_size > 0) { 13974309Smaybee todelete = MIN(arc_mfu_ghost->arcs_size, tbl_over); 13983403Sbmc arc_evict_ghost(arc_mfu_ghost, todelete); 1399789Sahrens } 1400789Sahrens } 1401789Sahrens } 1402789Sahrens 14031544Seschrock static void 14041544Seschrock arc_do_user_evicts(void) 14051544Seschrock { 14061544Seschrock mutex_enter(&arc_eviction_mtx); 14071544Seschrock while (arc_eviction_list != NULL) { 14081544Seschrock arc_buf_t *buf = arc_eviction_list; 14091544Seschrock arc_eviction_list = buf->b_next; 14101544Seschrock buf->b_hdr = NULL; 14111544Seschrock mutex_exit(&arc_eviction_mtx); 14121544Seschrock 14131819Smaybee if (buf->b_efunc != NULL) 14141819Smaybee VERIFY(buf->b_efunc(buf) == 0); 14151544Seschrock 14161544Seschrock buf->b_efunc = NULL; 14171544Seschrock buf->b_private = NULL; 14181544Seschrock kmem_cache_free(buf_cache, buf); 14191544Seschrock mutex_enter(&arc_eviction_mtx); 14201544Seschrock } 14211544Seschrock mutex_exit(&arc_eviction_mtx); 14221544Seschrock } 14231544Seschrock 1424789Sahrens /* 1425789Sahrens * Flush all *evictable* data from the cache. 1426789Sahrens * NOTE: this will not touch "active" (i.e. referenced) data. 1427789Sahrens */ 1428789Sahrens void 1429789Sahrens arc_flush(void) 1430789Sahrens { 14314309Smaybee while (list_head(&arc_mru->arcs_list[ARC_BUFC_DATA])) 14324309Smaybee (void) arc_evict(arc_mru, -1, FALSE, ARC_BUFC_DATA); 14334309Smaybee while (list_head(&arc_mru->arcs_list[ARC_BUFC_METADATA])) 14344309Smaybee (void) arc_evict(arc_mru, -1, FALSE, ARC_BUFC_METADATA); 14354309Smaybee while (list_head(&arc_mfu->arcs_list[ARC_BUFC_DATA])) 14364309Smaybee (void) arc_evict(arc_mfu, -1, FALSE, ARC_BUFC_DATA); 14374309Smaybee while (list_head(&arc_mfu->arcs_list[ARC_BUFC_METADATA])) 14384309Smaybee (void) arc_evict(arc_mfu, -1, FALSE, ARC_BUFC_METADATA); 1439789Sahrens 14403403Sbmc arc_evict_ghost(arc_mru_ghost, -1); 14413403Sbmc arc_evict_ghost(arc_mfu_ghost, -1); 14421544Seschrock 14431544Seschrock mutex_enter(&arc_reclaim_thr_lock); 14441544Seschrock arc_do_user_evicts(); 14451544Seschrock mutex_exit(&arc_reclaim_thr_lock); 14461544Seschrock ASSERT(arc_eviction_list == NULL); 1447789Sahrens } 1448789Sahrens 14493158Smaybee int arc_shrink_shift = 5; /* log2(fraction of arc to reclaim) */ 14502391Smaybee 1451789Sahrens void 14523158Smaybee arc_shrink(void) 1453789Sahrens { 14543403Sbmc if (arc_c > arc_c_min) { 14553158Smaybee uint64_t to_free; 1456789Sahrens 14572048Sstans #ifdef _KERNEL 14583403Sbmc to_free = MAX(arc_c >> arc_shrink_shift, ptob(needfree)); 14592048Sstans #else 14603403Sbmc to_free = arc_c >> arc_shrink_shift; 14612048Sstans #endif 14623403Sbmc if (arc_c > arc_c_min + to_free) 14633403Sbmc atomic_add_64(&arc_c, -to_free); 14643158Smaybee else 14653403Sbmc arc_c = arc_c_min; 14662048Sstans 14673403Sbmc atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift)); 14683403Sbmc if (arc_c > arc_size) 14693403Sbmc arc_c = MAX(arc_size, arc_c_min); 14703403Sbmc if (arc_p > arc_c) 14713403Sbmc arc_p = (arc_c >> 1); 14723403Sbmc ASSERT(arc_c >= arc_c_min); 14733403Sbmc ASSERT((int64_t)arc_p >= 0); 14743158Smaybee } 1475789Sahrens 14763403Sbmc if (arc_size > arc_c) 14773158Smaybee arc_adjust(); 1478789Sahrens } 1479789Sahrens 1480789Sahrens static int 1481789Sahrens arc_reclaim_needed(void) 1482789Sahrens { 1483789Sahrens uint64_t extra; 1484789Sahrens 1485789Sahrens #ifdef _KERNEL 14862048Sstans 14872048Sstans if (needfree) 14882048Sstans return (1); 14892048Sstans 1490789Sahrens /* 1491789Sahrens * take 'desfree' extra pages, so we reclaim sooner, rather than later 1492789Sahrens */ 1493789Sahrens extra = desfree; 1494789Sahrens 1495789Sahrens /* 1496789Sahrens * check that we're out of range of the pageout scanner. It starts to 1497789Sahrens * schedule paging if freemem is less than lotsfree and needfree. 1498789Sahrens * lotsfree is the high-water mark for pageout, and needfree is the 1499789Sahrens * number of needed free pages. We add extra pages here to make sure 1500789Sahrens * the scanner doesn't start up while we're freeing memory. 1501789Sahrens */ 1502789Sahrens if (freemem < lotsfree + needfree + extra) 1503789Sahrens return (1); 1504789Sahrens 1505789Sahrens /* 1506789Sahrens * check to make sure that swapfs has enough space so that anon 1507789Sahrens * reservations can still succeeed. anon_resvmem() checks that the 1508789Sahrens * availrmem is greater than swapfs_minfree, and the number of reserved 1509789Sahrens * swap pages. We also add a bit of extra here just to prevent 1510789Sahrens * circumstances from getting really dire. 1511789Sahrens */ 1512789Sahrens if (availrmem < swapfs_minfree + swapfs_reserve + extra) 1513789Sahrens return (1); 1514789Sahrens 15151936Smaybee #if defined(__i386) 1516789Sahrens /* 1517789Sahrens * If we're on an i386 platform, it's possible that we'll exhaust the 1518789Sahrens * kernel heap space before we ever run out of available physical 1519789Sahrens * memory. Most checks of the size of the heap_area compare against 1520789Sahrens * tune.t_minarmem, which is the minimum available real memory that we 1521789Sahrens * can have in the system. However, this is generally fixed at 25 pages 1522789Sahrens * which is so low that it's useless. In this comparison, we seek to 1523789Sahrens * calculate the total heap-size, and reclaim if more than 3/4ths of the 1524789Sahrens * heap is allocated. (Or, in the caclulation, if less than 1/4th is 1525789Sahrens * free) 1526789Sahrens */ 1527789Sahrens if (btop(vmem_size(heap_arena, VMEM_FREE)) < 1528789Sahrens (btop(vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC)) >> 2)) 1529789Sahrens return (1); 1530789Sahrens #endif 1531789Sahrens 1532789Sahrens #else 1533789Sahrens if (spa_get_random(100) == 0) 1534789Sahrens return (1); 1535789Sahrens #endif 1536789Sahrens return (0); 1537789Sahrens } 1538789Sahrens 1539789Sahrens static void 1540789Sahrens arc_kmem_reap_now(arc_reclaim_strategy_t strat) 1541789Sahrens { 1542789Sahrens size_t i; 1543789Sahrens kmem_cache_t *prev_cache = NULL; 15443290Sjohansen kmem_cache_t *prev_data_cache = NULL; 1545789Sahrens extern kmem_cache_t *zio_buf_cache[]; 15463290Sjohansen extern kmem_cache_t *zio_data_buf_cache[]; 1547789Sahrens 15481484Sek110237 #ifdef _KERNEL 15494309Smaybee if (arc_meta_used >= arc_meta_limit) { 15504309Smaybee /* 15514309Smaybee * We are exceeding our meta-data cache limit. 15524309Smaybee * Purge some DNLC entries to release holds on meta-data. 15534309Smaybee */ 15544309Smaybee dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent); 15554309Smaybee } 15561936Smaybee #if defined(__i386) 15571936Smaybee /* 15581936Smaybee * Reclaim unused memory from all kmem caches. 15591936Smaybee */ 15601936Smaybee kmem_reap(); 15611936Smaybee #endif 15621484Sek110237 #endif 15631484Sek110237 1564789Sahrens /* 15651544Seschrock * An agressive reclamation will shrink the cache size as well as 15661544Seschrock * reap free buffers from the arc kmem caches. 1567789Sahrens */ 1568789Sahrens if (strat == ARC_RECLAIM_AGGR) 15693158Smaybee arc_shrink(); 1570789Sahrens 1571789Sahrens for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) { 1572789Sahrens if (zio_buf_cache[i] != prev_cache) { 1573789Sahrens prev_cache = zio_buf_cache[i]; 1574789Sahrens kmem_cache_reap_now(zio_buf_cache[i]); 1575789Sahrens } 15763290Sjohansen if (zio_data_buf_cache[i] != prev_data_cache) { 15773290Sjohansen prev_data_cache = zio_data_buf_cache[i]; 15783290Sjohansen kmem_cache_reap_now(zio_data_buf_cache[i]); 15793290Sjohansen } 1580789Sahrens } 15811544Seschrock kmem_cache_reap_now(buf_cache); 15821544Seschrock kmem_cache_reap_now(hdr_cache); 1583789Sahrens } 1584789Sahrens 1585789Sahrens static void 1586789Sahrens arc_reclaim_thread(void) 1587789Sahrens { 1588789Sahrens clock_t growtime = 0; 1589789Sahrens arc_reclaim_strategy_t last_reclaim = ARC_RECLAIM_CONS; 1590789Sahrens callb_cpr_t cpr; 1591789Sahrens 1592789Sahrens CALLB_CPR_INIT(&cpr, &arc_reclaim_thr_lock, callb_generic_cpr, FTAG); 1593789Sahrens 1594789Sahrens mutex_enter(&arc_reclaim_thr_lock); 1595789Sahrens while (arc_thread_exit == 0) { 1596789Sahrens if (arc_reclaim_needed()) { 1597789Sahrens 15983403Sbmc if (arc_no_grow) { 1599789Sahrens if (last_reclaim == ARC_RECLAIM_CONS) { 1600789Sahrens last_reclaim = ARC_RECLAIM_AGGR; 1601789Sahrens } else { 1602789Sahrens last_reclaim = ARC_RECLAIM_CONS; 1603789Sahrens } 1604789Sahrens } else { 16053403Sbmc arc_no_grow = TRUE; 1606789Sahrens last_reclaim = ARC_RECLAIM_AGGR; 1607789Sahrens membar_producer(); 1608789Sahrens } 1609789Sahrens 1610789Sahrens /* reset the growth delay for every reclaim */ 1611789Sahrens growtime = lbolt + (arc_grow_retry * hz); 1612789Sahrens 1613789Sahrens arc_kmem_reap_now(last_reclaim); 1614789Sahrens 16154309Smaybee } else if (arc_no_grow && lbolt >= growtime) { 16163403Sbmc arc_no_grow = FALSE; 1617789Sahrens } 1618789Sahrens 16193403Sbmc if (2 * arc_c < arc_size + 16203403Sbmc arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size) 16213298Smaybee arc_adjust(); 16223298Smaybee 16231544Seschrock if (arc_eviction_list != NULL) 16241544Seschrock arc_do_user_evicts(); 16251544Seschrock 1626789Sahrens /* block until needed, or one second, whichever is shorter */ 1627789Sahrens CALLB_CPR_SAFE_BEGIN(&cpr); 1628789Sahrens (void) cv_timedwait(&arc_reclaim_thr_cv, 1629789Sahrens &arc_reclaim_thr_lock, (lbolt + hz)); 1630789Sahrens CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_thr_lock); 1631789Sahrens } 1632789Sahrens 1633789Sahrens arc_thread_exit = 0; 1634789Sahrens cv_broadcast(&arc_reclaim_thr_cv); 1635789Sahrens CALLB_CPR_EXIT(&cpr); /* drops arc_reclaim_thr_lock */ 1636789Sahrens thread_exit(); 1637789Sahrens } 1638789Sahrens 16391544Seschrock /* 16401544Seschrock * Adapt arc info given the number of bytes we are trying to add and 16411544Seschrock * the state that we are comming from. This function is only called 16421544Seschrock * when we are adding new content to the cache. 16431544Seschrock */ 1644789Sahrens static void 16451544Seschrock arc_adapt(int bytes, arc_state_t *state) 1646789Sahrens { 16471544Seschrock int mult; 16481544Seschrock 16491544Seschrock ASSERT(bytes > 0); 1650789Sahrens /* 16511544Seschrock * Adapt the target size of the MRU list: 16521544Seschrock * - if we just hit in the MRU ghost list, then increase 16531544Seschrock * the target size of the MRU list. 16541544Seschrock * - if we just hit in the MFU ghost list, then increase 16551544Seschrock * the target size of the MFU list by decreasing the 16561544Seschrock * target size of the MRU list. 1657789Sahrens */ 16583403Sbmc if (state == arc_mru_ghost) { 16593403Sbmc mult = ((arc_mru_ghost->arcs_size >= arc_mfu_ghost->arcs_size) ? 16603403Sbmc 1 : (arc_mfu_ghost->arcs_size/arc_mru_ghost->arcs_size)); 16611544Seschrock 16623403Sbmc arc_p = MIN(arc_c, arc_p + bytes * mult); 16633403Sbmc } else if (state == arc_mfu_ghost) { 16643403Sbmc mult = ((arc_mfu_ghost->arcs_size >= arc_mru_ghost->arcs_size) ? 16653403Sbmc 1 : (arc_mru_ghost->arcs_size/arc_mfu_ghost->arcs_size)); 16661544Seschrock 16673403Sbmc arc_p = MAX(0, (int64_t)arc_p - bytes * mult); 16681544Seschrock } 16693403Sbmc ASSERT((int64_t)arc_p >= 0); 1670789Sahrens 1671789Sahrens if (arc_reclaim_needed()) { 1672789Sahrens cv_signal(&arc_reclaim_thr_cv); 1673789Sahrens return; 1674789Sahrens } 1675789Sahrens 16763403Sbmc if (arc_no_grow) 1677789Sahrens return; 1678789Sahrens 16793403Sbmc if (arc_c >= arc_c_max) 16801544Seschrock return; 16811544Seschrock 1682789Sahrens /* 16831544Seschrock * If we're within (2 * maxblocksize) bytes of the target 16841544Seschrock * cache size, increment the target cache size 1685789Sahrens */ 16863403Sbmc if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) { 16873403Sbmc atomic_add_64(&arc_c, (int64_t)bytes); 16883403Sbmc if (arc_c > arc_c_max) 16893403Sbmc arc_c = arc_c_max; 16903403Sbmc else if (state == arc_anon) 16913403Sbmc atomic_add_64(&arc_p, (int64_t)bytes); 16923403Sbmc if (arc_p > arc_c) 16933403Sbmc arc_p = arc_c; 1694789Sahrens } 16953403Sbmc ASSERT((int64_t)arc_p >= 0); 1696789Sahrens } 1697789Sahrens 1698789Sahrens /* 16991544Seschrock * Check if the cache has reached its limits and eviction is required 17001544Seschrock * prior to insert. 1701789Sahrens */ 1702789Sahrens static int 17034309Smaybee arc_evict_needed(arc_buf_contents_t type) 1704789Sahrens { 17054309Smaybee if (type == ARC_BUFC_METADATA && arc_meta_used >= arc_meta_limit) 17064309Smaybee return (1); 17074309Smaybee 17084309Smaybee #ifdef _KERNEL 17094309Smaybee /* 17104309Smaybee * If zio data pages are being allocated out of a separate heap segment, 17114309Smaybee * then enforce that the size of available vmem for this area remains 17124309Smaybee * above about 1/32nd free. 17134309Smaybee */ 17144309Smaybee if (type == ARC_BUFC_DATA && zio_arena != NULL && 17154309Smaybee vmem_size(zio_arena, VMEM_FREE) < 17164309Smaybee (vmem_size(zio_arena, VMEM_ALLOC) >> 5)) 17174309Smaybee return (1); 17184309Smaybee #endif 17194309Smaybee 1720789Sahrens if (arc_reclaim_needed()) 1721789Sahrens return (1); 1722789Sahrens 17233403Sbmc return (arc_size > arc_c); 1724789Sahrens } 1725789Sahrens 1726789Sahrens /* 17272688Smaybee * The buffer, supplied as the first argument, needs a data block. 17282688Smaybee * So, if we are at cache max, determine which cache should be victimized. 17292688Smaybee * We have the following cases: 1730789Sahrens * 17313403Sbmc * 1. Insert for MRU, p > sizeof(arc_anon + arc_mru) -> 1732789Sahrens * In this situation if we're out of space, but the resident size of the MFU is 1733789Sahrens * under the limit, victimize the MFU cache to satisfy this insertion request. 1734789Sahrens * 17353403Sbmc * 2. Insert for MRU, p <= sizeof(arc_anon + arc_mru) -> 1736789Sahrens * Here, we've used up all of the available space for the MRU, so we need to 1737789Sahrens * evict from our own cache instead. Evict from the set of resident MRU 1738789Sahrens * entries. 1739789Sahrens * 17403403Sbmc * 3. Insert for MFU (c - p) > sizeof(arc_mfu) -> 1741789Sahrens * c minus p represents the MFU space in the cache, since p is the size of the 1742789Sahrens * cache that is dedicated to the MRU. In this situation there's still space on 1743789Sahrens * the MFU side, so the MRU side needs to be victimized. 1744789Sahrens * 17453403Sbmc * 4. Insert for MFU (c - p) < sizeof(arc_mfu) -> 1746789Sahrens * MFU's resident set is consuming more space than it has been allotted. In 1747789Sahrens * this situation, we must victimize our own cache, the MFU, for this insertion. 1748789Sahrens */ 1749789Sahrens static void 17502688Smaybee arc_get_data_buf(arc_buf_t *buf) 1751789Sahrens { 17523290Sjohansen arc_state_t *state = buf->b_hdr->b_state; 17533290Sjohansen uint64_t size = buf->b_hdr->b_size; 17543290Sjohansen arc_buf_contents_t type = buf->b_hdr->b_type; 17552688Smaybee 17562688Smaybee arc_adapt(size, state); 1757789Sahrens 17582688Smaybee /* 17592688Smaybee * We have not yet reached cache maximum size, 17602688Smaybee * just allocate a new buffer. 17612688Smaybee */ 17624309Smaybee if (!arc_evict_needed(type)) { 17633290Sjohansen if (type == ARC_BUFC_METADATA) { 17643290Sjohansen buf->b_data = zio_buf_alloc(size); 17654309Smaybee arc_space_consume(size); 17663290Sjohansen } else { 17673290Sjohansen ASSERT(type == ARC_BUFC_DATA); 17683290Sjohansen buf->b_data = zio_data_buf_alloc(size); 17694309Smaybee atomic_add_64(&arc_size, size); 17703290Sjohansen } 17712688Smaybee goto out; 17722688Smaybee } 17732688Smaybee 17742688Smaybee /* 17752688Smaybee * If we are prefetching from the mfu ghost list, this buffer 17762688Smaybee * will end up on the mru list; so steal space from there. 17772688Smaybee */ 17783403Sbmc if (state == arc_mfu_ghost) 17793403Sbmc state = buf->b_hdr->b_flags & ARC_PREFETCH ? arc_mru : arc_mfu; 17803403Sbmc else if (state == arc_mru_ghost) 17813403Sbmc state = arc_mru; 1782789Sahrens 17833403Sbmc if (state == arc_mru || state == arc_anon) { 17843403Sbmc uint64_t mru_used = arc_anon->arcs_size + arc_mru->arcs_size; 17854309Smaybee state = (arc_mfu->arcs_lsize[type] > 0 && 17864309Smaybee arc_p > mru_used) ? arc_mfu : arc_mru; 1787789Sahrens } else { 17882688Smaybee /* MFU cases */ 17893403Sbmc uint64_t mfu_space = arc_c - arc_p; 17904309Smaybee state = (arc_mru->arcs_lsize[type] > 0 && 17914309Smaybee mfu_space > arc_mfu->arcs_size) ? arc_mru : arc_mfu; 17922688Smaybee } 17933290Sjohansen if ((buf->b_data = arc_evict(state, size, TRUE, type)) == NULL) { 17943290Sjohansen if (type == ARC_BUFC_METADATA) { 17953290Sjohansen buf->b_data = zio_buf_alloc(size); 17964309Smaybee arc_space_consume(size); 17973290Sjohansen } else { 17983290Sjohansen ASSERT(type == ARC_BUFC_DATA); 17993290Sjohansen buf->b_data = zio_data_buf_alloc(size); 18004309Smaybee atomic_add_64(&arc_size, size); 18013290Sjohansen } 18023403Sbmc ARCSTAT_BUMP(arcstat_recycle_miss); 18032688Smaybee } 18042688Smaybee ASSERT(buf->b_data != NULL); 18052688Smaybee out: 18062688Smaybee /* 18072688Smaybee * Update the state size. Note that ghost states have a 18082688Smaybee * "ghost size" and so don't need to be updated. 18092688Smaybee */ 18102688Smaybee if (!GHOST_STATE(buf->b_hdr->b_state)) { 18112688Smaybee arc_buf_hdr_t *hdr = buf->b_hdr; 18122688Smaybee 18133403Sbmc atomic_add_64(&hdr->b_state->arcs_size, size); 18142688Smaybee if (list_link_active(&hdr->b_arc_node)) { 18152688Smaybee ASSERT(refcount_is_zero(&hdr->b_refcnt)); 18164309Smaybee atomic_add_64(&hdr->b_state->arcs_lsize[type], size); 1817789Sahrens } 18183298Smaybee /* 18193298Smaybee * If we are growing the cache, and we are adding anonymous 18203403Sbmc * data, and we have outgrown arc_p, update arc_p 18213298Smaybee */ 18223403Sbmc if (arc_size < arc_c && hdr->b_state == arc_anon && 18233403Sbmc arc_anon->arcs_size + arc_mru->arcs_size > arc_p) 18243403Sbmc arc_p = MIN(arc_c, arc_p + size); 1825789Sahrens } 1826789Sahrens } 1827789Sahrens 1828789Sahrens /* 1829789Sahrens * This routine is called whenever a buffer is accessed. 18301544Seschrock * NOTE: the hash lock is dropped in this function. 1831789Sahrens */ 1832789Sahrens static void 18332688Smaybee arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock) 1834789Sahrens { 1835789Sahrens ASSERT(MUTEX_HELD(hash_lock)); 1836789Sahrens 18373403Sbmc if (buf->b_state == arc_anon) { 1838789Sahrens /* 1839789Sahrens * This buffer is not in the cache, and does not 1840789Sahrens * appear in our "ghost" list. Add the new buffer 1841789Sahrens * to the MRU state. 1842789Sahrens */ 1843789Sahrens 1844789Sahrens ASSERT(buf->b_arc_access == 0); 1845789Sahrens buf->b_arc_access = lbolt; 18461544Seschrock DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf); 18473403Sbmc arc_change_state(arc_mru, buf, hash_lock); 1848789Sahrens 18493403Sbmc } else if (buf->b_state == arc_mru) { 1850789Sahrens /* 18512391Smaybee * If this buffer is here because of a prefetch, then either: 18522391Smaybee * - clear the flag if this is a "referencing" read 18532391Smaybee * (any subsequent access will bump this into the MFU state). 18542391Smaybee * or 18552391Smaybee * - move the buffer to the head of the list if this is 18562391Smaybee * another prefetch (to make it less likely to be evicted). 1857789Sahrens */ 1858789Sahrens if ((buf->b_flags & ARC_PREFETCH) != 0) { 18592391Smaybee if (refcount_count(&buf->b_refcnt) == 0) { 18602391Smaybee ASSERT(list_link_active(&buf->b_arc_node)); 18612391Smaybee } else { 18622391Smaybee buf->b_flags &= ~ARC_PREFETCH; 18633403Sbmc ARCSTAT_BUMP(arcstat_mru_hits); 18642391Smaybee } 18652391Smaybee buf->b_arc_access = lbolt; 1866789Sahrens return; 1867789Sahrens } 1868789Sahrens 1869789Sahrens /* 1870789Sahrens * This buffer has been "accessed" only once so far, 1871789Sahrens * but it is still in the cache. Move it to the MFU 1872789Sahrens * state. 1873789Sahrens */ 1874789Sahrens if (lbolt > buf->b_arc_access + ARC_MINTIME) { 1875789Sahrens /* 1876789Sahrens * More than 125ms have passed since we 1877789Sahrens * instantiated this buffer. Move it to the 1878789Sahrens * most frequently used state. 1879789Sahrens */ 1880789Sahrens buf->b_arc_access = lbolt; 18811544Seschrock DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 18823403Sbmc arc_change_state(arc_mfu, buf, hash_lock); 1883789Sahrens } 18843403Sbmc ARCSTAT_BUMP(arcstat_mru_hits); 18853403Sbmc } else if (buf->b_state == arc_mru_ghost) { 1886789Sahrens arc_state_t *new_state; 1887789Sahrens /* 1888789Sahrens * This buffer has been "accessed" recently, but 1889789Sahrens * was evicted from the cache. Move it to the 1890789Sahrens * MFU state. 1891789Sahrens */ 1892789Sahrens 1893789Sahrens if (buf->b_flags & ARC_PREFETCH) { 18943403Sbmc new_state = arc_mru; 18952391Smaybee if (refcount_count(&buf->b_refcnt) > 0) 18962391Smaybee buf->b_flags &= ~ARC_PREFETCH; 18971544Seschrock DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf); 1898789Sahrens } else { 18993403Sbmc new_state = arc_mfu; 19001544Seschrock DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 1901789Sahrens } 1902789Sahrens 1903789Sahrens buf->b_arc_access = lbolt; 1904789Sahrens arc_change_state(new_state, buf, hash_lock); 1905789Sahrens 19063403Sbmc ARCSTAT_BUMP(arcstat_mru_ghost_hits); 19073403Sbmc } else if (buf->b_state == arc_mfu) { 1908789Sahrens /* 1909789Sahrens * This buffer has been accessed more than once and is 1910789Sahrens * still in the cache. Keep it in the MFU state. 1911789Sahrens * 19122391Smaybee * NOTE: an add_reference() that occurred when we did 19132391Smaybee * the arc_read() will have kicked this off the list. 19142391Smaybee * If it was a prefetch, we will explicitly move it to 19152391Smaybee * the head of the list now. 1916789Sahrens */ 19172391Smaybee if ((buf->b_flags & ARC_PREFETCH) != 0) { 19182391Smaybee ASSERT(refcount_count(&buf->b_refcnt) == 0); 19192391Smaybee ASSERT(list_link_active(&buf->b_arc_node)); 19202391Smaybee } 19213403Sbmc ARCSTAT_BUMP(arcstat_mfu_hits); 19222391Smaybee buf->b_arc_access = lbolt; 19233403Sbmc } else if (buf->b_state == arc_mfu_ghost) { 19243403Sbmc arc_state_t *new_state = arc_mfu; 1925789Sahrens /* 1926789Sahrens * This buffer has been accessed more than once but has 1927789Sahrens * been evicted from the cache. Move it back to the 1928789Sahrens * MFU state. 1929789Sahrens */ 1930789Sahrens 19312391Smaybee if (buf->b_flags & ARC_PREFETCH) { 19322391Smaybee /* 19332391Smaybee * This is a prefetch access... 19342391Smaybee * move this block back to the MRU state. 19352391Smaybee */ 19362391Smaybee ASSERT3U(refcount_count(&buf->b_refcnt), ==, 0); 19373403Sbmc new_state = arc_mru; 19382391Smaybee } 19392391Smaybee 1940789Sahrens buf->b_arc_access = lbolt; 19411544Seschrock DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 19422391Smaybee arc_change_state(new_state, buf, hash_lock); 1943789Sahrens 19443403Sbmc ARCSTAT_BUMP(arcstat_mfu_ghost_hits); 1945789Sahrens } else { 1946789Sahrens ASSERT(!"invalid arc state"); 1947789Sahrens } 1948789Sahrens } 1949789Sahrens 1950789Sahrens /* a generic arc_done_func_t which you can use */ 1951789Sahrens /* ARGSUSED */ 1952789Sahrens void 1953789Sahrens arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg) 1954789Sahrens { 1955789Sahrens bcopy(buf->b_data, arg, buf->b_hdr->b_size); 19561544Seschrock VERIFY(arc_buf_remove_ref(buf, arg) == 1); 1957789Sahrens } 1958789Sahrens 19594309Smaybee /* a generic arc_done_func_t */ 1960789Sahrens void 1961789Sahrens arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg) 1962789Sahrens { 1963789Sahrens arc_buf_t **bufp = arg; 1964789Sahrens if (zio && zio->io_error) { 19651544Seschrock VERIFY(arc_buf_remove_ref(buf, arg) == 1); 1966789Sahrens *bufp = NULL; 1967789Sahrens } else { 1968789Sahrens *bufp = buf; 1969789Sahrens } 1970789Sahrens } 1971789Sahrens 1972789Sahrens static void 1973789Sahrens arc_read_done(zio_t *zio) 1974789Sahrens { 19751589Smaybee arc_buf_hdr_t *hdr, *found; 1976789Sahrens arc_buf_t *buf; 1977789Sahrens arc_buf_t *abuf; /* buffer we're assigning to callback */ 1978789Sahrens kmutex_t *hash_lock; 1979789Sahrens arc_callback_t *callback_list, *acb; 1980789Sahrens int freeable = FALSE; 1981789Sahrens 1982789Sahrens buf = zio->io_private; 1983789Sahrens hdr = buf->b_hdr; 1984789Sahrens 19851589Smaybee /* 19861589Smaybee * The hdr was inserted into hash-table and removed from lists 19871589Smaybee * prior to starting I/O. We should find this header, since 19881589Smaybee * it's in the hash table, and it should be legit since it's 19891589Smaybee * not possible to evict it during the I/O. The only possible 19901589Smaybee * reason for it not to be found is if we were freed during the 19911589Smaybee * read. 19921589Smaybee */ 19931589Smaybee found = buf_hash_find(zio->io_spa, &hdr->b_dva, hdr->b_birth, 19943093Sahrens &hash_lock); 1995789Sahrens 19961589Smaybee ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && hash_lock == NULL) || 19971589Smaybee (found == hdr && DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp)))); 1998789Sahrens 1999789Sahrens /* byteswap if necessary */ 2000789Sahrens callback_list = hdr->b_acb; 2001789Sahrens ASSERT(callback_list != NULL); 2002789Sahrens if (BP_SHOULD_BYTESWAP(zio->io_bp) && callback_list->acb_byteswap) 2003789Sahrens callback_list->acb_byteswap(buf->b_data, hdr->b_size); 2004789Sahrens 20053093Sahrens arc_cksum_compute(buf); 20063093Sahrens 2007789Sahrens /* create copies of the data buffer for the callers */ 2008789Sahrens abuf = buf; 2009789Sahrens for (acb = callback_list; acb; acb = acb->acb_next) { 2010789Sahrens if (acb->acb_done) { 20112688Smaybee if (abuf == NULL) 20122688Smaybee abuf = arc_buf_clone(buf); 2013789Sahrens acb->acb_buf = abuf; 2014789Sahrens abuf = NULL; 2015789Sahrens } 2016789Sahrens } 2017789Sahrens hdr->b_acb = NULL; 2018789Sahrens hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 20191544Seschrock ASSERT(!HDR_BUF_AVAILABLE(hdr)); 20201544Seschrock if (abuf == buf) 20211544Seschrock hdr->b_flags |= ARC_BUF_AVAILABLE; 2022789Sahrens 2023789Sahrens ASSERT(refcount_is_zero(&hdr->b_refcnt) || callback_list != NULL); 2024789Sahrens 2025789Sahrens if (zio->io_error != 0) { 2026789Sahrens hdr->b_flags |= ARC_IO_ERROR; 20273403Sbmc if (hdr->b_state != arc_anon) 20283403Sbmc arc_change_state(arc_anon, hdr, hash_lock); 20291544Seschrock if (HDR_IN_HASH_TABLE(hdr)) 20301544Seschrock buf_hash_remove(hdr); 2031789Sahrens freeable = refcount_is_zero(&hdr->b_refcnt); 20322391Smaybee /* convert checksum errors into IO errors */ 20331544Seschrock if (zio->io_error == ECKSUM) 20341544Seschrock zio->io_error = EIO; 2035789Sahrens } 2036789Sahrens 20371544Seschrock /* 20382391Smaybee * Broadcast before we drop the hash_lock to avoid the possibility 20392391Smaybee * that the hdr (and hence the cv) might be freed before we get to 20402391Smaybee * the cv_broadcast(). 20411544Seschrock */ 20421544Seschrock cv_broadcast(&hdr->b_cv); 20431544Seschrock 20441589Smaybee if (hash_lock) { 2045789Sahrens /* 2046789Sahrens * Only call arc_access on anonymous buffers. This is because 2047789Sahrens * if we've issued an I/O for an evicted buffer, we've already 2048789Sahrens * called arc_access (to prevent any simultaneous readers from 2049789Sahrens * getting confused). 2050789Sahrens */ 20513403Sbmc if (zio->io_error == 0 && hdr->b_state == arc_anon) 20522688Smaybee arc_access(hdr, hash_lock); 20532688Smaybee mutex_exit(hash_lock); 2054789Sahrens } else { 2055789Sahrens /* 2056789Sahrens * This block was freed while we waited for the read to 2057789Sahrens * complete. It has been removed from the hash table and 2058789Sahrens * moved to the anonymous state (so that it won't show up 2059789Sahrens * in the cache). 2060789Sahrens */ 20613403Sbmc ASSERT3P(hdr->b_state, ==, arc_anon); 2062789Sahrens freeable = refcount_is_zero(&hdr->b_refcnt); 2063789Sahrens } 2064789Sahrens 2065789Sahrens /* execute each callback and free its structure */ 2066789Sahrens while ((acb = callback_list) != NULL) { 2067789Sahrens if (acb->acb_done) 2068789Sahrens acb->acb_done(zio, acb->acb_buf, acb->acb_private); 2069789Sahrens 2070789Sahrens if (acb->acb_zio_dummy != NULL) { 2071789Sahrens acb->acb_zio_dummy->io_error = zio->io_error; 2072789Sahrens zio_nowait(acb->acb_zio_dummy); 2073789Sahrens } 2074789Sahrens 2075789Sahrens callback_list = acb->acb_next; 2076789Sahrens kmem_free(acb, sizeof (arc_callback_t)); 2077789Sahrens } 2078789Sahrens 2079789Sahrens if (freeable) 20801544Seschrock arc_hdr_destroy(hdr); 2081789Sahrens } 2082789Sahrens 2083789Sahrens /* 2084789Sahrens * "Read" the block block at the specified DVA (in bp) via the 2085789Sahrens * cache. If the block is found in the cache, invoke the provided 2086789Sahrens * callback immediately and return. Note that the `zio' parameter 2087789Sahrens * in the callback will be NULL in this case, since no IO was 2088789Sahrens * required. If the block is not in the cache pass the read request 2089789Sahrens * on to the spa with a substitute callback function, so that the 2090789Sahrens * requested block will be added to the cache. 2091789Sahrens * 2092789Sahrens * If a read request arrives for a block that has a read in-progress, 2093789Sahrens * either wait for the in-progress read to complete (and return the 2094789Sahrens * results); or, if this is a read with a "done" func, add a record 2095789Sahrens * to the read to invoke the "done" func when the read completes, 2096789Sahrens * and return; or just return. 2097789Sahrens * 2098789Sahrens * arc_read_done() will invoke all the requested "done" functions 2099789Sahrens * for readers of this block. 2100789Sahrens */ 2101789Sahrens int 2102789Sahrens arc_read(zio_t *pio, spa_t *spa, blkptr_t *bp, arc_byteswap_func_t *swap, 2103789Sahrens arc_done_func_t *done, void *private, int priority, int flags, 21042391Smaybee uint32_t *arc_flags, zbookmark_t *zb) 2105789Sahrens { 2106789Sahrens arc_buf_hdr_t *hdr; 2107789Sahrens arc_buf_t *buf; 2108789Sahrens kmutex_t *hash_lock; 2109789Sahrens zio_t *rzio; 2110789Sahrens 2111789Sahrens top: 2112789Sahrens hdr = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_lock); 21131544Seschrock if (hdr && hdr->b_datacnt > 0) { 2114789Sahrens 21152391Smaybee *arc_flags |= ARC_CACHED; 21162391Smaybee 2117789Sahrens if (HDR_IO_IN_PROGRESS(hdr)) { 21182391Smaybee 21192391Smaybee if (*arc_flags & ARC_WAIT) { 21202391Smaybee cv_wait(&hdr->b_cv, hash_lock); 21212391Smaybee mutex_exit(hash_lock); 21222391Smaybee goto top; 21232391Smaybee } 21242391Smaybee ASSERT(*arc_flags & ARC_NOWAIT); 21252391Smaybee 21262391Smaybee if (done) { 2127789Sahrens arc_callback_t *acb = NULL; 2128789Sahrens 2129789Sahrens acb = kmem_zalloc(sizeof (arc_callback_t), 2130789Sahrens KM_SLEEP); 2131789Sahrens acb->acb_done = done; 2132789Sahrens acb->acb_private = private; 2133789Sahrens acb->acb_byteswap = swap; 2134789Sahrens if (pio != NULL) 2135789Sahrens acb->acb_zio_dummy = zio_null(pio, 2136789Sahrens spa, NULL, NULL, flags); 2137789Sahrens 2138789Sahrens ASSERT(acb->acb_done != NULL); 2139789Sahrens acb->acb_next = hdr->b_acb; 2140789Sahrens hdr->b_acb = acb; 2141789Sahrens add_reference(hdr, hash_lock, private); 2142789Sahrens mutex_exit(hash_lock); 2143789Sahrens return (0); 2144789Sahrens } 2145789Sahrens mutex_exit(hash_lock); 2146789Sahrens return (0); 2147789Sahrens } 2148789Sahrens 21493403Sbmc ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 2150789Sahrens 21511544Seschrock if (done) { 21522688Smaybee add_reference(hdr, hash_lock, private); 21531544Seschrock /* 21541544Seschrock * If this block is already in use, create a new 21551544Seschrock * copy of the data so that we will be guaranteed 21561544Seschrock * that arc_release() will always succeed. 21571544Seschrock */ 21581544Seschrock buf = hdr->b_buf; 21591544Seschrock ASSERT(buf); 21601544Seschrock ASSERT(buf->b_data); 21612688Smaybee if (HDR_BUF_AVAILABLE(hdr)) { 21621544Seschrock ASSERT(buf->b_efunc == NULL); 21631544Seschrock hdr->b_flags &= ~ARC_BUF_AVAILABLE; 21642688Smaybee } else { 21652688Smaybee buf = arc_buf_clone(buf); 21661544Seschrock } 21672391Smaybee } else if (*arc_flags & ARC_PREFETCH && 21682391Smaybee refcount_count(&hdr->b_refcnt) == 0) { 21692391Smaybee hdr->b_flags |= ARC_PREFETCH; 2170789Sahrens } 2171789Sahrens DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); 21722688Smaybee arc_access(hdr, hash_lock); 21732688Smaybee mutex_exit(hash_lock); 21743403Sbmc ARCSTAT_BUMP(arcstat_hits); 21753403Sbmc ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 21763403Sbmc demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 21773403Sbmc data, metadata, hits); 21783403Sbmc 2179789Sahrens if (done) 2180789Sahrens done(NULL, buf, private); 2181789Sahrens } else { 2182789Sahrens uint64_t size = BP_GET_LSIZE(bp); 2183789Sahrens arc_callback_t *acb; 2184789Sahrens 2185789Sahrens if (hdr == NULL) { 2186789Sahrens /* this block is not in the cache */ 2187789Sahrens arc_buf_hdr_t *exists; 21883290Sjohansen arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp); 21893290Sjohansen buf = arc_buf_alloc(spa, size, private, type); 2190789Sahrens hdr = buf->b_hdr; 2191789Sahrens hdr->b_dva = *BP_IDENTITY(bp); 2192789Sahrens hdr->b_birth = bp->blk_birth; 2193789Sahrens hdr->b_cksum0 = bp->blk_cksum.zc_word[0]; 2194789Sahrens exists = buf_hash_insert(hdr, &hash_lock); 2195789Sahrens if (exists) { 2196789Sahrens /* somebody beat us to the hash insert */ 2197789Sahrens mutex_exit(hash_lock); 2198789Sahrens bzero(&hdr->b_dva, sizeof (dva_t)); 2199789Sahrens hdr->b_birth = 0; 2200789Sahrens hdr->b_cksum0 = 0; 22011544Seschrock (void) arc_buf_remove_ref(buf, private); 2202789Sahrens goto top; /* restart the IO request */ 2203789Sahrens } 22042391Smaybee /* if this is a prefetch, we don't have a reference */ 22052391Smaybee if (*arc_flags & ARC_PREFETCH) { 22062391Smaybee (void) remove_reference(hdr, hash_lock, 22072391Smaybee private); 22082391Smaybee hdr->b_flags |= ARC_PREFETCH; 22092391Smaybee } 22102391Smaybee if (BP_GET_LEVEL(bp) > 0) 22112391Smaybee hdr->b_flags |= ARC_INDIRECT; 2212789Sahrens } else { 2213789Sahrens /* this block is in the ghost cache */ 22141544Seschrock ASSERT(GHOST_STATE(hdr->b_state)); 22151544Seschrock ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 22162391Smaybee ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 0); 22172391Smaybee ASSERT(hdr->b_buf == NULL); 2218789Sahrens 22192391Smaybee /* if this is a prefetch, we don't have a reference */ 22202391Smaybee if (*arc_flags & ARC_PREFETCH) 22212391Smaybee hdr->b_flags |= ARC_PREFETCH; 22222391Smaybee else 22232391Smaybee add_reference(hdr, hash_lock, private); 2224789Sahrens buf = kmem_cache_alloc(buf_cache, KM_SLEEP); 22251544Seschrock buf->b_hdr = hdr; 22262688Smaybee buf->b_data = NULL; 22271544Seschrock buf->b_efunc = NULL; 22281544Seschrock buf->b_private = NULL; 22291544Seschrock buf->b_next = NULL; 22301544Seschrock hdr->b_buf = buf; 22312688Smaybee arc_get_data_buf(buf); 22321544Seschrock ASSERT(hdr->b_datacnt == 0); 22331544Seschrock hdr->b_datacnt = 1; 22342391Smaybee 2235789Sahrens } 2236789Sahrens 2237789Sahrens acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP); 2238789Sahrens acb->acb_done = done; 2239789Sahrens acb->acb_private = private; 2240789Sahrens acb->acb_byteswap = swap; 2241789Sahrens 2242789Sahrens ASSERT(hdr->b_acb == NULL); 2243789Sahrens hdr->b_acb = acb; 2244789Sahrens hdr->b_flags |= ARC_IO_IN_PROGRESS; 2245789Sahrens 2246789Sahrens /* 2247789Sahrens * If the buffer has been evicted, migrate it to a present state 2248789Sahrens * before issuing the I/O. Once we drop the hash-table lock, 2249789Sahrens * the header will be marked as I/O in progress and have an 2250789Sahrens * attached buffer. At this point, anybody who finds this 2251789Sahrens * buffer ought to notice that it's legit but has a pending I/O. 2252789Sahrens */ 2253789Sahrens 22541544Seschrock if (GHOST_STATE(hdr->b_state)) 22552688Smaybee arc_access(hdr, hash_lock); 22562688Smaybee mutex_exit(hash_lock); 2257789Sahrens 2258789Sahrens ASSERT3U(hdr->b_size, ==, size); 22591596Sahrens DTRACE_PROBE3(arc__miss, blkptr_t *, bp, uint64_t, size, 22601596Sahrens zbookmark_t *, zb); 22613403Sbmc ARCSTAT_BUMP(arcstat_misses); 22623403Sbmc ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH), 22633403Sbmc demand, prefetch, hdr->b_type != ARC_BUFC_METADATA, 22643403Sbmc data, metadata, misses); 22651544Seschrock 2266789Sahrens rzio = zio_read(pio, spa, bp, buf->b_data, size, 22671544Seschrock arc_read_done, buf, priority, flags, zb); 2268789Sahrens 22692391Smaybee if (*arc_flags & ARC_WAIT) 2270789Sahrens return (zio_wait(rzio)); 2271789Sahrens 22722391Smaybee ASSERT(*arc_flags & ARC_NOWAIT); 2273789Sahrens zio_nowait(rzio); 2274789Sahrens } 2275789Sahrens return (0); 2276789Sahrens } 2277789Sahrens 2278789Sahrens /* 2279789Sahrens * arc_read() variant to support pool traversal. If the block is already 2280789Sahrens * in the ARC, make a copy of it; otherwise, the caller will do the I/O. 2281789Sahrens * The idea is that we don't want pool traversal filling up memory, but 2282789Sahrens * if the ARC already has the data anyway, we shouldn't pay for the I/O. 2283789Sahrens */ 2284789Sahrens int 2285789Sahrens arc_tryread(spa_t *spa, blkptr_t *bp, void *data) 2286789Sahrens { 2287789Sahrens arc_buf_hdr_t *hdr; 2288789Sahrens kmutex_t *hash_mtx; 2289789Sahrens int rc = 0; 2290789Sahrens 2291789Sahrens hdr = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_mtx); 2292789Sahrens 22931544Seschrock if (hdr && hdr->b_datacnt > 0 && !HDR_IO_IN_PROGRESS(hdr)) { 22941544Seschrock arc_buf_t *buf = hdr->b_buf; 22951544Seschrock 22961544Seschrock ASSERT(buf); 22971544Seschrock while (buf->b_data == NULL) { 22981544Seschrock buf = buf->b_next; 22991544Seschrock ASSERT(buf); 23001544Seschrock } 23011544Seschrock bcopy(buf->b_data, data, hdr->b_size); 23021544Seschrock } else { 2303789Sahrens rc = ENOENT; 23041544Seschrock } 2305789Sahrens 2306789Sahrens if (hash_mtx) 2307789Sahrens mutex_exit(hash_mtx); 2308789Sahrens 2309789Sahrens return (rc); 2310789Sahrens } 2311789Sahrens 23121544Seschrock void 23131544Seschrock arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private) 23141544Seschrock { 23151544Seschrock ASSERT(buf->b_hdr != NULL); 23163403Sbmc ASSERT(buf->b_hdr->b_state != arc_anon); 23171544Seschrock ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt) || func == NULL); 23181544Seschrock buf->b_efunc = func; 23191544Seschrock buf->b_private = private; 23201544Seschrock } 23211544Seschrock 23221544Seschrock /* 23231544Seschrock * This is used by the DMU to let the ARC know that a buffer is 23241544Seschrock * being evicted, so the ARC should clean up. If this arc buf 23251544Seschrock * is not yet in the evicted state, it will be put there. 23261544Seschrock */ 23271544Seschrock int 23281544Seschrock arc_buf_evict(arc_buf_t *buf) 23291544Seschrock { 23302887Smaybee arc_buf_hdr_t *hdr; 23311544Seschrock kmutex_t *hash_lock; 23321544Seschrock arc_buf_t **bufp; 23331544Seschrock 23342887Smaybee mutex_enter(&arc_eviction_mtx); 23352887Smaybee hdr = buf->b_hdr; 23361544Seschrock if (hdr == NULL) { 23371544Seschrock /* 23381544Seschrock * We are in arc_do_user_evicts(). 23391544Seschrock */ 23401544Seschrock ASSERT(buf->b_data == NULL); 23412887Smaybee mutex_exit(&arc_eviction_mtx); 23421544Seschrock return (0); 23431544Seschrock } 23442887Smaybee hash_lock = HDR_LOCK(hdr); 23452887Smaybee mutex_exit(&arc_eviction_mtx); 23461544Seschrock 23471544Seschrock mutex_enter(hash_lock); 23481544Seschrock 23492724Smaybee if (buf->b_data == NULL) { 23502724Smaybee /* 23512724Smaybee * We are on the eviction list. 23522724Smaybee */ 23532724Smaybee mutex_exit(hash_lock); 23542724Smaybee mutex_enter(&arc_eviction_mtx); 23552724Smaybee if (buf->b_hdr == NULL) { 23562724Smaybee /* 23572724Smaybee * We are already in arc_do_user_evicts(). 23582724Smaybee */ 23592724Smaybee mutex_exit(&arc_eviction_mtx); 23602724Smaybee return (0); 23612724Smaybee } else { 23622724Smaybee arc_buf_t copy = *buf; /* structure assignment */ 23632724Smaybee /* 23642724Smaybee * Process this buffer now 23652724Smaybee * but let arc_do_user_evicts() do the reaping. 23662724Smaybee */ 23672724Smaybee buf->b_efunc = NULL; 23682724Smaybee mutex_exit(&arc_eviction_mtx); 23692724Smaybee VERIFY(copy.b_efunc(©) == 0); 23702724Smaybee return (1); 23712724Smaybee } 23722724Smaybee } 23732724Smaybee 23742724Smaybee ASSERT(buf->b_hdr == hdr); 23752724Smaybee ASSERT3U(refcount_count(&hdr->b_refcnt), <, hdr->b_datacnt); 23763403Sbmc ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu); 23771544Seschrock 23781544Seschrock /* 23791544Seschrock * Pull this buffer off of the hdr 23801544Seschrock */ 23811544Seschrock bufp = &hdr->b_buf; 23821544Seschrock while (*bufp != buf) 23831544Seschrock bufp = &(*bufp)->b_next; 23841544Seschrock *bufp = buf->b_next; 23851544Seschrock 23861544Seschrock ASSERT(buf->b_data != NULL); 23872688Smaybee arc_buf_destroy(buf, FALSE, FALSE); 23881544Seschrock 23891544Seschrock if (hdr->b_datacnt == 0) { 23901544Seschrock arc_state_t *old_state = hdr->b_state; 23911544Seschrock arc_state_t *evicted_state; 23921544Seschrock 23931544Seschrock ASSERT(refcount_is_zero(&hdr->b_refcnt)); 23941544Seschrock 23951544Seschrock evicted_state = 23963403Sbmc (old_state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost; 23971544Seschrock 23983403Sbmc mutex_enter(&old_state->arcs_mtx); 23993403Sbmc mutex_enter(&evicted_state->arcs_mtx); 24001544Seschrock 24011544Seschrock arc_change_state(evicted_state, hdr, hash_lock); 24021544Seschrock ASSERT(HDR_IN_HASH_TABLE(hdr)); 24031544Seschrock hdr->b_flags = ARC_IN_HASH_TABLE; 24041544Seschrock 24053403Sbmc mutex_exit(&evicted_state->arcs_mtx); 24063403Sbmc mutex_exit(&old_state->arcs_mtx); 24071544Seschrock } 24081544Seschrock mutex_exit(hash_lock); 24091819Smaybee 24101544Seschrock VERIFY(buf->b_efunc(buf) == 0); 24111544Seschrock buf->b_efunc = NULL; 24121544Seschrock buf->b_private = NULL; 24131544Seschrock buf->b_hdr = NULL; 24141544Seschrock kmem_cache_free(buf_cache, buf); 24151544Seschrock return (1); 24161544Seschrock } 24171544Seschrock 2418789Sahrens /* 2419789Sahrens * Release this buffer from the cache. This must be done 2420789Sahrens * after a read and prior to modifying the buffer contents. 2421789Sahrens * If the buffer has more than one reference, we must make 2422789Sahrens * make a new hdr for the buffer. 2423789Sahrens */ 2424789Sahrens void 2425789Sahrens arc_release(arc_buf_t *buf, void *tag) 2426789Sahrens { 2427789Sahrens arc_buf_hdr_t *hdr = buf->b_hdr; 2428789Sahrens kmutex_t *hash_lock = HDR_LOCK(hdr); 2429789Sahrens 2430789Sahrens /* this buffer is not on any list */ 2431789Sahrens ASSERT(refcount_count(&hdr->b_refcnt) > 0); 2432789Sahrens 24333403Sbmc if (hdr->b_state == arc_anon) { 2434789Sahrens /* this buffer is already released */ 2435789Sahrens ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 1); 2436789Sahrens ASSERT(BUF_EMPTY(hdr)); 24371544Seschrock ASSERT(buf->b_efunc == NULL); 24383093Sahrens arc_buf_thaw(buf); 2439789Sahrens return; 2440789Sahrens } 2441789Sahrens 2442789Sahrens mutex_enter(hash_lock); 2443789Sahrens 24441544Seschrock /* 24451544Seschrock * Do we have more than one buf? 24461544Seschrock */ 24471544Seschrock if (hdr->b_buf != buf || buf->b_next != NULL) { 2448789Sahrens arc_buf_hdr_t *nhdr; 2449789Sahrens arc_buf_t **bufp; 2450789Sahrens uint64_t blksz = hdr->b_size; 2451789Sahrens spa_t *spa = hdr->b_spa; 24523290Sjohansen arc_buf_contents_t type = hdr->b_type; 2453789Sahrens 24541544Seschrock ASSERT(hdr->b_datacnt > 1); 2455789Sahrens /* 2456789Sahrens * Pull the data off of this buf and attach it to 2457789Sahrens * a new anonymous buf. 2458789Sahrens */ 24591544Seschrock (void) remove_reference(hdr, hash_lock, tag); 2460789Sahrens bufp = &hdr->b_buf; 24611544Seschrock while (*bufp != buf) 2462789Sahrens bufp = &(*bufp)->b_next; 2463789Sahrens *bufp = (*bufp)->b_next; 24643897Smaybee buf->b_next = NULL; 24651544Seschrock 24663403Sbmc ASSERT3U(hdr->b_state->arcs_size, >=, hdr->b_size); 24673403Sbmc atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size); 24681544Seschrock if (refcount_is_zero(&hdr->b_refcnt)) { 24694309Smaybee uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type]; 24704309Smaybee ASSERT3U(*size, >=, hdr->b_size); 24714309Smaybee atomic_add_64(size, -hdr->b_size); 24721544Seschrock } 24731544Seschrock hdr->b_datacnt -= 1; 24743547Smaybee arc_cksum_verify(buf); 24751544Seschrock 2476789Sahrens mutex_exit(hash_lock); 2477789Sahrens 2478789Sahrens nhdr = kmem_cache_alloc(hdr_cache, KM_SLEEP); 2479789Sahrens nhdr->b_size = blksz; 2480789Sahrens nhdr->b_spa = spa; 24813290Sjohansen nhdr->b_type = type; 2482789Sahrens nhdr->b_buf = buf; 24833403Sbmc nhdr->b_state = arc_anon; 2484789Sahrens nhdr->b_arc_access = 0; 2485789Sahrens nhdr->b_flags = 0; 24861544Seschrock nhdr->b_datacnt = 1; 24873547Smaybee nhdr->b_freeze_cksum = NULL; 24883897Smaybee (void) refcount_add(&nhdr->b_refcnt, tag); 2489789Sahrens buf->b_hdr = nhdr; 24903403Sbmc atomic_add_64(&arc_anon->arcs_size, blksz); 2491789Sahrens 2492789Sahrens hdr = nhdr; 2493789Sahrens } else { 24941544Seschrock ASSERT(refcount_count(&hdr->b_refcnt) == 1); 2495789Sahrens ASSERT(!list_link_active(&hdr->b_arc_node)); 2496789Sahrens ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 24973403Sbmc arc_change_state(arc_anon, hdr, hash_lock); 2498789Sahrens hdr->b_arc_access = 0; 2499789Sahrens mutex_exit(hash_lock); 2500789Sahrens bzero(&hdr->b_dva, sizeof (dva_t)); 2501789Sahrens hdr->b_birth = 0; 2502789Sahrens hdr->b_cksum0 = 0; 25033547Smaybee arc_buf_thaw(buf); 2504789Sahrens } 25051544Seschrock buf->b_efunc = NULL; 25061544Seschrock buf->b_private = NULL; 2507789Sahrens } 2508789Sahrens 2509789Sahrens int 2510789Sahrens arc_released(arc_buf_t *buf) 2511789Sahrens { 25123403Sbmc return (buf->b_data != NULL && buf->b_hdr->b_state == arc_anon); 25131544Seschrock } 25141544Seschrock 25151544Seschrock int 25161544Seschrock arc_has_callback(arc_buf_t *buf) 25171544Seschrock { 25181544Seschrock return (buf->b_efunc != NULL); 2519789Sahrens } 2520789Sahrens 25211544Seschrock #ifdef ZFS_DEBUG 25221544Seschrock int 25231544Seschrock arc_referenced(arc_buf_t *buf) 25241544Seschrock { 25251544Seschrock return (refcount_count(&buf->b_hdr->b_refcnt)); 25261544Seschrock } 25271544Seschrock #endif 25281544Seschrock 2529789Sahrens static void 25303547Smaybee arc_write_ready(zio_t *zio) 25313547Smaybee { 25323547Smaybee arc_write_callback_t *callback = zio->io_private; 25333547Smaybee arc_buf_t *buf = callback->awcb_buf; 25343547Smaybee 25353547Smaybee if (callback->awcb_ready) { 25363547Smaybee ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt)); 25373547Smaybee callback->awcb_ready(zio, buf, callback->awcb_private); 25383547Smaybee } 25393547Smaybee arc_cksum_compute(buf); 25403547Smaybee } 25413547Smaybee 25423547Smaybee static void 2543789Sahrens arc_write_done(zio_t *zio) 2544789Sahrens { 25453547Smaybee arc_write_callback_t *callback = zio->io_private; 25463547Smaybee arc_buf_t *buf = callback->awcb_buf; 25473547Smaybee arc_buf_hdr_t *hdr = buf->b_hdr; 2548789Sahrens 2549789Sahrens hdr->b_acb = NULL; 2550789Sahrens 2551789Sahrens /* this buffer is on no lists and is not in the hash table */ 25523403Sbmc ASSERT3P(hdr->b_state, ==, arc_anon); 2553789Sahrens 2554789Sahrens hdr->b_dva = *BP_IDENTITY(zio->io_bp); 2555789Sahrens hdr->b_birth = zio->io_bp->blk_birth; 2556789Sahrens hdr->b_cksum0 = zio->io_bp->blk_cksum.zc_word[0]; 25571544Seschrock /* 25581544Seschrock * If the block to be written was all-zero, we may have 25591544Seschrock * compressed it away. In this case no write was performed 25601544Seschrock * so there will be no dva/birth-date/checksum. The buffer 25611544Seschrock * must therefor remain anonymous (and uncached). 25621544Seschrock */ 2563789Sahrens if (!BUF_EMPTY(hdr)) { 2564789Sahrens arc_buf_hdr_t *exists; 2565789Sahrens kmutex_t *hash_lock; 2566789Sahrens 25673093Sahrens arc_cksum_verify(buf); 25683093Sahrens 2569789Sahrens exists = buf_hash_insert(hdr, &hash_lock); 2570789Sahrens if (exists) { 2571789Sahrens /* 2572789Sahrens * This can only happen if we overwrite for 2573789Sahrens * sync-to-convergence, because we remove 2574789Sahrens * buffers from the hash table when we arc_free(). 2575789Sahrens */ 2576789Sahrens ASSERT(DVA_EQUAL(BP_IDENTITY(&zio->io_bp_orig), 2577789Sahrens BP_IDENTITY(zio->io_bp))); 2578789Sahrens ASSERT3U(zio->io_bp_orig.blk_birth, ==, 2579789Sahrens zio->io_bp->blk_birth); 2580789Sahrens 2581789Sahrens ASSERT(refcount_is_zero(&exists->b_refcnt)); 25823403Sbmc arc_change_state(arc_anon, exists, hash_lock); 2583789Sahrens mutex_exit(hash_lock); 25841544Seschrock arc_hdr_destroy(exists); 2585789Sahrens exists = buf_hash_insert(hdr, &hash_lock); 2586789Sahrens ASSERT3P(exists, ==, NULL); 2587789Sahrens } 25881544Seschrock hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 25892688Smaybee arc_access(hdr, hash_lock); 25902688Smaybee mutex_exit(hash_lock); 25913547Smaybee } else if (callback->awcb_done == NULL) { 25921544Seschrock int destroy_hdr; 25931544Seschrock /* 25941544Seschrock * This is an anonymous buffer with no user callback, 25951544Seschrock * destroy it if there are no active references. 25961544Seschrock */ 25971544Seschrock mutex_enter(&arc_eviction_mtx); 25981544Seschrock destroy_hdr = refcount_is_zero(&hdr->b_refcnt); 25991544Seschrock hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 26001544Seschrock mutex_exit(&arc_eviction_mtx); 26011544Seschrock if (destroy_hdr) 26021544Seschrock arc_hdr_destroy(hdr); 26031544Seschrock } else { 26041544Seschrock hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 2605789Sahrens } 26061544Seschrock 26073547Smaybee if (callback->awcb_done) { 2608789Sahrens ASSERT(!refcount_is_zero(&hdr->b_refcnt)); 26093547Smaybee callback->awcb_done(zio, buf, callback->awcb_private); 2610789Sahrens } 2611789Sahrens 26123547Smaybee kmem_free(callback, sizeof (arc_write_callback_t)); 2613789Sahrens } 2614789Sahrens 26153547Smaybee zio_t * 26161775Sbillm arc_write(zio_t *pio, spa_t *spa, int checksum, int compress, int ncopies, 2617789Sahrens uint64_t txg, blkptr_t *bp, arc_buf_t *buf, 26183547Smaybee arc_done_func_t *ready, arc_done_func_t *done, void *private, int priority, 26193547Smaybee int flags, zbookmark_t *zb) 2620789Sahrens { 2621789Sahrens arc_buf_hdr_t *hdr = buf->b_hdr; 26223547Smaybee arc_write_callback_t *callback; 26233547Smaybee zio_t *zio; 2624789Sahrens 2625789Sahrens /* this is a private buffer - no locking required */ 26263403Sbmc ASSERT3P(hdr->b_state, ==, arc_anon); 2627789Sahrens ASSERT(BUF_EMPTY(hdr)); 2628789Sahrens ASSERT(!HDR_IO_ERROR(hdr)); 26292237Smaybee ASSERT((hdr->b_flags & ARC_IO_IN_PROGRESS) == 0); 26302237Smaybee ASSERT(hdr->b_acb == 0); 26313547Smaybee callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP); 26323547Smaybee callback->awcb_ready = ready; 26333547Smaybee callback->awcb_done = done; 26343547Smaybee callback->awcb_private = private; 26353547Smaybee callback->awcb_buf = buf; 26361544Seschrock hdr->b_flags |= ARC_IO_IN_PROGRESS; 26373547Smaybee zio = zio_write(pio, spa, checksum, compress, ncopies, txg, bp, 26383547Smaybee buf->b_data, hdr->b_size, arc_write_ready, arc_write_done, callback, 26393547Smaybee priority, flags, zb); 2640789Sahrens 26413547Smaybee return (zio); 2642789Sahrens } 2643789Sahrens 2644789Sahrens int 2645789Sahrens arc_free(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 2646789Sahrens zio_done_func_t *done, void *private, uint32_t arc_flags) 2647789Sahrens { 2648789Sahrens arc_buf_hdr_t *ab; 2649789Sahrens kmutex_t *hash_lock; 2650789Sahrens zio_t *zio; 2651789Sahrens 2652789Sahrens /* 2653789Sahrens * If this buffer is in the cache, release it, so it 2654789Sahrens * can be re-used. 2655789Sahrens */ 2656789Sahrens ab = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_lock); 2657789Sahrens if (ab != NULL) { 2658789Sahrens /* 2659789Sahrens * The checksum of blocks to free is not always 2660789Sahrens * preserved (eg. on the deadlist). However, if it is 2661789Sahrens * nonzero, it should match what we have in the cache. 2662789Sahrens */ 2663789Sahrens ASSERT(bp->blk_cksum.zc_word[0] == 0 || 2664789Sahrens ab->b_cksum0 == bp->blk_cksum.zc_word[0]); 26653403Sbmc if (ab->b_state != arc_anon) 26663403Sbmc arc_change_state(arc_anon, ab, hash_lock); 26672391Smaybee if (HDR_IO_IN_PROGRESS(ab)) { 26682391Smaybee /* 26692391Smaybee * This should only happen when we prefetch. 26702391Smaybee */ 26712391Smaybee ASSERT(ab->b_flags & ARC_PREFETCH); 26722391Smaybee ASSERT3U(ab->b_datacnt, ==, 1); 26732391Smaybee ab->b_flags |= ARC_FREED_IN_READ; 26742391Smaybee if (HDR_IN_HASH_TABLE(ab)) 26752391Smaybee buf_hash_remove(ab); 26762391Smaybee ab->b_arc_access = 0; 26772391Smaybee bzero(&ab->b_dva, sizeof (dva_t)); 26782391Smaybee ab->b_birth = 0; 26792391Smaybee ab->b_cksum0 = 0; 26802391Smaybee ab->b_buf->b_efunc = NULL; 26812391Smaybee ab->b_buf->b_private = NULL; 26822391Smaybee mutex_exit(hash_lock); 26832391Smaybee } else if (refcount_is_zero(&ab->b_refcnt)) { 2684789Sahrens mutex_exit(hash_lock); 26851544Seschrock arc_hdr_destroy(ab); 26863403Sbmc ARCSTAT_BUMP(arcstat_deleted); 2687789Sahrens } else { 26881589Smaybee /* 26892391Smaybee * We still have an active reference on this 26902391Smaybee * buffer. This can happen, e.g., from 26912391Smaybee * dbuf_unoverride(). 26921589Smaybee */ 26932391Smaybee ASSERT(!HDR_IN_HASH_TABLE(ab)); 2694789Sahrens ab->b_arc_access = 0; 2695789Sahrens bzero(&ab->b_dva, sizeof (dva_t)); 2696789Sahrens ab->b_birth = 0; 2697789Sahrens ab->b_cksum0 = 0; 26981544Seschrock ab->b_buf->b_efunc = NULL; 26991544Seschrock ab->b_buf->b_private = NULL; 2700789Sahrens mutex_exit(hash_lock); 2701789Sahrens } 2702789Sahrens } 2703789Sahrens 2704789Sahrens zio = zio_free(pio, spa, txg, bp, done, private); 2705789Sahrens 2706789Sahrens if (arc_flags & ARC_WAIT) 2707789Sahrens return (zio_wait(zio)); 2708789Sahrens 2709789Sahrens ASSERT(arc_flags & ARC_NOWAIT); 2710789Sahrens zio_nowait(zio); 2711789Sahrens 2712789Sahrens return (0); 2713789Sahrens } 2714789Sahrens 2715789Sahrens void 2716789Sahrens arc_tempreserve_clear(uint64_t tempreserve) 2717789Sahrens { 2718789Sahrens atomic_add_64(&arc_tempreserve, -tempreserve); 2719789Sahrens ASSERT((int64_t)arc_tempreserve >= 0); 2720789Sahrens } 2721789Sahrens 2722789Sahrens int 2723789Sahrens arc_tempreserve_space(uint64_t tempreserve) 2724789Sahrens { 2725789Sahrens #ifdef ZFS_DEBUG 2726789Sahrens /* 2727789Sahrens * Once in a while, fail for no reason. Everything should cope. 2728789Sahrens */ 2729789Sahrens if (spa_get_random(10000) == 0) { 2730789Sahrens dprintf("forcing random failure\n"); 2731789Sahrens return (ERESTART); 2732789Sahrens } 2733789Sahrens #endif 27343403Sbmc if (tempreserve > arc_c/4 && !arc_no_grow) 27353403Sbmc arc_c = MIN(arc_c_max, tempreserve * 4); 27363403Sbmc if (tempreserve > arc_c) 2737982Smaybee return (ENOMEM); 2738982Smaybee 2739789Sahrens /* 2740982Smaybee * Throttle writes when the amount of dirty data in the cache 2741982Smaybee * gets too large. We try to keep the cache less than half full 2742982Smaybee * of dirty blocks so that our sync times don't grow too large. 2743982Smaybee * Note: if two requests come in concurrently, we might let them 2744982Smaybee * both succeed, when one of them should fail. Not a huge deal. 2745982Smaybee * 2746982Smaybee * XXX The limit should be adjusted dynamically to keep the time 2747982Smaybee * to sync a dataset fixed (around 1-5 seconds?). 2748789Sahrens */ 2749789Sahrens 27503403Sbmc if (tempreserve + arc_tempreserve + arc_anon->arcs_size > arc_c / 2 && 27513403Sbmc arc_tempreserve + arc_anon->arcs_size > arc_c / 4) { 27524309Smaybee dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK " 27534309Smaybee "anon_data=%lluK tempreserve=%lluK arc_c=%lluK\n", 27544309Smaybee arc_tempreserve>>10, 27554309Smaybee arc_anon->arcs_lsize[ARC_BUFC_METADATA]>>10, 27564309Smaybee arc_anon->arcs_lsize[ARC_BUFC_DATA]>>10, 27573403Sbmc tempreserve>>10, arc_c>>10); 2758789Sahrens return (ERESTART); 2759789Sahrens } 2760789Sahrens atomic_add_64(&arc_tempreserve, tempreserve); 2761789Sahrens return (0); 2762789Sahrens } 2763789Sahrens 2764789Sahrens void 2765789Sahrens arc_init(void) 2766789Sahrens { 2767789Sahrens mutex_init(&arc_reclaim_thr_lock, NULL, MUTEX_DEFAULT, NULL); 2768789Sahrens cv_init(&arc_reclaim_thr_cv, NULL, CV_DEFAULT, NULL); 2769789Sahrens 27702391Smaybee /* Convert seconds to clock ticks */ 27712638Sperrin arc_min_prefetch_lifespan = 1 * hz; 27722391Smaybee 2773789Sahrens /* Start out with 1/8 of all memory */ 27743403Sbmc arc_c = physmem * PAGESIZE / 8; 2775789Sahrens 2776789Sahrens #ifdef _KERNEL 2777789Sahrens /* 2778789Sahrens * On architectures where the physical memory can be larger 2779789Sahrens * than the addressable space (intel in 32-bit mode), we may 2780789Sahrens * need to limit the cache to 1/8 of VM size. 2781789Sahrens */ 27823403Sbmc arc_c = MIN(arc_c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8); 2783789Sahrens #endif 2784789Sahrens 2785982Smaybee /* set min cache to 1/32 of all memory, or 64MB, whichever is more */ 27863403Sbmc arc_c_min = MAX(arc_c / 4, 64<<20); 2787982Smaybee /* set max to 3/4 of all memory, or all but 1GB, whichever is more */ 27883403Sbmc if (arc_c * 8 >= 1<<30) 27893403Sbmc arc_c_max = (arc_c * 8) - (1<<30); 2790789Sahrens else 27913403Sbmc arc_c_max = arc_c_min; 27923403Sbmc arc_c_max = MAX(arc_c * 6, arc_c_max); 27932885Sahrens 27942885Sahrens /* 27952885Sahrens * Allow the tunables to override our calculations if they are 27962885Sahrens * reasonable (ie. over 64MB) 27972885Sahrens */ 27982885Sahrens if (zfs_arc_max > 64<<20 && zfs_arc_max < physmem * PAGESIZE) 27993403Sbmc arc_c_max = zfs_arc_max; 28003403Sbmc if (zfs_arc_min > 64<<20 && zfs_arc_min <= arc_c_max) 28013403Sbmc arc_c_min = zfs_arc_min; 28022885Sahrens 28033403Sbmc arc_c = arc_c_max; 28043403Sbmc arc_p = (arc_c >> 1); 2805789Sahrens 28064309Smaybee /* limit meta-data to 1/4 of the arc capacity */ 28074309Smaybee arc_meta_limit = arc_c_max / 4; 28084645Sek110237 28094645Sek110237 /* Allow the tunable to override if it is reasonable */ 28104645Sek110237 if (zfs_arc_meta_limit > 0 && zfs_arc_meta_limit <= arc_c_max) 28114645Sek110237 arc_meta_limit = zfs_arc_meta_limit; 28124645Sek110237 28134309Smaybee if (arc_c_min < arc_meta_limit / 2 && zfs_arc_min == 0) 28144309Smaybee arc_c_min = arc_meta_limit / 2; 28154309Smaybee 2816789Sahrens /* if kmem_flags are set, lets try to use less memory */ 2817789Sahrens if (kmem_debugging()) 28183403Sbmc arc_c = arc_c / 2; 28193403Sbmc if (arc_c < arc_c_min) 28203403Sbmc arc_c = arc_c_min; 2821789Sahrens 28223403Sbmc arc_anon = &ARC_anon; 28233403Sbmc arc_mru = &ARC_mru; 28243403Sbmc arc_mru_ghost = &ARC_mru_ghost; 28253403Sbmc arc_mfu = &ARC_mfu; 28263403Sbmc arc_mfu_ghost = &ARC_mfu_ghost; 28273403Sbmc arc_size = 0; 2828789Sahrens 28293403Sbmc mutex_init(&arc_anon->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 28303403Sbmc mutex_init(&arc_mru->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 28313403Sbmc mutex_init(&arc_mru_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 28323403Sbmc mutex_init(&arc_mfu->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 28333403Sbmc mutex_init(&arc_mfu_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL); 28342688Smaybee 28354309Smaybee list_create(&arc_mru->arcs_list[ARC_BUFC_METADATA], 28364309Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 28374309Smaybee list_create(&arc_mru->arcs_list[ARC_BUFC_DATA], 28384309Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 28394309Smaybee list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA], 28404309Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 28414309Smaybee list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA], 28424309Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 28434309Smaybee list_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA], 28444309Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 28454309Smaybee list_create(&arc_mfu->arcs_list[ARC_BUFC_DATA], 28464309Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 28474309Smaybee list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA], 28484309Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 28494309Smaybee list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA], 28504309Smaybee sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node)); 2851789Sahrens 2852789Sahrens buf_init(); 2853789Sahrens 2854789Sahrens arc_thread_exit = 0; 28551544Seschrock arc_eviction_list = NULL; 28561544Seschrock mutex_init(&arc_eviction_mtx, NULL, MUTEX_DEFAULT, NULL); 28572887Smaybee bzero(&arc_eviction_hdr, sizeof (arc_buf_hdr_t)); 2858789Sahrens 28593403Sbmc arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED, 28603403Sbmc sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL); 28613403Sbmc 28623403Sbmc if (arc_ksp != NULL) { 28633403Sbmc arc_ksp->ks_data = &arc_stats; 28643403Sbmc kstat_install(arc_ksp); 28653403Sbmc } 28663403Sbmc 2867789Sahrens (void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0, 2868789Sahrens TS_RUN, minclsyspri); 28693158Smaybee 28703158Smaybee arc_dead = FALSE; 2871789Sahrens } 2872789Sahrens 2873789Sahrens void 2874789Sahrens arc_fini(void) 2875789Sahrens { 2876789Sahrens mutex_enter(&arc_reclaim_thr_lock); 2877789Sahrens arc_thread_exit = 1; 2878789Sahrens while (arc_thread_exit != 0) 2879789Sahrens cv_wait(&arc_reclaim_thr_cv, &arc_reclaim_thr_lock); 2880789Sahrens mutex_exit(&arc_reclaim_thr_lock); 2881789Sahrens 2882789Sahrens arc_flush(); 2883789Sahrens 2884789Sahrens arc_dead = TRUE; 2885789Sahrens 28863403Sbmc if (arc_ksp != NULL) { 28873403Sbmc kstat_delete(arc_ksp); 28883403Sbmc arc_ksp = NULL; 28893403Sbmc } 28903403Sbmc 28911544Seschrock mutex_destroy(&arc_eviction_mtx); 2892789Sahrens mutex_destroy(&arc_reclaim_thr_lock); 2893789Sahrens cv_destroy(&arc_reclaim_thr_cv); 2894789Sahrens 28954309Smaybee list_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]); 28964309Smaybee list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]); 28974309Smaybee list_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]); 28984309Smaybee list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]); 28994309Smaybee list_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]); 29004309Smaybee list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]); 29014309Smaybee list_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]); 29024309Smaybee list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]); 2903789Sahrens 29043403Sbmc mutex_destroy(&arc_anon->arcs_mtx); 29053403Sbmc mutex_destroy(&arc_mru->arcs_mtx); 29063403Sbmc mutex_destroy(&arc_mru_ghost->arcs_mtx); 29073403Sbmc mutex_destroy(&arc_mfu->arcs_mtx); 29083403Sbmc mutex_destroy(&arc_mfu_ghost->arcs_mtx); 29092856Snd150628 2910789Sahrens buf_fini(); 2911789Sahrens } 2912