xref: /onnv-gate/usr/src/uts/common/fs/zfs/arc.c (revision 4709)
1789Sahrens /*
2789Sahrens  * CDDL HEADER START
3789Sahrens  *
4789Sahrens  * The contents of this file are subject to the terms of the
51484Sek110237  * Common Development and Distribution License (the "License").
61484Sek110237  * You may not use this file except in compliance with the License.
7789Sahrens  *
8789Sahrens  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9789Sahrens  * or http://www.opensolaris.org/os/licensing.
10789Sahrens  * See the License for the specific language governing permissions
11789Sahrens  * and limitations under the License.
12789Sahrens  *
13789Sahrens  * When distributing Covered Code, include this CDDL HEADER in each
14789Sahrens  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15789Sahrens  * If applicable, add the following below this CDDL HEADER, with the
16789Sahrens  * fields enclosed by brackets "[]" replaced with your own identifying
17789Sahrens  * information: Portions Copyright [yyyy] [name of copyright owner]
18789Sahrens  *
19789Sahrens  * CDDL HEADER END
20789Sahrens  */
21789Sahrens /*
223403Sbmc  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23789Sahrens  * Use is subject to license terms.
24789Sahrens  */
25789Sahrens 
26789Sahrens #pragma ident	"%Z%%M%	%I%	%E% SMI"
27789Sahrens 
28789Sahrens /*
293403Sbmc  * DVA-based Adjustable Replacement Cache
30789Sahrens  *
311544Seschrock  * While much of the theory of operation used here is
321544Seschrock  * based on the self-tuning, low overhead replacement cache
33789Sahrens  * presented by Megiddo and Modha at FAST 2003, there are some
34789Sahrens  * significant differences:
35789Sahrens  *
36789Sahrens  * 1. The Megiddo and Modha model assumes any page is evictable.
37789Sahrens  * Pages in its cache cannot be "locked" into memory.  This makes
38789Sahrens  * the eviction algorithm simple: evict the last page in the list.
39789Sahrens  * This also make the performance characteristics easy to reason
40789Sahrens  * about.  Our cache is not so simple.  At any given moment, some
41789Sahrens  * subset of the blocks in the cache are un-evictable because we
42789Sahrens  * have handed out a reference to them.  Blocks are only evictable
43789Sahrens  * when there are no external references active.  This makes
44789Sahrens  * eviction far more problematic:  we choose to evict the evictable
45789Sahrens  * blocks that are the "lowest" in the list.
46789Sahrens  *
47789Sahrens  * There are times when it is not possible to evict the requested
48789Sahrens  * space.  In these circumstances we are unable to adjust the cache
49789Sahrens  * size.  To prevent the cache growing unbounded at these times we
50789Sahrens  * implement a "cache throttle" that slowes the flow of new data
51789Sahrens  * into the cache until we can make space avaiable.
52789Sahrens  *
53789Sahrens  * 2. The Megiddo and Modha model assumes a fixed cache size.
54789Sahrens  * Pages are evicted when the cache is full and there is a cache
55789Sahrens  * miss.  Our model has a variable sized cache.  It grows with
56789Sahrens  * high use, but also tries to react to memory preasure from the
57789Sahrens  * operating system: decreasing its size when system memory is
58789Sahrens  * tight.
59789Sahrens  *
60789Sahrens  * 3. The Megiddo and Modha model assumes a fixed page size. All
61789Sahrens  * elements of the cache are therefor exactly the same size.  So
62789Sahrens  * when adjusting the cache size following a cache miss, its simply
63789Sahrens  * a matter of choosing a single page to evict.  In our model, we
64789Sahrens  * have variable sized cache blocks (rangeing from 512 bytes to
65789Sahrens  * 128K bytes).  We therefor choose a set of blocks to evict to make
66789Sahrens  * space for a cache miss that approximates as closely as possible
67789Sahrens  * the space used by the new block.
68789Sahrens  *
69789Sahrens  * See also:  "ARC: A Self-Tuning, Low Overhead Replacement Cache"
70789Sahrens  * by N. Megiddo & D. Modha, FAST 2003
71789Sahrens  */
72789Sahrens 
73789Sahrens /*
74789Sahrens  * The locking model:
75789Sahrens  *
76789Sahrens  * A new reference to a cache buffer can be obtained in two
77789Sahrens  * ways: 1) via a hash table lookup using the DVA as a key,
78789Sahrens  * or 2) via one of the ARC lists.  The arc_read() inerface
79789Sahrens  * uses method 1, while the internal arc algorithms for
80789Sahrens  * adjusting the cache use method 2.  We therefor provide two
81789Sahrens  * types of locks: 1) the hash table lock array, and 2) the
82789Sahrens  * arc list locks.
83789Sahrens  *
84789Sahrens  * Buffers do not have their own mutexs, rather they rely on the
85789Sahrens  * hash table mutexs for the bulk of their protection (i.e. most
86789Sahrens  * fields in the arc_buf_hdr_t are protected by these mutexs).
87789Sahrens  *
88789Sahrens  * buf_hash_find() returns the appropriate mutex (held) when it
89789Sahrens  * locates the requested buffer in the hash table.  It returns
90789Sahrens  * NULL for the mutex if the buffer was not in the table.
91789Sahrens  *
92789Sahrens  * buf_hash_remove() expects the appropriate hash mutex to be
93789Sahrens  * already held before it is invoked.
94789Sahrens  *
95789Sahrens  * Each arc state also has a mutex which is used to protect the
96789Sahrens  * buffer list associated with the state.  When attempting to
97789Sahrens  * obtain a hash table lock while holding an arc list lock you
98789Sahrens  * must use: mutex_tryenter() to avoid deadlock.  Also note that
992688Smaybee  * the active state mutex must be held before the ghost state mutex.
100789Sahrens  *
1011544Seschrock  * Arc buffers may have an associated eviction callback function.
1021544Seschrock  * This function will be invoked prior to removing the buffer (e.g.
1031544Seschrock  * in arc_do_user_evicts()).  Note however that the data associated
1041544Seschrock  * with the buffer may be evicted prior to the callback.  The callback
1051544Seschrock  * must be made with *no locks held* (to prevent deadlock).  Additionally,
1061544Seschrock  * the users of callbacks must ensure that their private data is
1071544Seschrock  * protected from simultaneous callbacks from arc_buf_evict()
1081544Seschrock  * and arc_do_user_evicts().
1091544Seschrock  *
110789Sahrens  * Note that the majority of the performance stats are manipulated
111789Sahrens  * with atomic operations.
112789Sahrens  */
113789Sahrens 
114789Sahrens #include <sys/spa.h>
115789Sahrens #include <sys/zio.h>
1163093Sahrens #include <sys/zio_checksum.h>
117789Sahrens #include <sys/zfs_context.h>
118789Sahrens #include <sys/arc.h>
119789Sahrens #include <sys/refcount.h>
120789Sahrens #ifdef _KERNEL
121789Sahrens #include <sys/vmsystm.h>
122789Sahrens #include <vm/anon.h>
123789Sahrens #include <sys/fs/swapnode.h>
1241484Sek110237 #include <sys/dnlc.h>
125789Sahrens #endif
126789Sahrens #include <sys/callb.h>
1273403Sbmc #include <sys/kstat.h>
128789Sahrens 
129789Sahrens static kmutex_t		arc_reclaim_thr_lock;
130789Sahrens static kcondvar_t	arc_reclaim_thr_cv;	/* used to signal reclaim thr */
131789Sahrens static uint8_t		arc_thread_exit;
132789Sahrens 
1331484Sek110237 #define	ARC_REDUCE_DNLC_PERCENT	3
1341484Sek110237 uint_t arc_reduce_dnlc_percent = ARC_REDUCE_DNLC_PERCENT;
1351484Sek110237 
136789Sahrens typedef enum arc_reclaim_strategy {
137789Sahrens 	ARC_RECLAIM_AGGR,		/* Aggressive reclaim strategy */
138789Sahrens 	ARC_RECLAIM_CONS		/* Conservative reclaim strategy */
139789Sahrens } arc_reclaim_strategy_t;
140789Sahrens 
141789Sahrens /* number of seconds before growing cache again */
142789Sahrens static int		arc_grow_retry = 60;
143789Sahrens 
1442391Smaybee /*
1452638Sperrin  * minimum lifespan of a prefetch block in clock ticks
1462638Sperrin  * (initialized in arc_init())
1472391Smaybee  */
1482638Sperrin static int		arc_min_prefetch_lifespan;
1492391Smaybee 
150789Sahrens static int arc_dead;
151789Sahrens 
152789Sahrens /*
1532885Sahrens  * These tunables are for performance analysis.
1542885Sahrens  */
1552885Sahrens uint64_t zfs_arc_max;
1562885Sahrens uint64_t zfs_arc_min;
1574645Sek110237 uint64_t zfs_arc_meta_limit = 0;
1582885Sahrens 
1592885Sahrens /*
1604309Smaybee  * Note that buffers can be in one of 5 states:
161789Sahrens  *	ARC_anon	- anonymous (discussed below)
1621544Seschrock  *	ARC_mru		- recently used, currently cached
1631544Seschrock  *	ARC_mru_ghost	- recentely used, no longer in cache
1641544Seschrock  *	ARC_mfu		- frequently used, currently cached
1651544Seschrock  *	ARC_mfu_ghost	- frequently used, no longer in cache
1664309Smaybee  * When there are no active references to the buffer, they are
1674309Smaybee  * are linked onto a list in one of these arc states.  These are
1684309Smaybee  * the only buffers that can be evicted or deleted.  Within each
1694309Smaybee  * state there are multiple lists, one for meta-data and one for
1704309Smaybee  * non-meta-data.  Meta-data (indirect blocks, blocks of dnodes,
1714309Smaybee  * etc.) is tracked separately so that it can be managed more
1724309Smaybee  * explicitly: favored over data, limited explicitely.
173789Sahrens  *
174789Sahrens  * Anonymous buffers are buffers that are not associated with
175789Sahrens  * a DVA.  These are buffers that hold dirty block copies
176789Sahrens  * before they are written to stable storage.  By definition,
1771544Seschrock  * they are "ref'd" and are considered part of arc_mru
178789Sahrens  * that cannot be freed.  Generally, they will aquire a DVA
1791544Seschrock  * as they are written and migrate onto the arc_mru list.
180789Sahrens  */
181789Sahrens 
182789Sahrens typedef struct arc_state {
1834309Smaybee 	list_t	arcs_list[ARC_BUFC_NUMTYPES];	/* list of evictable buffers */
1844309Smaybee 	uint64_t arcs_lsize[ARC_BUFC_NUMTYPES];	/* amount of evictable data */
1854309Smaybee 	uint64_t arcs_size;	/* total amount of data in this state */
1863403Sbmc 	kmutex_t arcs_mtx;
187789Sahrens } arc_state_t;
188789Sahrens 
189789Sahrens /* The 5 states: */
190789Sahrens static arc_state_t ARC_anon;
1911544Seschrock static arc_state_t ARC_mru;
1921544Seschrock static arc_state_t ARC_mru_ghost;
1931544Seschrock static arc_state_t ARC_mfu;
1941544Seschrock static arc_state_t ARC_mfu_ghost;
195789Sahrens 
1963403Sbmc typedef struct arc_stats {
1973403Sbmc 	kstat_named_t arcstat_hits;
1983403Sbmc 	kstat_named_t arcstat_misses;
1993403Sbmc 	kstat_named_t arcstat_demand_data_hits;
2003403Sbmc 	kstat_named_t arcstat_demand_data_misses;
2013403Sbmc 	kstat_named_t arcstat_demand_metadata_hits;
2023403Sbmc 	kstat_named_t arcstat_demand_metadata_misses;
2033403Sbmc 	kstat_named_t arcstat_prefetch_data_hits;
2043403Sbmc 	kstat_named_t arcstat_prefetch_data_misses;
2053403Sbmc 	kstat_named_t arcstat_prefetch_metadata_hits;
2063403Sbmc 	kstat_named_t arcstat_prefetch_metadata_misses;
2073403Sbmc 	kstat_named_t arcstat_mru_hits;
2083403Sbmc 	kstat_named_t arcstat_mru_ghost_hits;
2093403Sbmc 	kstat_named_t arcstat_mfu_hits;
2103403Sbmc 	kstat_named_t arcstat_mfu_ghost_hits;
2113403Sbmc 	kstat_named_t arcstat_deleted;
2123403Sbmc 	kstat_named_t arcstat_recycle_miss;
2133403Sbmc 	kstat_named_t arcstat_mutex_miss;
2143403Sbmc 	kstat_named_t arcstat_evict_skip;
2153403Sbmc 	kstat_named_t arcstat_hash_elements;
2163403Sbmc 	kstat_named_t arcstat_hash_elements_max;
2173403Sbmc 	kstat_named_t arcstat_hash_collisions;
2183403Sbmc 	kstat_named_t arcstat_hash_chains;
2193403Sbmc 	kstat_named_t arcstat_hash_chain_max;
2203403Sbmc 	kstat_named_t arcstat_p;
2213403Sbmc 	kstat_named_t arcstat_c;
2223403Sbmc 	kstat_named_t arcstat_c_min;
2233403Sbmc 	kstat_named_t arcstat_c_max;
2243403Sbmc 	kstat_named_t arcstat_size;
2253403Sbmc } arc_stats_t;
2263403Sbmc 
2273403Sbmc static arc_stats_t arc_stats = {
2283403Sbmc 	{ "hits",			KSTAT_DATA_UINT64 },
2293403Sbmc 	{ "misses",			KSTAT_DATA_UINT64 },
2303403Sbmc 	{ "demand_data_hits",		KSTAT_DATA_UINT64 },
2313403Sbmc 	{ "demand_data_misses",		KSTAT_DATA_UINT64 },
2323403Sbmc 	{ "demand_metadata_hits",	KSTAT_DATA_UINT64 },
2333403Sbmc 	{ "demand_metadata_misses",	KSTAT_DATA_UINT64 },
2343403Sbmc 	{ "prefetch_data_hits",		KSTAT_DATA_UINT64 },
2353403Sbmc 	{ "prefetch_data_misses",	KSTAT_DATA_UINT64 },
2363403Sbmc 	{ "prefetch_metadata_hits",	KSTAT_DATA_UINT64 },
2373403Sbmc 	{ "prefetch_metadata_misses",	KSTAT_DATA_UINT64 },
2383403Sbmc 	{ "mru_hits",			KSTAT_DATA_UINT64 },
2393403Sbmc 	{ "mru_ghost_hits",		KSTAT_DATA_UINT64 },
2403403Sbmc 	{ "mfu_hits",			KSTAT_DATA_UINT64 },
2413403Sbmc 	{ "mfu_ghost_hits",		KSTAT_DATA_UINT64 },
2423403Sbmc 	{ "deleted",			KSTAT_DATA_UINT64 },
2433403Sbmc 	{ "recycle_miss",		KSTAT_DATA_UINT64 },
2443403Sbmc 	{ "mutex_miss",			KSTAT_DATA_UINT64 },
2453403Sbmc 	{ "evict_skip",			KSTAT_DATA_UINT64 },
2463403Sbmc 	{ "hash_elements",		KSTAT_DATA_UINT64 },
2473403Sbmc 	{ "hash_elements_max",		KSTAT_DATA_UINT64 },
2483403Sbmc 	{ "hash_collisions",		KSTAT_DATA_UINT64 },
2493403Sbmc 	{ "hash_chains",		KSTAT_DATA_UINT64 },
2503403Sbmc 	{ "hash_chain_max",		KSTAT_DATA_UINT64 },
2513403Sbmc 	{ "p",				KSTAT_DATA_UINT64 },
2523403Sbmc 	{ "c",				KSTAT_DATA_UINT64 },
2533403Sbmc 	{ "c_min",			KSTAT_DATA_UINT64 },
2543403Sbmc 	{ "c_max",			KSTAT_DATA_UINT64 },
2553403Sbmc 	{ "size",			KSTAT_DATA_UINT64 }
2563403Sbmc };
257789Sahrens 
2583403Sbmc #define	ARCSTAT(stat)	(arc_stats.stat.value.ui64)
2593403Sbmc 
2603403Sbmc #define	ARCSTAT_INCR(stat, val) \
2613403Sbmc 	atomic_add_64(&arc_stats.stat.value.ui64, (val));
2623403Sbmc 
2633403Sbmc #define	ARCSTAT_BUMP(stat) 	ARCSTAT_INCR(stat, 1)
2643403Sbmc #define	ARCSTAT_BUMPDOWN(stat)	ARCSTAT_INCR(stat, -1)
2653403Sbmc 
2663403Sbmc #define	ARCSTAT_MAX(stat, val) {					\
2673403Sbmc 	uint64_t m;							\
2683403Sbmc 	while ((val) > (m = arc_stats.stat.value.ui64) &&		\
2693403Sbmc 	    (m != atomic_cas_64(&arc_stats.stat.value.ui64, m, (val))))	\
2703403Sbmc 		continue;						\
2713403Sbmc }
2723403Sbmc 
2733403Sbmc #define	ARCSTAT_MAXSTAT(stat) \
2743403Sbmc 	ARCSTAT_MAX(stat##_max, arc_stats.stat.value.ui64)
275789Sahrens 
2763403Sbmc /*
2773403Sbmc  * We define a macro to allow ARC hits/misses to be easily broken down by
2783403Sbmc  * two separate conditions, giving a total of four different subtypes for
2793403Sbmc  * each of hits and misses (so eight statistics total).
2803403Sbmc  */
2813403Sbmc #define	ARCSTAT_CONDSTAT(cond1, stat1, notstat1, cond2, stat2, notstat2, stat) \
2823403Sbmc 	if (cond1) {							\
2833403Sbmc 		if (cond2) {						\
2843403Sbmc 			ARCSTAT_BUMP(arcstat_##stat1##_##stat2##_##stat); \
2853403Sbmc 		} else {						\
2863403Sbmc 			ARCSTAT_BUMP(arcstat_##stat1##_##notstat2##_##stat); \
2873403Sbmc 		}							\
2883403Sbmc 	} else {							\
2893403Sbmc 		if (cond2) {						\
2903403Sbmc 			ARCSTAT_BUMP(arcstat_##notstat1##_##stat2##_##stat); \
2913403Sbmc 		} else {						\
2923403Sbmc 			ARCSTAT_BUMP(arcstat_##notstat1##_##notstat2##_##stat);\
2933403Sbmc 		}							\
2943403Sbmc 	}
295789Sahrens 
2963403Sbmc kstat_t			*arc_ksp;
2973403Sbmc static arc_state_t 	*arc_anon;
2983403Sbmc static arc_state_t	*arc_mru;
2993403Sbmc static arc_state_t	*arc_mru_ghost;
3003403Sbmc static arc_state_t	*arc_mfu;
3013403Sbmc static arc_state_t	*arc_mfu_ghost;
3023403Sbmc 
3033403Sbmc /*
3043403Sbmc  * There are several ARC variables that are critical to export as kstats --
3053403Sbmc  * but we don't want to have to grovel around in the kstat whenever we wish to
3063403Sbmc  * manipulate them.  For these variables, we therefore define them to be in
3073403Sbmc  * terms of the statistic variable.  This assures that we are not introducing
3083403Sbmc  * the possibility of inconsistency by having shadow copies of the variables,
3093403Sbmc  * while still allowing the code to be readable.
3103403Sbmc  */
3113403Sbmc #define	arc_size	ARCSTAT(arcstat_size)	/* actual total arc size */
3123403Sbmc #define	arc_p		ARCSTAT(arcstat_p)	/* target size of MRU */
3133403Sbmc #define	arc_c		ARCSTAT(arcstat_c)	/* target size of cache */
3143403Sbmc #define	arc_c_min	ARCSTAT(arcstat_c_min)	/* min target cache size */
3153403Sbmc #define	arc_c_max	ARCSTAT(arcstat_c_max)	/* max target cache size */
3163403Sbmc 
3173403Sbmc static int		arc_no_grow;	/* Don't try to grow cache size */
3183403Sbmc static uint64_t		arc_tempreserve;
3194309Smaybee static uint64_t		arc_meta_used;
3204309Smaybee static uint64_t		arc_meta_limit;
3214309Smaybee static uint64_t		arc_meta_max = 0;
322789Sahrens 
323789Sahrens typedef struct arc_callback arc_callback_t;
324789Sahrens 
325789Sahrens struct arc_callback {
3263547Smaybee 	void			*acb_private;
327789Sahrens 	arc_done_func_t		*acb_done;
328789Sahrens 	arc_byteswap_func_t	*acb_byteswap;
329789Sahrens 	arc_buf_t		*acb_buf;
330789Sahrens 	zio_t			*acb_zio_dummy;
331789Sahrens 	arc_callback_t		*acb_next;
332789Sahrens };
333789Sahrens 
3343547Smaybee typedef struct arc_write_callback arc_write_callback_t;
3353547Smaybee 
3363547Smaybee struct arc_write_callback {
3373547Smaybee 	void		*awcb_private;
3383547Smaybee 	arc_done_func_t	*awcb_ready;
3393547Smaybee 	arc_done_func_t	*awcb_done;
3403547Smaybee 	arc_buf_t	*awcb_buf;
3413547Smaybee };
3423547Smaybee 
343789Sahrens struct arc_buf_hdr {
344789Sahrens 	/* protected by hash lock */
345789Sahrens 	dva_t			b_dva;
346789Sahrens 	uint64_t		b_birth;
347789Sahrens 	uint64_t		b_cksum0;
348789Sahrens 
3493093Sahrens 	kmutex_t		b_freeze_lock;
3503093Sahrens 	zio_cksum_t		*b_freeze_cksum;
3513093Sahrens 
352789Sahrens 	arc_buf_hdr_t		*b_hash_next;
353789Sahrens 	arc_buf_t		*b_buf;
354789Sahrens 	uint32_t		b_flags;
3551544Seschrock 	uint32_t		b_datacnt;
356789Sahrens 
3573290Sjohansen 	arc_callback_t		*b_acb;
358789Sahrens 	kcondvar_t		b_cv;
3593290Sjohansen 
3603290Sjohansen 	/* immutable */
3613290Sjohansen 	arc_buf_contents_t	b_type;
3623290Sjohansen 	uint64_t		b_size;
3633290Sjohansen 	spa_t			*b_spa;
364789Sahrens 
365789Sahrens 	/* protected by arc state mutex */
366789Sahrens 	arc_state_t		*b_state;
367789Sahrens 	list_node_t		b_arc_node;
368789Sahrens 
369789Sahrens 	/* updated atomically */
370789Sahrens 	clock_t			b_arc_access;
371789Sahrens 
372789Sahrens 	/* self protecting */
373789Sahrens 	refcount_t		b_refcnt;
374789Sahrens };
375789Sahrens 
3761544Seschrock static arc_buf_t *arc_eviction_list;
3771544Seschrock static kmutex_t arc_eviction_mtx;
3782887Smaybee static arc_buf_hdr_t arc_eviction_hdr;
3792688Smaybee static void arc_get_data_buf(arc_buf_t *buf);
3802688Smaybee static void arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock);
3814309Smaybee static int arc_evict_needed(arc_buf_contents_t type);
382*4709Smaybee static void arc_evict_ghost(arc_state_t *state, int64_t bytes);
3831544Seschrock 
3841544Seschrock #define	GHOST_STATE(state)	\
3853403Sbmc 	((state) == arc_mru_ghost || (state) == arc_mfu_ghost)
3861544Seschrock 
387789Sahrens /*
388789Sahrens  * Private ARC flags.  These flags are private ARC only flags that will show up
389789Sahrens  * in b_flags in the arc_hdr_buf_t.  Some flags are publicly declared, and can
390789Sahrens  * be passed in as arc_flags in things like arc_read.  However, these flags
391789Sahrens  * should never be passed and should only be set by ARC code.  When adding new
392789Sahrens  * public flags, make sure not to smash the private ones.
393789Sahrens  */
394789Sahrens 
3951544Seschrock #define	ARC_IN_HASH_TABLE	(1 << 9)	/* this buffer is hashed */
396789Sahrens #define	ARC_IO_IN_PROGRESS	(1 << 10)	/* I/O in progress for buf */
397789Sahrens #define	ARC_IO_ERROR		(1 << 11)	/* I/O failed for buf */
398789Sahrens #define	ARC_FREED_IN_READ	(1 << 12)	/* buf freed while in read */
3991544Seschrock #define	ARC_BUF_AVAILABLE	(1 << 13)	/* block not in active use */
4002391Smaybee #define	ARC_INDIRECT		(1 << 14)	/* this is an indirect block */
401789Sahrens 
4021544Seschrock #define	HDR_IN_HASH_TABLE(hdr)	((hdr)->b_flags & ARC_IN_HASH_TABLE)
403789Sahrens #define	HDR_IO_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_IO_IN_PROGRESS)
404789Sahrens #define	HDR_IO_ERROR(hdr)	((hdr)->b_flags & ARC_IO_ERROR)
405789Sahrens #define	HDR_FREED_IN_READ(hdr)	((hdr)->b_flags & ARC_FREED_IN_READ)
4061544Seschrock #define	HDR_BUF_AVAILABLE(hdr)	((hdr)->b_flags & ARC_BUF_AVAILABLE)
407789Sahrens 
408789Sahrens /*
409789Sahrens  * Hash table routines
410789Sahrens  */
411789Sahrens 
412789Sahrens #define	HT_LOCK_PAD	64
413789Sahrens 
414789Sahrens struct ht_lock {
415789Sahrens 	kmutex_t	ht_lock;
416789Sahrens #ifdef _KERNEL
417789Sahrens 	unsigned char	pad[(HT_LOCK_PAD - sizeof (kmutex_t))];
418789Sahrens #endif
419789Sahrens };
420789Sahrens 
421789Sahrens #define	BUF_LOCKS 256
422789Sahrens typedef struct buf_hash_table {
423789Sahrens 	uint64_t ht_mask;
424789Sahrens 	arc_buf_hdr_t **ht_table;
425789Sahrens 	struct ht_lock ht_locks[BUF_LOCKS];
426789Sahrens } buf_hash_table_t;
427789Sahrens 
428789Sahrens static buf_hash_table_t buf_hash_table;
429789Sahrens 
430789Sahrens #define	BUF_HASH_INDEX(spa, dva, birth) \
431789Sahrens 	(buf_hash(spa, dva, birth) & buf_hash_table.ht_mask)
432789Sahrens #define	BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)])
433789Sahrens #define	BUF_HASH_LOCK(idx)	(&(BUF_HASH_LOCK_NTRY(idx).ht_lock))
434789Sahrens #define	HDR_LOCK(buf) \
435789Sahrens 	(BUF_HASH_LOCK(BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth)))
436789Sahrens 
437789Sahrens uint64_t zfs_crc64_table[256];
438789Sahrens 
439789Sahrens static uint64_t
440789Sahrens buf_hash(spa_t *spa, dva_t *dva, uint64_t birth)
441789Sahrens {
442789Sahrens 	uintptr_t spav = (uintptr_t)spa;
443789Sahrens 	uint8_t *vdva = (uint8_t *)dva;
444789Sahrens 	uint64_t crc = -1ULL;
445789Sahrens 	int i;
446789Sahrens 
447789Sahrens 	ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
448789Sahrens 
449789Sahrens 	for (i = 0; i < sizeof (dva_t); i++)
450789Sahrens 		crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF];
451789Sahrens 
452789Sahrens 	crc ^= (spav>>8) ^ birth;
453789Sahrens 
454789Sahrens 	return (crc);
455789Sahrens }
456789Sahrens 
457789Sahrens #define	BUF_EMPTY(buf)						\
458789Sahrens 	((buf)->b_dva.dva_word[0] == 0 &&			\
459789Sahrens 	(buf)->b_dva.dva_word[1] == 0 &&			\
460789Sahrens 	(buf)->b_birth == 0)
461789Sahrens 
462789Sahrens #define	BUF_EQUAL(spa, dva, birth, buf)				\
463789Sahrens 	((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) &&	\
464789Sahrens 	((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) &&	\
465789Sahrens 	((buf)->b_birth == birth) && ((buf)->b_spa == spa)
466789Sahrens 
467789Sahrens static arc_buf_hdr_t *
468789Sahrens buf_hash_find(spa_t *spa, dva_t *dva, uint64_t birth, kmutex_t **lockp)
469789Sahrens {
470789Sahrens 	uint64_t idx = BUF_HASH_INDEX(spa, dva, birth);
471789Sahrens 	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
472789Sahrens 	arc_buf_hdr_t *buf;
473789Sahrens 
474789Sahrens 	mutex_enter(hash_lock);
475789Sahrens 	for (buf = buf_hash_table.ht_table[idx]; buf != NULL;
476789Sahrens 	    buf = buf->b_hash_next) {
477789Sahrens 		if (BUF_EQUAL(spa, dva, birth, buf)) {
478789Sahrens 			*lockp = hash_lock;
479789Sahrens 			return (buf);
480789Sahrens 		}
481789Sahrens 	}
482789Sahrens 	mutex_exit(hash_lock);
483789Sahrens 	*lockp = NULL;
484789Sahrens 	return (NULL);
485789Sahrens }
486789Sahrens 
487789Sahrens /*
488789Sahrens  * Insert an entry into the hash table.  If there is already an element
489789Sahrens  * equal to elem in the hash table, then the already existing element
490789Sahrens  * will be returned and the new element will not be inserted.
491789Sahrens  * Otherwise returns NULL.
492789Sahrens  */
493789Sahrens static arc_buf_hdr_t *
494789Sahrens buf_hash_insert(arc_buf_hdr_t *buf, kmutex_t **lockp)
495789Sahrens {
496789Sahrens 	uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth);
497789Sahrens 	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
498789Sahrens 	arc_buf_hdr_t *fbuf;
4993403Sbmc 	uint32_t i;
500789Sahrens 
5011544Seschrock 	ASSERT(!HDR_IN_HASH_TABLE(buf));
502789Sahrens 	*lockp = hash_lock;
503789Sahrens 	mutex_enter(hash_lock);
504789Sahrens 	for (fbuf = buf_hash_table.ht_table[idx], i = 0; fbuf != NULL;
505789Sahrens 	    fbuf = fbuf->b_hash_next, i++) {
506789Sahrens 		if (BUF_EQUAL(buf->b_spa, &buf->b_dva, buf->b_birth, fbuf))
507789Sahrens 			return (fbuf);
508789Sahrens 	}
509789Sahrens 
510789Sahrens 	buf->b_hash_next = buf_hash_table.ht_table[idx];
511789Sahrens 	buf_hash_table.ht_table[idx] = buf;
5121544Seschrock 	buf->b_flags |= ARC_IN_HASH_TABLE;
513789Sahrens 
514789Sahrens 	/* collect some hash table performance data */
515789Sahrens 	if (i > 0) {
5163403Sbmc 		ARCSTAT_BUMP(arcstat_hash_collisions);
517789Sahrens 		if (i == 1)
5183403Sbmc 			ARCSTAT_BUMP(arcstat_hash_chains);
5193403Sbmc 
5203403Sbmc 		ARCSTAT_MAX(arcstat_hash_chain_max, i);
521789Sahrens 	}
5223403Sbmc 
5233403Sbmc 	ARCSTAT_BUMP(arcstat_hash_elements);
5243403Sbmc 	ARCSTAT_MAXSTAT(arcstat_hash_elements);
525789Sahrens 
526789Sahrens 	return (NULL);
527789Sahrens }
528789Sahrens 
529789Sahrens static void
530789Sahrens buf_hash_remove(arc_buf_hdr_t *buf)
531789Sahrens {
532789Sahrens 	arc_buf_hdr_t *fbuf, **bufp;
533789Sahrens 	uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth);
534789Sahrens 
535789Sahrens 	ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx)));
5361544Seschrock 	ASSERT(HDR_IN_HASH_TABLE(buf));
537789Sahrens 
538789Sahrens 	bufp = &buf_hash_table.ht_table[idx];
539789Sahrens 	while ((fbuf = *bufp) != buf) {
540789Sahrens 		ASSERT(fbuf != NULL);
541789Sahrens 		bufp = &fbuf->b_hash_next;
542789Sahrens 	}
543789Sahrens 	*bufp = buf->b_hash_next;
544789Sahrens 	buf->b_hash_next = NULL;
5451544Seschrock 	buf->b_flags &= ~ARC_IN_HASH_TABLE;
546789Sahrens 
547789Sahrens 	/* collect some hash table performance data */
5483403Sbmc 	ARCSTAT_BUMPDOWN(arcstat_hash_elements);
5493403Sbmc 
550789Sahrens 	if (buf_hash_table.ht_table[idx] &&
551789Sahrens 	    buf_hash_table.ht_table[idx]->b_hash_next == NULL)
5523403Sbmc 		ARCSTAT_BUMPDOWN(arcstat_hash_chains);
553789Sahrens }
554789Sahrens 
555789Sahrens /*
556789Sahrens  * Global data structures and functions for the buf kmem cache.
557789Sahrens  */
558789Sahrens static kmem_cache_t *hdr_cache;
559789Sahrens static kmem_cache_t *buf_cache;
560789Sahrens 
561789Sahrens static void
562789Sahrens buf_fini(void)
563789Sahrens {
564789Sahrens 	int i;
565789Sahrens 
566789Sahrens 	kmem_free(buf_hash_table.ht_table,
567789Sahrens 	    (buf_hash_table.ht_mask + 1) * sizeof (void *));
568789Sahrens 	for (i = 0; i < BUF_LOCKS; i++)
569789Sahrens 		mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock);
570789Sahrens 	kmem_cache_destroy(hdr_cache);
571789Sahrens 	kmem_cache_destroy(buf_cache);
572789Sahrens }
573789Sahrens 
574789Sahrens /*
575789Sahrens  * Constructor callback - called when the cache is empty
576789Sahrens  * and a new buf is requested.
577789Sahrens  */
578789Sahrens /* ARGSUSED */
579789Sahrens static int
580789Sahrens hdr_cons(void *vbuf, void *unused, int kmflag)
581789Sahrens {
582789Sahrens 	arc_buf_hdr_t *buf = vbuf;
583789Sahrens 
584789Sahrens 	bzero(buf, sizeof (arc_buf_hdr_t));
585789Sahrens 	refcount_create(&buf->b_refcnt);
586789Sahrens 	cv_init(&buf->b_cv, NULL, CV_DEFAULT, NULL);
587789Sahrens 	return (0);
588789Sahrens }
589789Sahrens 
590789Sahrens /*
591789Sahrens  * Destructor callback - called when a cached buf is
592789Sahrens  * no longer required.
593789Sahrens  */
594789Sahrens /* ARGSUSED */
595789Sahrens static void
596789Sahrens hdr_dest(void *vbuf, void *unused)
597789Sahrens {
598789Sahrens 	arc_buf_hdr_t *buf = vbuf;
599789Sahrens 
600789Sahrens 	refcount_destroy(&buf->b_refcnt);
601789Sahrens 	cv_destroy(&buf->b_cv);
602789Sahrens }
603789Sahrens 
604789Sahrens /*
605789Sahrens  * Reclaim callback -- invoked when memory is low.
606789Sahrens  */
607789Sahrens /* ARGSUSED */
608789Sahrens static void
609789Sahrens hdr_recl(void *unused)
610789Sahrens {
611789Sahrens 	dprintf("hdr_recl called\n");
6123158Smaybee 	/*
6133158Smaybee 	 * umem calls the reclaim func when we destroy the buf cache,
6143158Smaybee 	 * which is after we do arc_fini().
6153158Smaybee 	 */
6163158Smaybee 	if (!arc_dead)
6173158Smaybee 		cv_signal(&arc_reclaim_thr_cv);
618789Sahrens }
619789Sahrens 
620789Sahrens static void
621789Sahrens buf_init(void)
622789Sahrens {
623789Sahrens 	uint64_t *ct;
6241544Seschrock 	uint64_t hsize = 1ULL << 12;
625789Sahrens 	int i, j;
626789Sahrens 
627789Sahrens 	/*
628789Sahrens 	 * The hash table is big enough to fill all of physical memory
6291544Seschrock 	 * with an average 64K block size.  The table will take up
6301544Seschrock 	 * totalmem*sizeof(void*)/64K (eg. 128KB/GB with 8-byte pointers).
631789Sahrens 	 */
6321544Seschrock 	while (hsize * 65536 < physmem * PAGESIZE)
633789Sahrens 		hsize <<= 1;
6341544Seschrock retry:
635789Sahrens 	buf_hash_table.ht_mask = hsize - 1;
6361544Seschrock 	buf_hash_table.ht_table =
6371544Seschrock 	    kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP);
6381544Seschrock 	if (buf_hash_table.ht_table == NULL) {
6391544Seschrock 		ASSERT(hsize > (1ULL << 8));
6401544Seschrock 		hsize >>= 1;
6411544Seschrock 		goto retry;
6421544Seschrock 	}
643789Sahrens 
644789Sahrens 	hdr_cache = kmem_cache_create("arc_buf_hdr_t", sizeof (arc_buf_hdr_t),
645789Sahrens 	    0, hdr_cons, hdr_dest, hdr_recl, NULL, NULL, 0);
646789Sahrens 	buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t),
647789Sahrens 	    0, NULL, NULL, NULL, NULL, NULL, 0);
648789Sahrens 
649789Sahrens 	for (i = 0; i < 256; i++)
650789Sahrens 		for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--)
651789Sahrens 			*ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY);
652789Sahrens 
653789Sahrens 	for (i = 0; i < BUF_LOCKS; i++) {
654789Sahrens 		mutex_init(&buf_hash_table.ht_locks[i].ht_lock,
655789Sahrens 		    NULL, MUTEX_DEFAULT, NULL);
656789Sahrens 	}
657789Sahrens }
658789Sahrens 
659789Sahrens #define	ARC_MINTIME	(hz>>4) /* 62 ms */
660789Sahrens 
661789Sahrens static void
6623093Sahrens arc_cksum_verify(arc_buf_t *buf)
6633093Sahrens {
6643093Sahrens 	zio_cksum_t zc;
6653093Sahrens 
6663312Sahrens 	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
6673093Sahrens 		return;
6683093Sahrens 
6693093Sahrens 	mutex_enter(&buf->b_hdr->b_freeze_lock);
6703265Sahrens 	if (buf->b_hdr->b_freeze_cksum == NULL ||
6713265Sahrens 	    (buf->b_hdr->b_flags & ARC_IO_ERROR)) {
6723093Sahrens 		mutex_exit(&buf->b_hdr->b_freeze_lock);
6733093Sahrens 		return;
6743093Sahrens 	}
6753093Sahrens 	fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
6763093Sahrens 	if (!ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc))
6773093Sahrens 		panic("buffer modified while frozen!");
6783093Sahrens 	mutex_exit(&buf->b_hdr->b_freeze_lock);
6793093Sahrens }
6803093Sahrens 
6813093Sahrens static void
6823093Sahrens arc_cksum_compute(arc_buf_t *buf)
6833093Sahrens {
6843312Sahrens 	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
6853093Sahrens 		return;
6863093Sahrens 
6873093Sahrens 	mutex_enter(&buf->b_hdr->b_freeze_lock);
6883093Sahrens 	if (buf->b_hdr->b_freeze_cksum != NULL) {
6893093Sahrens 		mutex_exit(&buf->b_hdr->b_freeze_lock);
6903093Sahrens 		return;
6913093Sahrens 	}
6923093Sahrens 	buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP);
6933093Sahrens 	fletcher_2_native(buf->b_data, buf->b_hdr->b_size,
6943093Sahrens 	    buf->b_hdr->b_freeze_cksum);
6953093Sahrens 	mutex_exit(&buf->b_hdr->b_freeze_lock);
6963093Sahrens }
6973093Sahrens 
6983093Sahrens void
6993093Sahrens arc_buf_thaw(arc_buf_t *buf)
7003093Sahrens {
7013312Sahrens 	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
7023093Sahrens 		return;
7033093Sahrens 
7043403Sbmc 	if (buf->b_hdr->b_state != arc_anon)
7053093Sahrens 		panic("modifying non-anon buffer!");
7063093Sahrens 	if (buf->b_hdr->b_flags & ARC_IO_IN_PROGRESS)
7073093Sahrens 		panic("modifying buffer while i/o in progress!");
7083093Sahrens 	arc_cksum_verify(buf);
7093093Sahrens 	mutex_enter(&buf->b_hdr->b_freeze_lock);
7103093Sahrens 	if (buf->b_hdr->b_freeze_cksum != NULL) {
7113093Sahrens 		kmem_free(buf->b_hdr->b_freeze_cksum, sizeof (zio_cksum_t));
7123093Sahrens 		buf->b_hdr->b_freeze_cksum = NULL;
7133093Sahrens 	}
7143093Sahrens 	mutex_exit(&buf->b_hdr->b_freeze_lock);
7153093Sahrens }
7163093Sahrens 
7173093Sahrens void
7183093Sahrens arc_buf_freeze(arc_buf_t *buf)
7193093Sahrens {
7203312Sahrens 	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
7213312Sahrens 		return;
7223312Sahrens 
7233093Sahrens 	ASSERT(buf->b_hdr->b_freeze_cksum != NULL ||
7243403Sbmc 	    buf->b_hdr->b_state == arc_anon);
7253093Sahrens 	arc_cksum_compute(buf);
7263093Sahrens }
7273093Sahrens 
7283093Sahrens static void
729789Sahrens add_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag)
730789Sahrens {
731789Sahrens 	ASSERT(MUTEX_HELD(hash_lock));
732789Sahrens 
733789Sahrens 	if ((refcount_add(&ab->b_refcnt, tag) == 1) &&
7343403Sbmc 	    (ab->b_state != arc_anon)) {
7353700Sek110237 		uint64_t delta = ab->b_size * ab->b_datacnt;
7364309Smaybee 		list_t *list = &ab->b_state->arcs_list[ab->b_type];
7374309Smaybee 		uint64_t *size = &ab->b_state->arcs_lsize[ab->b_type];
738789Sahrens 
7393403Sbmc 		ASSERT(!MUTEX_HELD(&ab->b_state->arcs_mtx));
7403403Sbmc 		mutex_enter(&ab->b_state->arcs_mtx);
741789Sahrens 		ASSERT(list_link_active(&ab->b_arc_node));
7424309Smaybee 		list_remove(list, ab);
7431544Seschrock 		if (GHOST_STATE(ab->b_state)) {
7441544Seschrock 			ASSERT3U(ab->b_datacnt, ==, 0);
7451544Seschrock 			ASSERT3P(ab->b_buf, ==, NULL);
7461544Seschrock 			delta = ab->b_size;
7471544Seschrock 		}
7481544Seschrock 		ASSERT(delta > 0);
7494309Smaybee 		ASSERT3U(*size, >=, delta);
7504309Smaybee 		atomic_add_64(size, -delta);
7513403Sbmc 		mutex_exit(&ab->b_state->arcs_mtx);
7522391Smaybee 		/* remove the prefetch flag is we get a reference */
7532391Smaybee 		if (ab->b_flags & ARC_PREFETCH)
7542391Smaybee 			ab->b_flags &= ~ARC_PREFETCH;
755789Sahrens 	}
756789Sahrens }
757789Sahrens 
758789Sahrens static int
759789Sahrens remove_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag)
760789Sahrens {
761789Sahrens 	int cnt;
7623403Sbmc 	arc_state_t *state = ab->b_state;
763789Sahrens 
7643403Sbmc 	ASSERT(state == arc_anon || MUTEX_HELD(hash_lock));
7653403Sbmc 	ASSERT(!GHOST_STATE(state));
766789Sahrens 
767789Sahrens 	if (((cnt = refcount_remove(&ab->b_refcnt, tag)) == 0) &&
7683403Sbmc 	    (state != arc_anon)) {
7694309Smaybee 		uint64_t *size = &state->arcs_lsize[ab->b_type];
7704309Smaybee 
7713403Sbmc 		ASSERT(!MUTEX_HELD(&state->arcs_mtx));
7723403Sbmc 		mutex_enter(&state->arcs_mtx);
773789Sahrens 		ASSERT(!list_link_active(&ab->b_arc_node));
7744309Smaybee 		list_insert_head(&state->arcs_list[ab->b_type], ab);
7751544Seschrock 		ASSERT(ab->b_datacnt > 0);
7764309Smaybee 		atomic_add_64(size, ab->b_size * ab->b_datacnt);
7773403Sbmc 		mutex_exit(&state->arcs_mtx);
778789Sahrens 	}
779789Sahrens 	return (cnt);
780789Sahrens }
781789Sahrens 
782789Sahrens /*
783789Sahrens  * Move the supplied buffer to the indicated state.  The mutex
784789Sahrens  * for the buffer must be held by the caller.
785789Sahrens  */
786789Sahrens static void
7871544Seschrock arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *ab, kmutex_t *hash_lock)
788789Sahrens {
7891544Seschrock 	arc_state_t *old_state = ab->b_state;
7903700Sek110237 	int64_t refcnt = refcount_count(&ab->b_refcnt);
7913700Sek110237 	uint64_t from_delta, to_delta;
792789Sahrens 
793789Sahrens 	ASSERT(MUTEX_HELD(hash_lock));
7941544Seschrock 	ASSERT(new_state != old_state);
7951544Seschrock 	ASSERT(refcnt == 0 || ab->b_datacnt > 0);
7961544Seschrock 	ASSERT(ab->b_datacnt == 0 || !GHOST_STATE(new_state));
7971544Seschrock 
7981544Seschrock 	from_delta = to_delta = ab->b_datacnt * ab->b_size;
799789Sahrens 
800789Sahrens 	/*
801789Sahrens 	 * If this buffer is evictable, transfer it from the
802789Sahrens 	 * old state list to the new state list.
803789Sahrens 	 */
8041544Seschrock 	if (refcnt == 0) {
8053403Sbmc 		if (old_state != arc_anon) {
8063403Sbmc 			int use_mutex = !MUTEX_HELD(&old_state->arcs_mtx);
8074309Smaybee 			uint64_t *size = &old_state->arcs_lsize[ab->b_type];
8081544Seschrock 
8091544Seschrock 			if (use_mutex)
8103403Sbmc 				mutex_enter(&old_state->arcs_mtx);
8111544Seschrock 
8121544Seschrock 			ASSERT(list_link_active(&ab->b_arc_node));
8134309Smaybee 			list_remove(&old_state->arcs_list[ab->b_type], ab);
814789Sahrens 
8152391Smaybee 			/*
8162391Smaybee 			 * If prefetching out of the ghost cache,
8172391Smaybee 			 * we will have a non-null datacnt.
8182391Smaybee 			 */
8192391Smaybee 			if (GHOST_STATE(old_state) && ab->b_datacnt == 0) {
8202391Smaybee 				/* ghost elements have a ghost size */
8211544Seschrock 				ASSERT(ab->b_buf == NULL);
8221544Seschrock 				from_delta = ab->b_size;
823789Sahrens 			}
8244309Smaybee 			ASSERT3U(*size, >=, from_delta);
8254309Smaybee 			atomic_add_64(size, -from_delta);
8261544Seschrock 
8271544Seschrock 			if (use_mutex)
8283403Sbmc 				mutex_exit(&old_state->arcs_mtx);
829789Sahrens 		}
8303403Sbmc 		if (new_state != arc_anon) {
8313403Sbmc 			int use_mutex = !MUTEX_HELD(&new_state->arcs_mtx);
8324309Smaybee 			uint64_t *size = &new_state->arcs_lsize[ab->b_type];
833789Sahrens 
8341544Seschrock 			if (use_mutex)
8353403Sbmc 				mutex_enter(&new_state->arcs_mtx);
8361544Seschrock 
8374309Smaybee 			list_insert_head(&new_state->arcs_list[ab->b_type], ab);
8381544Seschrock 
8391544Seschrock 			/* ghost elements have a ghost size */
8401544Seschrock 			if (GHOST_STATE(new_state)) {
8411544Seschrock 				ASSERT(ab->b_datacnt == 0);
8421544Seschrock 				ASSERT(ab->b_buf == NULL);
8431544Seschrock 				to_delta = ab->b_size;
8441544Seschrock 			}
8454309Smaybee 			atomic_add_64(size, to_delta);
8461544Seschrock 
8471544Seschrock 			if (use_mutex)
8483403Sbmc 				mutex_exit(&new_state->arcs_mtx);
849789Sahrens 		}
850789Sahrens 	}
851789Sahrens 
852789Sahrens 	ASSERT(!BUF_EMPTY(ab));
8533403Sbmc 	if (new_state == arc_anon && old_state != arc_anon) {
854789Sahrens 		buf_hash_remove(ab);
855789Sahrens 	}
856789Sahrens 
8571544Seschrock 	/* adjust state sizes */
8581544Seschrock 	if (to_delta)
8593403Sbmc 		atomic_add_64(&new_state->arcs_size, to_delta);
8601544Seschrock 	if (from_delta) {
8613403Sbmc 		ASSERT3U(old_state->arcs_size, >=, from_delta);
8623403Sbmc 		atomic_add_64(&old_state->arcs_size, -from_delta);
863789Sahrens 	}
864789Sahrens 	ab->b_state = new_state;
865789Sahrens }
866789Sahrens 
8674309Smaybee void
8684309Smaybee arc_space_consume(uint64_t space)
8694309Smaybee {
8704309Smaybee 	atomic_add_64(&arc_meta_used, space);
8714309Smaybee 	atomic_add_64(&arc_size, space);
8724309Smaybee }
8734309Smaybee 
8744309Smaybee void
8754309Smaybee arc_space_return(uint64_t space)
8764309Smaybee {
8774309Smaybee 	ASSERT(arc_meta_used >= space);
8784309Smaybee 	if (arc_meta_max < arc_meta_used)
8794309Smaybee 		arc_meta_max = arc_meta_used;
8804309Smaybee 	atomic_add_64(&arc_meta_used, -space);
8814309Smaybee 	ASSERT(arc_size >= space);
8824309Smaybee 	atomic_add_64(&arc_size, -space);
8834309Smaybee }
8844309Smaybee 
8854309Smaybee void *
8864309Smaybee arc_data_buf_alloc(uint64_t size)
8874309Smaybee {
8884309Smaybee 	if (arc_evict_needed(ARC_BUFC_DATA))
8894309Smaybee 		cv_signal(&arc_reclaim_thr_cv);
8904309Smaybee 	atomic_add_64(&arc_size, size);
8914309Smaybee 	return (zio_data_buf_alloc(size));
8924309Smaybee }
8934309Smaybee 
8944309Smaybee void
8954309Smaybee arc_data_buf_free(void *buf, uint64_t size)
8964309Smaybee {
8974309Smaybee 	zio_data_buf_free(buf, size);
8984309Smaybee 	ASSERT(arc_size >= size);
8994309Smaybee 	atomic_add_64(&arc_size, -size);
9004309Smaybee }
9014309Smaybee 
902789Sahrens arc_buf_t *
9033290Sjohansen arc_buf_alloc(spa_t *spa, int size, void *tag, arc_buf_contents_t type)
904789Sahrens {
905789Sahrens 	arc_buf_hdr_t *hdr;
906789Sahrens 	arc_buf_t *buf;
907789Sahrens 
908789Sahrens 	ASSERT3U(size, >, 0);
909789Sahrens 	hdr = kmem_cache_alloc(hdr_cache, KM_SLEEP);
910789Sahrens 	ASSERT(BUF_EMPTY(hdr));
911789Sahrens 	hdr->b_size = size;
9123290Sjohansen 	hdr->b_type = type;
913789Sahrens 	hdr->b_spa = spa;
9143403Sbmc 	hdr->b_state = arc_anon;
915789Sahrens 	hdr->b_arc_access = 0;
916789Sahrens 	buf = kmem_cache_alloc(buf_cache, KM_SLEEP);
917789Sahrens 	buf->b_hdr = hdr;
9182688Smaybee 	buf->b_data = NULL;
9191544Seschrock 	buf->b_efunc = NULL;
9201544Seschrock 	buf->b_private = NULL;
921789Sahrens 	buf->b_next = NULL;
922789Sahrens 	hdr->b_buf = buf;
9232688Smaybee 	arc_get_data_buf(buf);
9241544Seschrock 	hdr->b_datacnt = 1;
925789Sahrens 	hdr->b_flags = 0;
926789Sahrens 	ASSERT(refcount_is_zero(&hdr->b_refcnt));
927789Sahrens 	(void) refcount_add(&hdr->b_refcnt, tag);
928789Sahrens 
929789Sahrens 	return (buf);
930789Sahrens }
931789Sahrens 
9322688Smaybee static arc_buf_t *
9332688Smaybee arc_buf_clone(arc_buf_t *from)
9341544Seschrock {
9352688Smaybee 	arc_buf_t *buf;
9362688Smaybee 	arc_buf_hdr_t *hdr = from->b_hdr;
9372688Smaybee 	uint64_t size = hdr->b_size;
9381544Seschrock 
9392688Smaybee 	buf = kmem_cache_alloc(buf_cache, KM_SLEEP);
9402688Smaybee 	buf->b_hdr = hdr;
9412688Smaybee 	buf->b_data = NULL;
9422688Smaybee 	buf->b_efunc = NULL;
9432688Smaybee 	buf->b_private = NULL;
9442688Smaybee 	buf->b_next = hdr->b_buf;
9452688Smaybee 	hdr->b_buf = buf;
9462688Smaybee 	arc_get_data_buf(buf);
9472688Smaybee 	bcopy(from->b_data, buf->b_data, size);
9482688Smaybee 	hdr->b_datacnt += 1;
9492688Smaybee 	return (buf);
9501544Seschrock }
9511544Seschrock 
9521544Seschrock void
9531544Seschrock arc_buf_add_ref(arc_buf_t *buf, void* tag)
9541544Seschrock {
9552887Smaybee 	arc_buf_hdr_t *hdr;
9561544Seschrock 	kmutex_t *hash_lock;
9571544Seschrock 
9582724Smaybee 	/*
9592724Smaybee 	 * Check to see if this buffer is currently being evicted via
9602887Smaybee 	 * arc_do_user_evicts().
9612724Smaybee 	 */
9622887Smaybee 	mutex_enter(&arc_eviction_mtx);
9632887Smaybee 	hdr = buf->b_hdr;
9642887Smaybee 	if (hdr == NULL) {
9652887Smaybee 		mutex_exit(&arc_eviction_mtx);
9662724Smaybee 		return;
9672887Smaybee 	}
9682887Smaybee 	hash_lock = HDR_LOCK(hdr);
9692887Smaybee 	mutex_exit(&arc_eviction_mtx);
9702724Smaybee 
9712724Smaybee 	mutex_enter(hash_lock);
9721544Seschrock 	if (buf->b_data == NULL) {
9731544Seschrock 		/*
9741544Seschrock 		 * This buffer is evicted.
9751544Seschrock 		 */
9762724Smaybee 		mutex_exit(hash_lock);
9771544Seschrock 		return;
9781544Seschrock 	}
9791544Seschrock 
9802724Smaybee 	ASSERT(buf->b_hdr == hdr);
9813403Sbmc 	ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
9821544Seschrock 	add_reference(hdr, hash_lock, tag);
9832688Smaybee 	arc_access(hdr, hash_lock);
9842688Smaybee 	mutex_exit(hash_lock);
9853403Sbmc 	ARCSTAT_BUMP(arcstat_hits);
9863403Sbmc 	ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
9873403Sbmc 	    demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
9883403Sbmc 	    data, metadata, hits);
9891544Seschrock }
9901544Seschrock 
991789Sahrens static void
9922688Smaybee arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t all)
9931544Seschrock {
9941544Seschrock 	arc_buf_t **bufp;
9951544Seschrock 
9961544Seschrock 	/* free up data associated with the buf */
9971544Seschrock 	if (buf->b_data) {
9981544Seschrock 		arc_state_t *state = buf->b_hdr->b_state;
9991544Seschrock 		uint64_t size = buf->b_hdr->b_size;
10003290Sjohansen 		arc_buf_contents_t type = buf->b_hdr->b_type;
10011544Seschrock 
10023093Sahrens 		arc_cksum_verify(buf);
10032688Smaybee 		if (!recycle) {
10043290Sjohansen 			if (type == ARC_BUFC_METADATA) {
10053290Sjohansen 				zio_buf_free(buf->b_data, size);
10064309Smaybee 				arc_space_return(size);
10073290Sjohansen 			} else {
10083290Sjohansen 				ASSERT(type == ARC_BUFC_DATA);
10093290Sjohansen 				zio_data_buf_free(buf->b_data, size);
10104309Smaybee 				atomic_add_64(&arc_size, -size);
10113290Sjohansen 			}
10122688Smaybee 		}
10131544Seschrock 		if (list_link_active(&buf->b_hdr->b_arc_node)) {
10144309Smaybee 			uint64_t *cnt = &state->arcs_lsize[type];
10154309Smaybee 
10161544Seschrock 			ASSERT(refcount_is_zero(&buf->b_hdr->b_refcnt));
10173403Sbmc 			ASSERT(state != arc_anon);
10184309Smaybee 
10194309Smaybee 			ASSERT3U(*cnt, >=, size);
10204309Smaybee 			atomic_add_64(cnt, -size);
10211544Seschrock 		}
10223403Sbmc 		ASSERT3U(state->arcs_size, >=, size);
10233403Sbmc 		atomic_add_64(&state->arcs_size, -size);
10241544Seschrock 		buf->b_data = NULL;
10251544Seschrock 		ASSERT(buf->b_hdr->b_datacnt > 0);
10261544Seschrock 		buf->b_hdr->b_datacnt -= 1;
10271544Seschrock 	}
10281544Seschrock 
10291544Seschrock 	/* only remove the buf if requested */
10301544Seschrock 	if (!all)
10311544Seschrock 		return;
10321544Seschrock 
10331544Seschrock 	/* remove the buf from the hdr list */
10341544Seschrock 	for (bufp = &buf->b_hdr->b_buf; *bufp != buf; bufp = &(*bufp)->b_next)
10351544Seschrock 		continue;
10361544Seschrock 	*bufp = buf->b_next;
10371544Seschrock 
10381544Seschrock 	ASSERT(buf->b_efunc == NULL);
10391544Seschrock 
10401544Seschrock 	/* clean up the buf */
10411544Seschrock 	buf->b_hdr = NULL;
10421544Seschrock 	kmem_cache_free(buf_cache, buf);
10431544Seschrock }
10441544Seschrock 
10451544Seschrock static void
10461544Seschrock arc_hdr_destroy(arc_buf_hdr_t *hdr)
1047789Sahrens {
1048789Sahrens 	ASSERT(refcount_is_zero(&hdr->b_refcnt));
10493403Sbmc 	ASSERT3P(hdr->b_state, ==, arc_anon);
10501544Seschrock 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
1051789Sahrens 
1052789Sahrens 	if (!BUF_EMPTY(hdr)) {
10531544Seschrock 		ASSERT(!HDR_IN_HASH_TABLE(hdr));
1054789Sahrens 		bzero(&hdr->b_dva, sizeof (dva_t));
1055789Sahrens 		hdr->b_birth = 0;
1056789Sahrens 		hdr->b_cksum0 = 0;
1057789Sahrens 	}
10581544Seschrock 	while (hdr->b_buf) {
1059789Sahrens 		arc_buf_t *buf = hdr->b_buf;
1060789Sahrens 
10611544Seschrock 		if (buf->b_efunc) {
10621544Seschrock 			mutex_enter(&arc_eviction_mtx);
10631544Seschrock 			ASSERT(buf->b_hdr != NULL);
10642688Smaybee 			arc_buf_destroy(hdr->b_buf, FALSE, FALSE);
10651544Seschrock 			hdr->b_buf = buf->b_next;
10662887Smaybee 			buf->b_hdr = &arc_eviction_hdr;
10671544Seschrock 			buf->b_next = arc_eviction_list;
10681544Seschrock 			arc_eviction_list = buf;
10691544Seschrock 			mutex_exit(&arc_eviction_mtx);
10701544Seschrock 		} else {
10712688Smaybee 			arc_buf_destroy(hdr->b_buf, FALSE, TRUE);
10721544Seschrock 		}
1073789Sahrens 	}
10743093Sahrens 	if (hdr->b_freeze_cksum != NULL) {
10753093Sahrens 		kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
10763093Sahrens 		hdr->b_freeze_cksum = NULL;
10773093Sahrens 	}
10781544Seschrock 
1079789Sahrens 	ASSERT(!list_link_active(&hdr->b_arc_node));
1080789Sahrens 	ASSERT3P(hdr->b_hash_next, ==, NULL);
1081789Sahrens 	ASSERT3P(hdr->b_acb, ==, NULL);
1082789Sahrens 	kmem_cache_free(hdr_cache, hdr);
1083789Sahrens }
1084789Sahrens 
1085789Sahrens void
1086789Sahrens arc_buf_free(arc_buf_t *buf, void *tag)
1087789Sahrens {
1088789Sahrens 	arc_buf_hdr_t *hdr = buf->b_hdr;
10893403Sbmc 	int hashed = hdr->b_state != arc_anon;
10901544Seschrock 
10911544Seschrock 	ASSERT(buf->b_efunc == NULL);
10921544Seschrock 	ASSERT(buf->b_data != NULL);
10931544Seschrock 
10941544Seschrock 	if (hashed) {
10951544Seschrock 		kmutex_t *hash_lock = HDR_LOCK(hdr);
10961544Seschrock 
10971544Seschrock 		mutex_enter(hash_lock);
10981544Seschrock 		(void) remove_reference(hdr, hash_lock, tag);
10991544Seschrock 		if (hdr->b_datacnt > 1)
11002688Smaybee 			arc_buf_destroy(buf, FALSE, TRUE);
11011544Seschrock 		else
11021544Seschrock 			hdr->b_flags |= ARC_BUF_AVAILABLE;
11031544Seschrock 		mutex_exit(hash_lock);
11041544Seschrock 	} else if (HDR_IO_IN_PROGRESS(hdr)) {
11051544Seschrock 		int destroy_hdr;
11061544Seschrock 		/*
11071544Seschrock 		 * We are in the middle of an async write.  Don't destroy
11081544Seschrock 		 * this buffer unless the write completes before we finish
11091544Seschrock 		 * decrementing the reference count.
11101544Seschrock 		 */
11111544Seschrock 		mutex_enter(&arc_eviction_mtx);
11121544Seschrock 		(void) remove_reference(hdr, NULL, tag);
11131544Seschrock 		ASSERT(refcount_is_zero(&hdr->b_refcnt));
11141544Seschrock 		destroy_hdr = !HDR_IO_IN_PROGRESS(hdr);
11151544Seschrock 		mutex_exit(&arc_eviction_mtx);
11161544Seschrock 		if (destroy_hdr)
11171544Seschrock 			arc_hdr_destroy(hdr);
11181544Seschrock 	} else {
11191544Seschrock 		if (remove_reference(hdr, NULL, tag) > 0) {
11201544Seschrock 			ASSERT(HDR_IO_ERROR(hdr));
11212688Smaybee 			arc_buf_destroy(buf, FALSE, TRUE);
11221544Seschrock 		} else {
11231544Seschrock 			arc_hdr_destroy(hdr);
11241544Seschrock 		}
11251544Seschrock 	}
11261544Seschrock }
11271544Seschrock 
11281544Seschrock int
11291544Seschrock arc_buf_remove_ref(arc_buf_t *buf, void* tag)
11301544Seschrock {
11311544Seschrock 	arc_buf_hdr_t *hdr = buf->b_hdr;
1132789Sahrens 	kmutex_t *hash_lock = HDR_LOCK(hdr);
11331544Seschrock 	int no_callback = (buf->b_efunc == NULL);
11341544Seschrock 
11353403Sbmc 	if (hdr->b_state == arc_anon) {
11361544Seschrock 		arc_buf_free(buf, tag);
11371544Seschrock 		return (no_callback);
11381544Seschrock 	}
1139789Sahrens 
1140789Sahrens 	mutex_enter(hash_lock);
11413403Sbmc 	ASSERT(hdr->b_state != arc_anon);
11421544Seschrock 	ASSERT(buf->b_data != NULL);
1143789Sahrens 
11441544Seschrock 	(void) remove_reference(hdr, hash_lock, tag);
11451544Seschrock 	if (hdr->b_datacnt > 1) {
11461544Seschrock 		if (no_callback)
11472688Smaybee 			arc_buf_destroy(buf, FALSE, TRUE);
11481544Seschrock 	} else if (no_callback) {
11491544Seschrock 		ASSERT(hdr->b_buf == buf && buf->b_next == NULL);
11501544Seschrock 		hdr->b_flags |= ARC_BUF_AVAILABLE;
1151789Sahrens 	}
11521544Seschrock 	ASSERT(no_callback || hdr->b_datacnt > 1 ||
11531544Seschrock 	    refcount_is_zero(&hdr->b_refcnt));
1154789Sahrens 	mutex_exit(hash_lock);
11551544Seschrock 	return (no_callback);
1156789Sahrens }
1157789Sahrens 
1158789Sahrens int
1159789Sahrens arc_buf_size(arc_buf_t *buf)
1160789Sahrens {
1161789Sahrens 	return (buf->b_hdr->b_size);
1162789Sahrens }
1163789Sahrens 
1164789Sahrens /*
1165789Sahrens  * Evict buffers from list until we've removed the specified number of
1166789Sahrens  * bytes.  Move the removed buffers to the appropriate evict state.
11672688Smaybee  * If the recycle flag is set, then attempt to "recycle" a buffer:
11682688Smaybee  * - look for a buffer to evict that is `bytes' long.
11692688Smaybee  * - return the data block from this buffer rather than freeing it.
11702688Smaybee  * This flag is used by callers that are trying to make space for a
11712688Smaybee  * new buffer in a full arc cache.
1172789Sahrens  */
11732688Smaybee static void *
11743290Sjohansen arc_evict(arc_state_t *state, int64_t bytes, boolean_t recycle,
11753290Sjohansen     arc_buf_contents_t type)
1176789Sahrens {
1177789Sahrens 	arc_state_t *evicted_state;
11782688Smaybee 	uint64_t bytes_evicted = 0, skipped = 0, missed = 0;
11792918Smaybee 	arc_buf_hdr_t *ab, *ab_prev = NULL;
11804309Smaybee 	list_t *list = &state->arcs_list[type];
1181789Sahrens 	kmutex_t *hash_lock;
11822688Smaybee 	boolean_t have_lock;
11832918Smaybee 	void *stolen = NULL;
1184789Sahrens 
11853403Sbmc 	ASSERT(state == arc_mru || state == arc_mfu);
1186789Sahrens 
11873403Sbmc 	evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
1188789Sahrens 
11893403Sbmc 	mutex_enter(&state->arcs_mtx);
11903403Sbmc 	mutex_enter(&evicted_state->arcs_mtx);
1191789Sahrens 
11924309Smaybee 	for (ab = list_tail(list); ab; ab = ab_prev) {
11934309Smaybee 		ab_prev = list_prev(list, ab);
11942391Smaybee 		/* prefetch buffers have a minimum lifespan */
11952688Smaybee 		if (HDR_IO_IN_PROGRESS(ab) ||
11962688Smaybee 		    (ab->b_flags & (ARC_PREFETCH|ARC_INDIRECT) &&
11972688Smaybee 		    lbolt - ab->b_arc_access < arc_min_prefetch_lifespan)) {
11982391Smaybee 			skipped++;
11992391Smaybee 			continue;
12002391Smaybee 		}
12012918Smaybee 		/* "lookahead" for better eviction candidate */
12022918Smaybee 		if (recycle && ab->b_size != bytes &&
12032918Smaybee 		    ab_prev && ab_prev->b_size == bytes)
12042688Smaybee 			continue;
1205789Sahrens 		hash_lock = HDR_LOCK(ab);
12062688Smaybee 		have_lock = MUTEX_HELD(hash_lock);
12072688Smaybee 		if (have_lock || mutex_tryenter(hash_lock)) {
1208789Sahrens 			ASSERT3U(refcount_count(&ab->b_refcnt), ==, 0);
12091544Seschrock 			ASSERT(ab->b_datacnt > 0);
12101544Seschrock 			while (ab->b_buf) {
12111544Seschrock 				arc_buf_t *buf = ab->b_buf;
12122688Smaybee 				if (buf->b_data) {
12131544Seschrock 					bytes_evicted += ab->b_size;
12143290Sjohansen 					if (recycle && ab->b_type == type &&
12153290Sjohansen 					    ab->b_size == bytes) {
12162918Smaybee 						stolen = buf->b_data;
12172918Smaybee 						recycle = FALSE;
12182918Smaybee 					}
12192688Smaybee 				}
12201544Seschrock 				if (buf->b_efunc) {
12211544Seschrock 					mutex_enter(&arc_eviction_mtx);
12222918Smaybee 					arc_buf_destroy(buf,
12232918Smaybee 					    buf->b_data == stolen, FALSE);
12241544Seschrock 					ab->b_buf = buf->b_next;
12252887Smaybee 					buf->b_hdr = &arc_eviction_hdr;
12261544Seschrock 					buf->b_next = arc_eviction_list;
12271544Seschrock 					arc_eviction_list = buf;
12281544Seschrock 					mutex_exit(&arc_eviction_mtx);
12291544Seschrock 				} else {
12302918Smaybee 					arc_buf_destroy(buf,
12312918Smaybee 					    buf->b_data == stolen, TRUE);
12321544Seschrock 				}
12331544Seschrock 			}
12341544Seschrock 			ASSERT(ab->b_datacnt == 0);
1235789Sahrens 			arc_change_state(evicted_state, ab, hash_lock);
12361544Seschrock 			ASSERT(HDR_IN_HASH_TABLE(ab));
12371544Seschrock 			ab->b_flags = ARC_IN_HASH_TABLE;
1238789Sahrens 			DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, ab);
12392688Smaybee 			if (!have_lock)
12402688Smaybee 				mutex_exit(hash_lock);
12411544Seschrock 			if (bytes >= 0 && bytes_evicted >= bytes)
1242789Sahrens 				break;
1243789Sahrens 		} else {
12442688Smaybee 			missed += 1;
1245789Sahrens 		}
1246789Sahrens 	}
12473403Sbmc 
12483403Sbmc 	mutex_exit(&evicted_state->arcs_mtx);
12493403Sbmc 	mutex_exit(&state->arcs_mtx);
1250789Sahrens 
1251789Sahrens 	if (bytes_evicted < bytes)
1252789Sahrens 		dprintf("only evicted %lld bytes from %x",
1253789Sahrens 		    (longlong_t)bytes_evicted, state);
1254789Sahrens 
12552688Smaybee 	if (skipped)
12563403Sbmc 		ARCSTAT_INCR(arcstat_evict_skip, skipped);
12573403Sbmc 
12582688Smaybee 	if (missed)
12593403Sbmc 		ARCSTAT_INCR(arcstat_mutex_miss, missed);
12603403Sbmc 
1261*4709Smaybee 	/*
1262*4709Smaybee 	 * We have just evicted some date into the ghost state, make
1263*4709Smaybee 	 * sure we also adjust the ghost state size if necessary.
1264*4709Smaybee 	 */
1265*4709Smaybee 	if (arc_no_grow &&
1266*4709Smaybee 	    arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size > arc_c) {
1267*4709Smaybee 		int64_t mru_over = arc_anon->arcs_size + arc_mru->arcs_size +
1268*4709Smaybee 		    arc_mru_ghost->arcs_size - arc_c;
1269*4709Smaybee 
1270*4709Smaybee 		if (mru_over > 0 && arc_mru_ghost->arcs_lsize[type] > 0) {
1271*4709Smaybee 			int64_t todelete =
1272*4709Smaybee 			    MIN(arc_mru_ghost->arcs_lsize[type], mru_over);
1273*4709Smaybee 			arc_evict_ghost(arc_mru_ghost, todelete);
1274*4709Smaybee 		} else if (arc_mfu_ghost->arcs_lsize[type] > 0) {
1275*4709Smaybee 			int64_t todelete = MIN(arc_mfu_ghost->arcs_lsize[type],
1276*4709Smaybee 			    arc_mru_ghost->arcs_size +
1277*4709Smaybee 			    arc_mfu_ghost->arcs_size - arc_c);
1278*4709Smaybee 			arc_evict_ghost(arc_mfu_ghost, todelete);
1279*4709Smaybee 		}
1280*4709Smaybee 	}
1281*4709Smaybee 
12822918Smaybee 	return (stolen);
1283789Sahrens }
1284789Sahrens 
1285789Sahrens /*
1286789Sahrens  * Remove buffers from list until we've removed the specified number of
1287789Sahrens  * bytes.  Destroy the buffers that are removed.
1288789Sahrens  */
1289789Sahrens static void
12901544Seschrock arc_evict_ghost(arc_state_t *state, int64_t bytes)
1291789Sahrens {
1292789Sahrens 	arc_buf_hdr_t *ab, *ab_prev;
12934309Smaybee 	list_t *list = &state->arcs_list[ARC_BUFC_DATA];
1294789Sahrens 	kmutex_t *hash_lock;
12951544Seschrock 	uint64_t bytes_deleted = 0;
12963700Sek110237 	uint64_t bufs_skipped = 0;
1297789Sahrens 
12981544Seschrock 	ASSERT(GHOST_STATE(state));
1299789Sahrens top:
13003403Sbmc 	mutex_enter(&state->arcs_mtx);
13014309Smaybee 	for (ab = list_tail(list); ab; ab = ab_prev) {
13024309Smaybee 		ab_prev = list_prev(list, ab);
1303789Sahrens 		hash_lock = HDR_LOCK(ab);
1304789Sahrens 		if (mutex_tryenter(hash_lock)) {
13052391Smaybee 			ASSERT(!HDR_IO_IN_PROGRESS(ab));
13061544Seschrock 			ASSERT(ab->b_buf == NULL);
13073403Sbmc 			arc_change_state(arc_anon, ab, hash_lock);
1308789Sahrens 			mutex_exit(hash_lock);
13093403Sbmc 			ARCSTAT_BUMP(arcstat_deleted);
13101544Seschrock 			bytes_deleted += ab->b_size;
13111544Seschrock 			arc_hdr_destroy(ab);
1312789Sahrens 			DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, ab);
1313789Sahrens 			if (bytes >= 0 && bytes_deleted >= bytes)
1314789Sahrens 				break;
1315789Sahrens 		} else {
1316789Sahrens 			if (bytes < 0) {
13173403Sbmc 				mutex_exit(&state->arcs_mtx);
1318789Sahrens 				mutex_enter(hash_lock);
1319789Sahrens 				mutex_exit(hash_lock);
1320789Sahrens 				goto top;
1321789Sahrens 			}
1322789Sahrens 			bufs_skipped += 1;
1323789Sahrens 		}
1324789Sahrens 	}
13253403Sbmc 	mutex_exit(&state->arcs_mtx);
1326789Sahrens 
13274309Smaybee 	if (list == &state->arcs_list[ARC_BUFC_DATA] &&
13284309Smaybee 	    (bytes < 0 || bytes_deleted < bytes)) {
13294309Smaybee 		list = &state->arcs_list[ARC_BUFC_METADATA];
13304309Smaybee 		goto top;
13314309Smaybee 	}
13324309Smaybee 
1333789Sahrens 	if (bufs_skipped) {
13343403Sbmc 		ARCSTAT_INCR(arcstat_mutex_miss, bufs_skipped);
1335789Sahrens 		ASSERT(bytes >= 0);
1336789Sahrens 	}
1337789Sahrens 
1338789Sahrens 	if (bytes_deleted < bytes)
1339789Sahrens 		dprintf("only deleted %lld bytes from %p",
1340789Sahrens 		    (longlong_t)bytes_deleted, state);
1341789Sahrens }
1342789Sahrens 
1343789Sahrens static void
1344789Sahrens arc_adjust(void)
1345789Sahrens {
13463403Sbmc 	int64_t top_sz, mru_over, arc_over, todelete;
1347789Sahrens 
13483403Sbmc 	top_sz = arc_anon->arcs_size + arc_mru->arcs_size;
1349789Sahrens 
13504309Smaybee 	if (top_sz > arc_p && arc_mru->arcs_lsize[ARC_BUFC_DATA] > 0) {
13514309Smaybee 		int64_t toevict =
13524309Smaybee 		    MIN(arc_mru->arcs_lsize[ARC_BUFC_DATA], top_sz - arc_p);
13534309Smaybee 		(void) arc_evict(arc_mru, toevict, FALSE, ARC_BUFC_DATA);
13544309Smaybee 		top_sz = arc_anon->arcs_size + arc_mru->arcs_size;
13554309Smaybee 	}
13564309Smaybee 
13574309Smaybee 	if (top_sz > arc_p && arc_mru->arcs_lsize[ARC_BUFC_METADATA] > 0) {
13584309Smaybee 		int64_t toevict =
13594309Smaybee 		    MIN(arc_mru->arcs_lsize[ARC_BUFC_METADATA], top_sz - arc_p);
13604309Smaybee 		(void) arc_evict(arc_mru, toevict, FALSE, ARC_BUFC_METADATA);
13613403Sbmc 		top_sz = arc_anon->arcs_size + arc_mru->arcs_size;
1362789Sahrens 	}
1363789Sahrens 
13643403Sbmc 	mru_over = top_sz + arc_mru_ghost->arcs_size - arc_c;
1365789Sahrens 
1366789Sahrens 	if (mru_over > 0) {
13674309Smaybee 		if (arc_mru_ghost->arcs_size > 0) {
13684309Smaybee 			todelete = MIN(arc_mru_ghost->arcs_size, mru_over);
13693403Sbmc 			arc_evict_ghost(arc_mru_ghost, todelete);
1370789Sahrens 		}
1371789Sahrens 	}
1372789Sahrens 
13733403Sbmc 	if ((arc_over = arc_size - arc_c) > 0) {
13741544Seschrock 		int64_t tbl_over;
1375789Sahrens 
13764309Smaybee 		if (arc_mfu->arcs_lsize[ARC_BUFC_DATA] > 0) {
13774309Smaybee 			int64_t toevict =
13784309Smaybee 			    MIN(arc_mfu->arcs_lsize[ARC_BUFC_DATA], arc_over);
13793403Sbmc 			(void) arc_evict(arc_mfu, toevict, FALSE,
13804309Smaybee 			    ARC_BUFC_DATA);
13814309Smaybee 			arc_over = arc_size - arc_c;
1382789Sahrens 		}
1383789Sahrens 
13844309Smaybee 		if (arc_over > 0 &&
13854309Smaybee 		    arc_mfu->arcs_lsize[ARC_BUFC_METADATA] > 0) {
13864309Smaybee 			int64_t toevict =
13874309Smaybee 			    MIN(arc_mfu->arcs_lsize[ARC_BUFC_METADATA],
13884309Smaybee 			    arc_over);
13894309Smaybee 			(void) arc_evict(arc_mfu, toevict, FALSE,
13904309Smaybee 			    ARC_BUFC_METADATA);
13914309Smaybee 		}
13924309Smaybee 
13934309Smaybee 		tbl_over = arc_size + arc_mru_ghost->arcs_size +
13944309Smaybee 		    arc_mfu_ghost->arcs_size - arc_c * 2;
13954309Smaybee 
13964309Smaybee 		if (tbl_over > 0 && arc_mfu_ghost->arcs_size > 0) {
13974309Smaybee 			todelete = MIN(arc_mfu_ghost->arcs_size, tbl_over);
13983403Sbmc 			arc_evict_ghost(arc_mfu_ghost, todelete);
1399789Sahrens 		}
1400789Sahrens 	}
1401789Sahrens }
1402789Sahrens 
14031544Seschrock static void
14041544Seschrock arc_do_user_evicts(void)
14051544Seschrock {
14061544Seschrock 	mutex_enter(&arc_eviction_mtx);
14071544Seschrock 	while (arc_eviction_list != NULL) {
14081544Seschrock 		arc_buf_t *buf = arc_eviction_list;
14091544Seschrock 		arc_eviction_list = buf->b_next;
14101544Seschrock 		buf->b_hdr = NULL;
14111544Seschrock 		mutex_exit(&arc_eviction_mtx);
14121544Seschrock 
14131819Smaybee 		if (buf->b_efunc != NULL)
14141819Smaybee 			VERIFY(buf->b_efunc(buf) == 0);
14151544Seschrock 
14161544Seschrock 		buf->b_efunc = NULL;
14171544Seschrock 		buf->b_private = NULL;
14181544Seschrock 		kmem_cache_free(buf_cache, buf);
14191544Seschrock 		mutex_enter(&arc_eviction_mtx);
14201544Seschrock 	}
14211544Seschrock 	mutex_exit(&arc_eviction_mtx);
14221544Seschrock }
14231544Seschrock 
1424789Sahrens /*
1425789Sahrens  * Flush all *evictable* data from the cache.
1426789Sahrens  * NOTE: this will not touch "active" (i.e. referenced) data.
1427789Sahrens  */
1428789Sahrens void
1429789Sahrens arc_flush(void)
1430789Sahrens {
14314309Smaybee 	while (list_head(&arc_mru->arcs_list[ARC_BUFC_DATA]))
14324309Smaybee 		(void) arc_evict(arc_mru, -1, FALSE, ARC_BUFC_DATA);
14334309Smaybee 	while (list_head(&arc_mru->arcs_list[ARC_BUFC_METADATA]))
14344309Smaybee 		(void) arc_evict(arc_mru, -1, FALSE, ARC_BUFC_METADATA);
14354309Smaybee 	while (list_head(&arc_mfu->arcs_list[ARC_BUFC_DATA]))
14364309Smaybee 		(void) arc_evict(arc_mfu, -1, FALSE, ARC_BUFC_DATA);
14374309Smaybee 	while (list_head(&arc_mfu->arcs_list[ARC_BUFC_METADATA]))
14384309Smaybee 		(void) arc_evict(arc_mfu, -1, FALSE, ARC_BUFC_METADATA);
1439789Sahrens 
14403403Sbmc 	arc_evict_ghost(arc_mru_ghost, -1);
14413403Sbmc 	arc_evict_ghost(arc_mfu_ghost, -1);
14421544Seschrock 
14431544Seschrock 	mutex_enter(&arc_reclaim_thr_lock);
14441544Seschrock 	arc_do_user_evicts();
14451544Seschrock 	mutex_exit(&arc_reclaim_thr_lock);
14461544Seschrock 	ASSERT(arc_eviction_list == NULL);
1447789Sahrens }
1448789Sahrens 
14493158Smaybee int arc_shrink_shift = 5;		/* log2(fraction of arc to reclaim) */
14502391Smaybee 
1451789Sahrens void
14523158Smaybee arc_shrink(void)
1453789Sahrens {
14543403Sbmc 	if (arc_c > arc_c_min) {
14553158Smaybee 		uint64_t to_free;
1456789Sahrens 
14572048Sstans #ifdef _KERNEL
14583403Sbmc 		to_free = MAX(arc_c >> arc_shrink_shift, ptob(needfree));
14592048Sstans #else
14603403Sbmc 		to_free = arc_c >> arc_shrink_shift;
14612048Sstans #endif
14623403Sbmc 		if (arc_c > arc_c_min + to_free)
14633403Sbmc 			atomic_add_64(&arc_c, -to_free);
14643158Smaybee 		else
14653403Sbmc 			arc_c = arc_c_min;
14662048Sstans 
14673403Sbmc 		atomic_add_64(&arc_p, -(arc_p >> arc_shrink_shift));
14683403Sbmc 		if (arc_c > arc_size)
14693403Sbmc 			arc_c = MAX(arc_size, arc_c_min);
14703403Sbmc 		if (arc_p > arc_c)
14713403Sbmc 			arc_p = (arc_c >> 1);
14723403Sbmc 		ASSERT(arc_c >= arc_c_min);
14733403Sbmc 		ASSERT((int64_t)arc_p >= 0);
14743158Smaybee 	}
1475789Sahrens 
14763403Sbmc 	if (arc_size > arc_c)
14773158Smaybee 		arc_adjust();
1478789Sahrens }
1479789Sahrens 
1480789Sahrens static int
1481789Sahrens arc_reclaim_needed(void)
1482789Sahrens {
1483789Sahrens 	uint64_t extra;
1484789Sahrens 
1485789Sahrens #ifdef _KERNEL
14862048Sstans 
14872048Sstans 	if (needfree)
14882048Sstans 		return (1);
14892048Sstans 
1490789Sahrens 	/*
1491789Sahrens 	 * take 'desfree' extra pages, so we reclaim sooner, rather than later
1492789Sahrens 	 */
1493789Sahrens 	extra = desfree;
1494789Sahrens 
1495789Sahrens 	/*
1496789Sahrens 	 * check that we're out of range of the pageout scanner.  It starts to
1497789Sahrens 	 * schedule paging if freemem is less than lotsfree and needfree.
1498789Sahrens 	 * lotsfree is the high-water mark for pageout, and needfree is the
1499789Sahrens 	 * number of needed free pages.  We add extra pages here to make sure
1500789Sahrens 	 * the scanner doesn't start up while we're freeing memory.
1501789Sahrens 	 */
1502789Sahrens 	if (freemem < lotsfree + needfree + extra)
1503789Sahrens 		return (1);
1504789Sahrens 
1505789Sahrens 	/*
1506789Sahrens 	 * check to make sure that swapfs has enough space so that anon
1507789Sahrens 	 * reservations can still succeeed. anon_resvmem() checks that the
1508789Sahrens 	 * availrmem is greater than swapfs_minfree, and the number of reserved
1509789Sahrens 	 * swap pages.  We also add a bit of extra here just to prevent
1510789Sahrens 	 * circumstances from getting really dire.
1511789Sahrens 	 */
1512789Sahrens 	if (availrmem < swapfs_minfree + swapfs_reserve + extra)
1513789Sahrens 		return (1);
1514789Sahrens 
15151936Smaybee #if defined(__i386)
1516789Sahrens 	/*
1517789Sahrens 	 * If we're on an i386 platform, it's possible that we'll exhaust the
1518789Sahrens 	 * kernel heap space before we ever run out of available physical
1519789Sahrens 	 * memory.  Most checks of the size of the heap_area compare against
1520789Sahrens 	 * tune.t_minarmem, which is the minimum available real memory that we
1521789Sahrens 	 * can have in the system.  However, this is generally fixed at 25 pages
1522789Sahrens 	 * which is so low that it's useless.  In this comparison, we seek to
1523789Sahrens 	 * calculate the total heap-size, and reclaim if more than 3/4ths of the
1524789Sahrens 	 * heap is allocated.  (Or, in the caclulation, if less than 1/4th is
1525789Sahrens 	 * free)
1526789Sahrens 	 */
1527789Sahrens 	if (btop(vmem_size(heap_arena, VMEM_FREE)) <
1528789Sahrens 	    (btop(vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC)) >> 2))
1529789Sahrens 		return (1);
1530789Sahrens #endif
1531789Sahrens 
1532789Sahrens #else
1533789Sahrens 	if (spa_get_random(100) == 0)
1534789Sahrens 		return (1);
1535789Sahrens #endif
1536789Sahrens 	return (0);
1537789Sahrens }
1538789Sahrens 
1539789Sahrens static void
1540789Sahrens arc_kmem_reap_now(arc_reclaim_strategy_t strat)
1541789Sahrens {
1542789Sahrens 	size_t			i;
1543789Sahrens 	kmem_cache_t		*prev_cache = NULL;
15443290Sjohansen 	kmem_cache_t		*prev_data_cache = NULL;
1545789Sahrens 	extern kmem_cache_t	*zio_buf_cache[];
15463290Sjohansen 	extern kmem_cache_t	*zio_data_buf_cache[];
1547789Sahrens 
15481484Sek110237 #ifdef _KERNEL
15494309Smaybee 	if (arc_meta_used >= arc_meta_limit) {
15504309Smaybee 		/*
15514309Smaybee 		 * We are exceeding our meta-data cache limit.
15524309Smaybee 		 * Purge some DNLC entries to release holds on meta-data.
15534309Smaybee 		 */
15544309Smaybee 		dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent);
15554309Smaybee 	}
15561936Smaybee #if defined(__i386)
15571936Smaybee 	/*
15581936Smaybee 	 * Reclaim unused memory from all kmem caches.
15591936Smaybee 	 */
15601936Smaybee 	kmem_reap();
15611936Smaybee #endif
15621484Sek110237 #endif
15631484Sek110237 
1564789Sahrens 	/*
15651544Seschrock 	 * An agressive reclamation will shrink the cache size as well as
15661544Seschrock 	 * reap free buffers from the arc kmem caches.
1567789Sahrens 	 */
1568789Sahrens 	if (strat == ARC_RECLAIM_AGGR)
15693158Smaybee 		arc_shrink();
1570789Sahrens 
1571789Sahrens 	for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) {
1572789Sahrens 		if (zio_buf_cache[i] != prev_cache) {
1573789Sahrens 			prev_cache = zio_buf_cache[i];
1574789Sahrens 			kmem_cache_reap_now(zio_buf_cache[i]);
1575789Sahrens 		}
15763290Sjohansen 		if (zio_data_buf_cache[i] != prev_data_cache) {
15773290Sjohansen 			prev_data_cache = zio_data_buf_cache[i];
15783290Sjohansen 			kmem_cache_reap_now(zio_data_buf_cache[i]);
15793290Sjohansen 		}
1580789Sahrens 	}
15811544Seschrock 	kmem_cache_reap_now(buf_cache);
15821544Seschrock 	kmem_cache_reap_now(hdr_cache);
1583789Sahrens }
1584789Sahrens 
1585789Sahrens static void
1586789Sahrens arc_reclaim_thread(void)
1587789Sahrens {
1588789Sahrens 	clock_t			growtime = 0;
1589789Sahrens 	arc_reclaim_strategy_t	last_reclaim = ARC_RECLAIM_CONS;
1590789Sahrens 	callb_cpr_t		cpr;
1591789Sahrens 
1592789Sahrens 	CALLB_CPR_INIT(&cpr, &arc_reclaim_thr_lock, callb_generic_cpr, FTAG);
1593789Sahrens 
1594789Sahrens 	mutex_enter(&arc_reclaim_thr_lock);
1595789Sahrens 	while (arc_thread_exit == 0) {
1596789Sahrens 		if (arc_reclaim_needed()) {
1597789Sahrens 
15983403Sbmc 			if (arc_no_grow) {
1599789Sahrens 				if (last_reclaim == ARC_RECLAIM_CONS) {
1600789Sahrens 					last_reclaim = ARC_RECLAIM_AGGR;
1601789Sahrens 				} else {
1602789Sahrens 					last_reclaim = ARC_RECLAIM_CONS;
1603789Sahrens 				}
1604789Sahrens 			} else {
16053403Sbmc 				arc_no_grow = TRUE;
1606789Sahrens 				last_reclaim = ARC_RECLAIM_AGGR;
1607789Sahrens 				membar_producer();
1608789Sahrens 			}
1609789Sahrens 
1610789Sahrens 			/* reset the growth delay for every reclaim */
1611789Sahrens 			growtime = lbolt + (arc_grow_retry * hz);
1612789Sahrens 
1613789Sahrens 			arc_kmem_reap_now(last_reclaim);
1614789Sahrens 
16154309Smaybee 		} else if (arc_no_grow && lbolt >= growtime) {
16163403Sbmc 			arc_no_grow = FALSE;
1617789Sahrens 		}
1618789Sahrens 
16193403Sbmc 		if (2 * arc_c < arc_size +
16203403Sbmc 		    arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size)
16213298Smaybee 			arc_adjust();
16223298Smaybee 
16231544Seschrock 		if (arc_eviction_list != NULL)
16241544Seschrock 			arc_do_user_evicts();
16251544Seschrock 
1626789Sahrens 		/* block until needed, or one second, whichever is shorter */
1627789Sahrens 		CALLB_CPR_SAFE_BEGIN(&cpr);
1628789Sahrens 		(void) cv_timedwait(&arc_reclaim_thr_cv,
1629789Sahrens 		    &arc_reclaim_thr_lock, (lbolt + hz));
1630789Sahrens 		CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_thr_lock);
1631789Sahrens 	}
1632789Sahrens 
1633789Sahrens 	arc_thread_exit = 0;
1634789Sahrens 	cv_broadcast(&arc_reclaim_thr_cv);
1635789Sahrens 	CALLB_CPR_EXIT(&cpr);		/* drops arc_reclaim_thr_lock */
1636789Sahrens 	thread_exit();
1637789Sahrens }
1638789Sahrens 
16391544Seschrock /*
16401544Seschrock  * Adapt arc info given the number of bytes we are trying to add and
16411544Seschrock  * the state that we are comming from.  This function is only called
16421544Seschrock  * when we are adding new content to the cache.
16431544Seschrock  */
1644789Sahrens static void
16451544Seschrock arc_adapt(int bytes, arc_state_t *state)
1646789Sahrens {
16471544Seschrock 	int mult;
16481544Seschrock 
16491544Seschrock 	ASSERT(bytes > 0);
1650789Sahrens 	/*
16511544Seschrock 	 * Adapt the target size of the MRU list:
16521544Seschrock 	 *	- if we just hit in the MRU ghost list, then increase
16531544Seschrock 	 *	  the target size of the MRU list.
16541544Seschrock 	 *	- if we just hit in the MFU ghost list, then increase
16551544Seschrock 	 *	  the target size of the MFU list by decreasing the
16561544Seschrock 	 *	  target size of the MRU list.
1657789Sahrens 	 */
16583403Sbmc 	if (state == arc_mru_ghost) {
16593403Sbmc 		mult = ((arc_mru_ghost->arcs_size >= arc_mfu_ghost->arcs_size) ?
16603403Sbmc 		    1 : (arc_mfu_ghost->arcs_size/arc_mru_ghost->arcs_size));
16611544Seschrock 
16623403Sbmc 		arc_p = MIN(arc_c, arc_p + bytes * mult);
16633403Sbmc 	} else if (state == arc_mfu_ghost) {
16643403Sbmc 		mult = ((arc_mfu_ghost->arcs_size >= arc_mru_ghost->arcs_size) ?
16653403Sbmc 		    1 : (arc_mru_ghost->arcs_size/arc_mfu_ghost->arcs_size));
16661544Seschrock 
16673403Sbmc 		arc_p = MAX(0, (int64_t)arc_p - bytes * mult);
16681544Seschrock 	}
16693403Sbmc 	ASSERT((int64_t)arc_p >= 0);
1670789Sahrens 
1671789Sahrens 	if (arc_reclaim_needed()) {
1672789Sahrens 		cv_signal(&arc_reclaim_thr_cv);
1673789Sahrens 		return;
1674789Sahrens 	}
1675789Sahrens 
16763403Sbmc 	if (arc_no_grow)
1677789Sahrens 		return;
1678789Sahrens 
16793403Sbmc 	if (arc_c >= arc_c_max)
16801544Seschrock 		return;
16811544Seschrock 
1682789Sahrens 	/*
16831544Seschrock 	 * If we're within (2 * maxblocksize) bytes of the target
16841544Seschrock 	 * cache size, increment the target cache size
1685789Sahrens 	 */
16863403Sbmc 	if (arc_size > arc_c - (2ULL << SPA_MAXBLOCKSHIFT)) {
16873403Sbmc 		atomic_add_64(&arc_c, (int64_t)bytes);
16883403Sbmc 		if (arc_c > arc_c_max)
16893403Sbmc 			arc_c = arc_c_max;
16903403Sbmc 		else if (state == arc_anon)
16913403Sbmc 			atomic_add_64(&arc_p, (int64_t)bytes);
16923403Sbmc 		if (arc_p > arc_c)
16933403Sbmc 			arc_p = arc_c;
1694789Sahrens 	}
16953403Sbmc 	ASSERT((int64_t)arc_p >= 0);
1696789Sahrens }
1697789Sahrens 
1698789Sahrens /*
16991544Seschrock  * Check if the cache has reached its limits and eviction is required
17001544Seschrock  * prior to insert.
1701789Sahrens  */
1702789Sahrens static int
17034309Smaybee arc_evict_needed(arc_buf_contents_t type)
1704789Sahrens {
17054309Smaybee 	if (type == ARC_BUFC_METADATA && arc_meta_used >= arc_meta_limit)
17064309Smaybee 		return (1);
17074309Smaybee 
17084309Smaybee #ifdef _KERNEL
17094309Smaybee 	/*
17104309Smaybee 	 * If zio data pages are being allocated out of a separate heap segment,
17114309Smaybee 	 * then enforce that the size of available vmem for this area remains
17124309Smaybee 	 * above about 1/32nd free.
17134309Smaybee 	 */
17144309Smaybee 	if (type == ARC_BUFC_DATA && zio_arena != NULL &&
17154309Smaybee 	    vmem_size(zio_arena, VMEM_FREE) <
17164309Smaybee 	    (vmem_size(zio_arena, VMEM_ALLOC) >> 5))
17174309Smaybee 		return (1);
17184309Smaybee #endif
17194309Smaybee 
1720789Sahrens 	if (arc_reclaim_needed())
1721789Sahrens 		return (1);
1722789Sahrens 
17233403Sbmc 	return (arc_size > arc_c);
1724789Sahrens }
1725789Sahrens 
1726789Sahrens /*
17272688Smaybee  * The buffer, supplied as the first argument, needs a data block.
17282688Smaybee  * So, if we are at cache max, determine which cache should be victimized.
17292688Smaybee  * We have the following cases:
1730789Sahrens  *
17313403Sbmc  * 1. Insert for MRU, p > sizeof(arc_anon + arc_mru) ->
1732789Sahrens  * In this situation if we're out of space, but the resident size of the MFU is
1733789Sahrens  * under the limit, victimize the MFU cache to satisfy this insertion request.
1734789Sahrens  *
17353403Sbmc  * 2. Insert for MRU, p <= sizeof(arc_anon + arc_mru) ->
1736789Sahrens  * Here, we've used up all of the available space for the MRU, so we need to
1737789Sahrens  * evict from our own cache instead.  Evict from the set of resident MRU
1738789Sahrens  * entries.
1739789Sahrens  *
17403403Sbmc  * 3. Insert for MFU (c - p) > sizeof(arc_mfu) ->
1741789Sahrens  * c minus p represents the MFU space in the cache, since p is the size of the
1742789Sahrens  * cache that is dedicated to the MRU.  In this situation there's still space on
1743789Sahrens  * the MFU side, so the MRU side needs to be victimized.
1744789Sahrens  *
17453403Sbmc  * 4. Insert for MFU (c - p) < sizeof(arc_mfu) ->
1746789Sahrens  * MFU's resident set is consuming more space than it has been allotted.  In
1747789Sahrens  * this situation, we must victimize our own cache, the MFU, for this insertion.
1748789Sahrens  */
1749789Sahrens static void
17502688Smaybee arc_get_data_buf(arc_buf_t *buf)
1751789Sahrens {
17523290Sjohansen 	arc_state_t		*state = buf->b_hdr->b_state;
17533290Sjohansen 	uint64_t		size = buf->b_hdr->b_size;
17543290Sjohansen 	arc_buf_contents_t	type = buf->b_hdr->b_type;
17552688Smaybee 
17562688Smaybee 	arc_adapt(size, state);
1757789Sahrens 
17582688Smaybee 	/*
17592688Smaybee 	 * We have not yet reached cache maximum size,
17602688Smaybee 	 * just allocate a new buffer.
17612688Smaybee 	 */
17624309Smaybee 	if (!arc_evict_needed(type)) {
17633290Sjohansen 		if (type == ARC_BUFC_METADATA) {
17643290Sjohansen 			buf->b_data = zio_buf_alloc(size);
17654309Smaybee 			arc_space_consume(size);
17663290Sjohansen 		} else {
17673290Sjohansen 			ASSERT(type == ARC_BUFC_DATA);
17683290Sjohansen 			buf->b_data = zio_data_buf_alloc(size);
17694309Smaybee 			atomic_add_64(&arc_size, size);
17703290Sjohansen 		}
17712688Smaybee 		goto out;
17722688Smaybee 	}
17732688Smaybee 
17742688Smaybee 	/*
17752688Smaybee 	 * If we are prefetching from the mfu ghost list, this buffer
17762688Smaybee 	 * will end up on the mru list; so steal space from there.
17772688Smaybee 	 */
17783403Sbmc 	if (state == arc_mfu_ghost)
17793403Sbmc 		state = buf->b_hdr->b_flags & ARC_PREFETCH ? arc_mru : arc_mfu;
17803403Sbmc 	else if (state == arc_mru_ghost)
17813403Sbmc 		state = arc_mru;
1782789Sahrens 
17833403Sbmc 	if (state == arc_mru || state == arc_anon) {
17843403Sbmc 		uint64_t mru_used = arc_anon->arcs_size + arc_mru->arcs_size;
17854309Smaybee 		state = (arc_mfu->arcs_lsize[type] > 0 &&
17864309Smaybee 		    arc_p > mru_used) ? arc_mfu : arc_mru;
1787789Sahrens 	} else {
17882688Smaybee 		/* MFU cases */
17893403Sbmc 		uint64_t mfu_space = arc_c - arc_p;
17904309Smaybee 		state =  (arc_mru->arcs_lsize[type] > 0 &&
17914309Smaybee 		    mfu_space > arc_mfu->arcs_size) ? arc_mru : arc_mfu;
17922688Smaybee 	}
17933290Sjohansen 	if ((buf->b_data = arc_evict(state, size, TRUE, type)) == NULL) {
17943290Sjohansen 		if (type == ARC_BUFC_METADATA) {
17953290Sjohansen 			buf->b_data = zio_buf_alloc(size);
17964309Smaybee 			arc_space_consume(size);
17973290Sjohansen 		} else {
17983290Sjohansen 			ASSERT(type == ARC_BUFC_DATA);
17993290Sjohansen 			buf->b_data = zio_data_buf_alloc(size);
18004309Smaybee 			atomic_add_64(&arc_size, size);
18013290Sjohansen 		}
18023403Sbmc 		ARCSTAT_BUMP(arcstat_recycle_miss);
18032688Smaybee 	}
18042688Smaybee 	ASSERT(buf->b_data != NULL);
18052688Smaybee out:
18062688Smaybee 	/*
18072688Smaybee 	 * Update the state size.  Note that ghost states have a
18082688Smaybee 	 * "ghost size" and so don't need to be updated.
18092688Smaybee 	 */
18102688Smaybee 	if (!GHOST_STATE(buf->b_hdr->b_state)) {
18112688Smaybee 		arc_buf_hdr_t *hdr = buf->b_hdr;
18122688Smaybee 
18133403Sbmc 		atomic_add_64(&hdr->b_state->arcs_size, size);
18142688Smaybee 		if (list_link_active(&hdr->b_arc_node)) {
18152688Smaybee 			ASSERT(refcount_is_zero(&hdr->b_refcnt));
18164309Smaybee 			atomic_add_64(&hdr->b_state->arcs_lsize[type], size);
1817789Sahrens 		}
18183298Smaybee 		/*
18193298Smaybee 		 * If we are growing the cache, and we are adding anonymous
18203403Sbmc 		 * data, and we have outgrown arc_p, update arc_p
18213298Smaybee 		 */
18223403Sbmc 		if (arc_size < arc_c && hdr->b_state == arc_anon &&
18233403Sbmc 		    arc_anon->arcs_size + arc_mru->arcs_size > arc_p)
18243403Sbmc 			arc_p = MIN(arc_c, arc_p + size);
1825789Sahrens 	}
1826789Sahrens }
1827789Sahrens 
1828789Sahrens /*
1829789Sahrens  * This routine is called whenever a buffer is accessed.
18301544Seschrock  * NOTE: the hash lock is dropped in this function.
1831789Sahrens  */
1832789Sahrens static void
18332688Smaybee arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock)
1834789Sahrens {
1835789Sahrens 	ASSERT(MUTEX_HELD(hash_lock));
1836789Sahrens 
18373403Sbmc 	if (buf->b_state == arc_anon) {
1838789Sahrens 		/*
1839789Sahrens 		 * This buffer is not in the cache, and does not
1840789Sahrens 		 * appear in our "ghost" list.  Add the new buffer
1841789Sahrens 		 * to the MRU state.
1842789Sahrens 		 */
1843789Sahrens 
1844789Sahrens 		ASSERT(buf->b_arc_access == 0);
1845789Sahrens 		buf->b_arc_access = lbolt;
18461544Seschrock 		DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf);
18473403Sbmc 		arc_change_state(arc_mru, buf, hash_lock);
1848789Sahrens 
18493403Sbmc 	} else if (buf->b_state == arc_mru) {
1850789Sahrens 		/*
18512391Smaybee 		 * If this buffer is here because of a prefetch, then either:
18522391Smaybee 		 * - clear the flag if this is a "referencing" read
18532391Smaybee 		 *   (any subsequent access will bump this into the MFU state).
18542391Smaybee 		 * or
18552391Smaybee 		 * - move the buffer to the head of the list if this is
18562391Smaybee 		 *   another prefetch (to make it less likely to be evicted).
1857789Sahrens 		 */
1858789Sahrens 		if ((buf->b_flags & ARC_PREFETCH) != 0) {
18592391Smaybee 			if (refcount_count(&buf->b_refcnt) == 0) {
18602391Smaybee 				ASSERT(list_link_active(&buf->b_arc_node));
18612391Smaybee 			} else {
18622391Smaybee 				buf->b_flags &= ~ARC_PREFETCH;
18633403Sbmc 				ARCSTAT_BUMP(arcstat_mru_hits);
18642391Smaybee 			}
18652391Smaybee 			buf->b_arc_access = lbolt;
1866789Sahrens 			return;
1867789Sahrens 		}
1868789Sahrens 
1869789Sahrens 		/*
1870789Sahrens 		 * This buffer has been "accessed" only once so far,
1871789Sahrens 		 * but it is still in the cache. Move it to the MFU
1872789Sahrens 		 * state.
1873789Sahrens 		 */
1874789Sahrens 		if (lbolt > buf->b_arc_access + ARC_MINTIME) {
1875789Sahrens 			/*
1876789Sahrens 			 * More than 125ms have passed since we
1877789Sahrens 			 * instantiated this buffer.  Move it to the
1878789Sahrens 			 * most frequently used state.
1879789Sahrens 			 */
1880789Sahrens 			buf->b_arc_access = lbolt;
18811544Seschrock 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
18823403Sbmc 			arc_change_state(arc_mfu, buf, hash_lock);
1883789Sahrens 		}
18843403Sbmc 		ARCSTAT_BUMP(arcstat_mru_hits);
18853403Sbmc 	} else if (buf->b_state == arc_mru_ghost) {
1886789Sahrens 		arc_state_t	*new_state;
1887789Sahrens 		/*
1888789Sahrens 		 * This buffer has been "accessed" recently, but
1889789Sahrens 		 * was evicted from the cache.  Move it to the
1890789Sahrens 		 * MFU state.
1891789Sahrens 		 */
1892789Sahrens 
1893789Sahrens 		if (buf->b_flags & ARC_PREFETCH) {
18943403Sbmc 			new_state = arc_mru;
18952391Smaybee 			if (refcount_count(&buf->b_refcnt) > 0)
18962391Smaybee 				buf->b_flags &= ~ARC_PREFETCH;
18971544Seschrock 			DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf);
1898789Sahrens 		} else {
18993403Sbmc 			new_state = arc_mfu;
19001544Seschrock 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
1901789Sahrens 		}
1902789Sahrens 
1903789Sahrens 		buf->b_arc_access = lbolt;
1904789Sahrens 		arc_change_state(new_state, buf, hash_lock);
1905789Sahrens 
19063403Sbmc 		ARCSTAT_BUMP(arcstat_mru_ghost_hits);
19073403Sbmc 	} else if (buf->b_state == arc_mfu) {
1908789Sahrens 		/*
1909789Sahrens 		 * This buffer has been accessed more than once and is
1910789Sahrens 		 * still in the cache.  Keep it in the MFU state.
1911789Sahrens 		 *
19122391Smaybee 		 * NOTE: an add_reference() that occurred when we did
19132391Smaybee 		 * the arc_read() will have kicked this off the list.
19142391Smaybee 		 * If it was a prefetch, we will explicitly move it to
19152391Smaybee 		 * the head of the list now.
1916789Sahrens 		 */
19172391Smaybee 		if ((buf->b_flags & ARC_PREFETCH) != 0) {
19182391Smaybee 			ASSERT(refcount_count(&buf->b_refcnt) == 0);
19192391Smaybee 			ASSERT(list_link_active(&buf->b_arc_node));
19202391Smaybee 		}
19213403Sbmc 		ARCSTAT_BUMP(arcstat_mfu_hits);
19222391Smaybee 		buf->b_arc_access = lbolt;
19233403Sbmc 	} else if (buf->b_state == arc_mfu_ghost) {
19243403Sbmc 		arc_state_t	*new_state = arc_mfu;
1925789Sahrens 		/*
1926789Sahrens 		 * This buffer has been accessed more than once but has
1927789Sahrens 		 * been evicted from the cache.  Move it back to the
1928789Sahrens 		 * MFU state.
1929789Sahrens 		 */
1930789Sahrens 
19312391Smaybee 		if (buf->b_flags & ARC_PREFETCH) {
19322391Smaybee 			/*
19332391Smaybee 			 * This is a prefetch access...
19342391Smaybee 			 * move this block back to the MRU state.
19352391Smaybee 			 */
19362391Smaybee 			ASSERT3U(refcount_count(&buf->b_refcnt), ==, 0);
19373403Sbmc 			new_state = arc_mru;
19382391Smaybee 		}
19392391Smaybee 
1940789Sahrens 		buf->b_arc_access = lbolt;
19411544Seschrock 		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
19422391Smaybee 		arc_change_state(new_state, buf, hash_lock);
1943789Sahrens 
19443403Sbmc 		ARCSTAT_BUMP(arcstat_mfu_ghost_hits);
1945789Sahrens 	} else {
1946789Sahrens 		ASSERT(!"invalid arc state");
1947789Sahrens 	}
1948789Sahrens }
1949789Sahrens 
1950789Sahrens /* a generic arc_done_func_t which you can use */
1951789Sahrens /* ARGSUSED */
1952789Sahrens void
1953789Sahrens arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg)
1954789Sahrens {
1955789Sahrens 	bcopy(buf->b_data, arg, buf->b_hdr->b_size);
19561544Seschrock 	VERIFY(arc_buf_remove_ref(buf, arg) == 1);
1957789Sahrens }
1958789Sahrens 
19594309Smaybee /* a generic arc_done_func_t */
1960789Sahrens void
1961789Sahrens arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg)
1962789Sahrens {
1963789Sahrens 	arc_buf_t **bufp = arg;
1964789Sahrens 	if (zio && zio->io_error) {
19651544Seschrock 		VERIFY(arc_buf_remove_ref(buf, arg) == 1);
1966789Sahrens 		*bufp = NULL;
1967789Sahrens 	} else {
1968789Sahrens 		*bufp = buf;
1969789Sahrens 	}
1970789Sahrens }
1971789Sahrens 
1972789Sahrens static void
1973789Sahrens arc_read_done(zio_t *zio)
1974789Sahrens {
19751589Smaybee 	arc_buf_hdr_t	*hdr, *found;
1976789Sahrens 	arc_buf_t	*buf;
1977789Sahrens 	arc_buf_t	*abuf;	/* buffer we're assigning to callback */
1978789Sahrens 	kmutex_t	*hash_lock;
1979789Sahrens 	arc_callback_t	*callback_list, *acb;
1980789Sahrens 	int		freeable = FALSE;
1981789Sahrens 
1982789Sahrens 	buf = zio->io_private;
1983789Sahrens 	hdr = buf->b_hdr;
1984789Sahrens 
19851589Smaybee 	/*
19861589Smaybee 	 * The hdr was inserted into hash-table and removed from lists
19871589Smaybee 	 * prior to starting I/O.  We should find this header, since
19881589Smaybee 	 * it's in the hash table, and it should be legit since it's
19891589Smaybee 	 * not possible to evict it during the I/O.  The only possible
19901589Smaybee 	 * reason for it not to be found is if we were freed during the
19911589Smaybee 	 * read.
19921589Smaybee 	 */
19931589Smaybee 	found = buf_hash_find(zio->io_spa, &hdr->b_dva, hdr->b_birth,
19943093Sahrens 	    &hash_lock);
1995789Sahrens 
19961589Smaybee 	ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && hash_lock == NULL) ||
19971589Smaybee 	    (found == hdr && DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))));
1998789Sahrens 
1999789Sahrens 	/* byteswap if necessary */
2000789Sahrens 	callback_list = hdr->b_acb;
2001789Sahrens 	ASSERT(callback_list != NULL);
2002789Sahrens 	if (BP_SHOULD_BYTESWAP(zio->io_bp) && callback_list->acb_byteswap)
2003789Sahrens 		callback_list->acb_byteswap(buf->b_data, hdr->b_size);
2004789Sahrens 
20053093Sahrens 	arc_cksum_compute(buf);
20063093Sahrens 
2007789Sahrens 	/* create copies of the data buffer for the callers */
2008789Sahrens 	abuf = buf;
2009789Sahrens 	for (acb = callback_list; acb; acb = acb->acb_next) {
2010789Sahrens 		if (acb->acb_done) {
20112688Smaybee 			if (abuf == NULL)
20122688Smaybee 				abuf = arc_buf_clone(buf);
2013789Sahrens 			acb->acb_buf = abuf;
2014789Sahrens 			abuf = NULL;
2015789Sahrens 		}
2016789Sahrens 	}
2017789Sahrens 	hdr->b_acb = NULL;
2018789Sahrens 	hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
20191544Seschrock 	ASSERT(!HDR_BUF_AVAILABLE(hdr));
20201544Seschrock 	if (abuf == buf)
20211544Seschrock 		hdr->b_flags |= ARC_BUF_AVAILABLE;
2022789Sahrens 
2023789Sahrens 	ASSERT(refcount_is_zero(&hdr->b_refcnt) || callback_list != NULL);
2024789Sahrens 
2025789Sahrens 	if (zio->io_error != 0) {
2026789Sahrens 		hdr->b_flags |= ARC_IO_ERROR;
20273403Sbmc 		if (hdr->b_state != arc_anon)
20283403Sbmc 			arc_change_state(arc_anon, hdr, hash_lock);
20291544Seschrock 		if (HDR_IN_HASH_TABLE(hdr))
20301544Seschrock 			buf_hash_remove(hdr);
2031789Sahrens 		freeable = refcount_is_zero(&hdr->b_refcnt);
20322391Smaybee 		/* convert checksum errors into IO errors */
20331544Seschrock 		if (zio->io_error == ECKSUM)
20341544Seschrock 			zio->io_error = EIO;
2035789Sahrens 	}
2036789Sahrens 
20371544Seschrock 	/*
20382391Smaybee 	 * Broadcast before we drop the hash_lock to avoid the possibility
20392391Smaybee 	 * that the hdr (and hence the cv) might be freed before we get to
20402391Smaybee 	 * the cv_broadcast().
20411544Seschrock 	 */
20421544Seschrock 	cv_broadcast(&hdr->b_cv);
20431544Seschrock 
20441589Smaybee 	if (hash_lock) {
2045789Sahrens 		/*
2046789Sahrens 		 * Only call arc_access on anonymous buffers.  This is because
2047789Sahrens 		 * if we've issued an I/O for an evicted buffer, we've already
2048789Sahrens 		 * called arc_access (to prevent any simultaneous readers from
2049789Sahrens 		 * getting confused).
2050789Sahrens 		 */
20513403Sbmc 		if (zio->io_error == 0 && hdr->b_state == arc_anon)
20522688Smaybee 			arc_access(hdr, hash_lock);
20532688Smaybee 		mutex_exit(hash_lock);
2054789Sahrens 	} else {
2055789Sahrens 		/*
2056789Sahrens 		 * This block was freed while we waited for the read to
2057789Sahrens 		 * complete.  It has been removed from the hash table and
2058789Sahrens 		 * moved to the anonymous state (so that it won't show up
2059789Sahrens 		 * in the cache).
2060789Sahrens 		 */
20613403Sbmc 		ASSERT3P(hdr->b_state, ==, arc_anon);
2062789Sahrens 		freeable = refcount_is_zero(&hdr->b_refcnt);
2063789Sahrens 	}
2064789Sahrens 
2065789Sahrens 	/* execute each callback and free its structure */
2066789Sahrens 	while ((acb = callback_list) != NULL) {
2067789Sahrens 		if (acb->acb_done)
2068789Sahrens 			acb->acb_done(zio, acb->acb_buf, acb->acb_private);
2069789Sahrens 
2070789Sahrens 		if (acb->acb_zio_dummy != NULL) {
2071789Sahrens 			acb->acb_zio_dummy->io_error = zio->io_error;
2072789Sahrens 			zio_nowait(acb->acb_zio_dummy);
2073789Sahrens 		}
2074789Sahrens 
2075789Sahrens 		callback_list = acb->acb_next;
2076789Sahrens 		kmem_free(acb, sizeof (arc_callback_t));
2077789Sahrens 	}
2078789Sahrens 
2079789Sahrens 	if (freeable)
20801544Seschrock 		arc_hdr_destroy(hdr);
2081789Sahrens }
2082789Sahrens 
2083789Sahrens /*
2084789Sahrens  * "Read" the block block at the specified DVA (in bp) via the
2085789Sahrens  * cache.  If the block is found in the cache, invoke the provided
2086789Sahrens  * callback immediately and return.  Note that the `zio' parameter
2087789Sahrens  * in the callback will be NULL in this case, since no IO was
2088789Sahrens  * required.  If the block is not in the cache pass the read request
2089789Sahrens  * on to the spa with a substitute callback function, so that the
2090789Sahrens  * requested block will be added to the cache.
2091789Sahrens  *
2092789Sahrens  * If a read request arrives for a block that has a read in-progress,
2093789Sahrens  * either wait for the in-progress read to complete (and return the
2094789Sahrens  * results); or, if this is a read with a "done" func, add a record
2095789Sahrens  * to the read to invoke the "done" func when the read completes,
2096789Sahrens  * and return; or just return.
2097789Sahrens  *
2098789Sahrens  * arc_read_done() will invoke all the requested "done" functions
2099789Sahrens  * for readers of this block.
2100789Sahrens  */
2101789Sahrens int
2102789Sahrens arc_read(zio_t *pio, spa_t *spa, blkptr_t *bp, arc_byteswap_func_t *swap,
2103789Sahrens     arc_done_func_t *done, void *private, int priority, int flags,
21042391Smaybee     uint32_t *arc_flags, zbookmark_t *zb)
2105789Sahrens {
2106789Sahrens 	arc_buf_hdr_t *hdr;
2107789Sahrens 	arc_buf_t *buf;
2108789Sahrens 	kmutex_t *hash_lock;
2109789Sahrens 	zio_t	*rzio;
2110789Sahrens 
2111789Sahrens top:
2112789Sahrens 	hdr = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_lock);
21131544Seschrock 	if (hdr && hdr->b_datacnt > 0) {
2114789Sahrens 
21152391Smaybee 		*arc_flags |= ARC_CACHED;
21162391Smaybee 
2117789Sahrens 		if (HDR_IO_IN_PROGRESS(hdr)) {
21182391Smaybee 
21192391Smaybee 			if (*arc_flags & ARC_WAIT) {
21202391Smaybee 				cv_wait(&hdr->b_cv, hash_lock);
21212391Smaybee 				mutex_exit(hash_lock);
21222391Smaybee 				goto top;
21232391Smaybee 			}
21242391Smaybee 			ASSERT(*arc_flags & ARC_NOWAIT);
21252391Smaybee 
21262391Smaybee 			if (done) {
2127789Sahrens 				arc_callback_t	*acb = NULL;
2128789Sahrens 
2129789Sahrens 				acb = kmem_zalloc(sizeof (arc_callback_t),
2130789Sahrens 				    KM_SLEEP);
2131789Sahrens 				acb->acb_done = done;
2132789Sahrens 				acb->acb_private = private;
2133789Sahrens 				acb->acb_byteswap = swap;
2134789Sahrens 				if (pio != NULL)
2135789Sahrens 					acb->acb_zio_dummy = zio_null(pio,
2136789Sahrens 					    spa, NULL, NULL, flags);
2137789Sahrens 
2138789Sahrens 				ASSERT(acb->acb_done != NULL);
2139789Sahrens 				acb->acb_next = hdr->b_acb;
2140789Sahrens 				hdr->b_acb = acb;
2141789Sahrens 				add_reference(hdr, hash_lock, private);
2142789Sahrens 				mutex_exit(hash_lock);
2143789Sahrens 				return (0);
2144789Sahrens 			}
2145789Sahrens 			mutex_exit(hash_lock);
2146789Sahrens 			return (0);
2147789Sahrens 		}
2148789Sahrens 
21493403Sbmc 		ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
2150789Sahrens 
21511544Seschrock 		if (done) {
21522688Smaybee 			add_reference(hdr, hash_lock, private);
21531544Seschrock 			/*
21541544Seschrock 			 * If this block is already in use, create a new
21551544Seschrock 			 * copy of the data so that we will be guaranteed
21561544Seschrock 			 * that arc_release() will always succeed.
21571544Seschrock 			 */
21581544Seschrock 			buf = hdr->b_buf;
21591544Seschrock 			ASSERT(buf);
21601544Seschrock 			ASSERT(buf->b_data);
21612688Smaybee 			if (HDR_BUF_AVAILABLE(hdr)) {
21621544Seschrock 				ASSERT(buf->b_efunc == NULL);
21631544Seschrock 				hdr->b_flags &= ~ARC_BUF_AVAILABLE;
21642688Smaybee 			} else {
21652688Smaybee 				buf = arc_buf_clone(buf);
21661544Seschrock 			}
21672391Smaybee 		} else if (*arc_flags & ARC_PREFETCH &&
21682391Smaybee 		    refcount_count(&hdr->b_refcnt) == 0) {
21692391Smaybee 			hdr->b_flags |= ARC_PREFETCH;
2170789Sahrens 		}
2171789Sahrens 		DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
21722688Smaybee 		arc_access(hdr, hash_lock);
21732688Smaybee 		mutex_exit(hash_lock);
21743403Sbmc 		ARCSTAT_BUMP(arcstat_hits);
21753403Sbmc 		ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
21763403Sbmc 		    demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
21773403Sbmc 		    data, metadata, hits);
21783403Sbmc 
2179789Sahrens 		if (done)
2180789Sahrens 			done(NULL, buf, private);
2181789Sahrens 	} else {
2182789Sahrens 		uint64_t size = BP_GET_LSIZE(bp);
2183789Sahrens 		arc_callback_t	*acb;
2184789Sahrens 
2185789Sahrens 		if (hdr == NULL) {
2186789Sahrens 			/* this block is not in the cache */
2187789Sahrens 			arc_buf_hdr_t	*exists;
21883290Sjohansen 			arc_buf_contents_t type = BP_GET_BUFC_TYPE(bp);
21893290Sjohansen 			buf = arc_buf_alloc(spa, size, private, type);
2190789Sahrens 			hdr = buf->b_hdr;
2191789Sahrens 			hdr->b_dva = *BP_IDENTITY(bp);
2192789Sahrens 			hdr->b_birth = bp->blk_birth;
2193789Sahrens 			hdr->b_cksum0 = bp->blk_cksum.zc_word[0];
2194789Sahrens 			exists = buf_hash_insert(hdr, &hash_lock);
2195789Sahrens 			if (exists) {
2196789Sahrens 				/* somebody beat us to the hash insert */
2197789Sahrens 				mutex_exit(hash_lock);
2198789Sahrens 				bzero(&hdr->b_dva, sizeof (dva_t));
2199789Sahrens 				hdr->b_birth = 0;
2200789Sahrens 				hdr->b_cksum0 = 0;
22011544Seschrock 				(void) arc_buf_remove_ref(buf, private);
2202789Sahrens 				goto top; /* restart the IO request */
2203789Sahrens 			}
22042391Smaybee 			/* if this is a prefetch, we don't have a reference */
22052391Smaybee 			if (*arc_flags & ARC_PREFETCH) {
22062391Smaybee 				(void) remove_reference(hdr, hash_lock,
22072391Smaybee 				    private);
22082391Smaybee 				hdr->b_flags |= ARC_PREFETCH;
22092391Smaybee 			}
22102391Smaybee 			if (BP_GET_LEVEL(bp) > 0)
22112391Smaybee 				hdr->b_flags |= ARC_INDIRECT;
2212789Sahrens 		} else {
2213789Sahrens 			/* this block is in the ghost cache */
22141544Seschrock 			ASSERT(GHOST_STATE(hdr->b_state));
22151544Seschrock 			ASSERT(!HDR_IO_IN_PROGRESS(hdr));
22162391Smaybee 			ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 0);
22172391Smaybee 			ASSERT(hdr->b_buf == NULL);
2218789Sahrens 
22192391Smaybee 			/* if this is a prefetch, we don't have a reference */
22202391Smaybee 			if (*arc_flags & ARC_PREFETCH)
22212391Smaybee 				hdr->b_flags |= ARC_PREFETCH;
22222391Smaybee 			else
22232391Smaybee 				add_reference(hdr, hash_lock, private);
2224789Sahrens 			buf = kmem_cache_alloc(buf_cache, KM_SLEEP);
22251544Seschrock 			buf->b_hdr = hdr;
22262688Smaybee 			buf->b_data = NULL;
22271544Seschrock 			buf->b_efunc = NULL;
22281544Seschrock 			buf->b_private = NULL;
22291544Seschrock 			buf->b_next = NULL;
22301544Seschrock 			hdr->b_buf = buf;
22312688Smaybee 			arc_get_data_buf(buf);
22321544Seschrock 			ASSERT(hdr->b_datacnt == 0);
22331544Seschrock 			hdr->b_datacnt = 1;
22342391Smaybee 
2235789Sahrens 		}
2236789Sahrens 
2237789Sahrens 		acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
2238789Sahrens 		acb->acb_done = done;
2239789Sahrens 		acb->acb_private = private;
2240789Sahrens 		acb->acb_byteswap = swap;
2241789Sahrens 
2242789Sahrens 		ASSERT(hdr->b_acb == NULL);
2243789Sahrens 		hdr->b_acb = acb;
2244789Sahrens 		hdr->b_flags |= ARC_IO_IN_PROGRESS;
2245789Sahrens 
2246789Sahrens 		/*
2247789Sahrens 		 * If the buffer has been evicted, migrate it to a present state
2248789Sahrens 		 * before issuing the I/O.  Once we drop the hash-table lock,
2249789Sahrens 		 * the header will be marked as I/O in progress and have an
2250789Sahrens 		 * attached buffer.  At this point, anybody who finds this
2251789Sahrens 		 * buffer ought to notice that it's legit but has a pending I/O.
2252789Sahrens 		 */
2253789Sahrens 
22541544Seschrock 		if (GHOST_STATE(hdr->b_state))
22552688Smaybee 			arc_access(hdr, hash_lock);
22562688Smaybee 		mutex_exit(hash_lock);
2257789Sahrens 
2258789Sahrens 		ASSERT3U(hdr->b_size, ==, size);
22591596Sahrens 		DTRACE_PROBE3(arc__miss, blkptr_t *, bp, uint64_t, size,
22601596Sahrens 		    zbookmark_t *, zb);
22613403Sbmc 		ARCSTAT_BUMP(arcstat_misses);
22623403Sbmc 		ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_PREFETCH),
22633403Sbmc 		    demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
22643403Sbmc 		    data, metadata, misses);
22651544Seschrock 
2266789Sahrens 		rzio = zio_read(pio, spa, bp, buf->b_data, size,
22671544Seschrock 		    arc_read_done, buf, priority, flags, zb);
2268789Sahrens 
22692391Smaybee 		if (*arc_flags & ARC_WAIT)
2270789Sahrens 			return (zio_wait(rzio));
2271789Sahrens 
22722391Smaybee 		ASSERT(*arc_flags & ARC_NOWAIT);
2273789Sahrens 		zio_nowait(rzio);
2274789Sahrens 	}
2275789Sahrens 	return (0);
2276789Sahrens }
2277789Sahrens 
2278789Sahrens /*
2279789Sahrens  * arc_read() variant to support pool traversal.  If the block is already
2280789Sahrens  * in the ARC, make a copy of it; otherwise, the caller will do the I/O.
2281789Sahrens  * The idea is that we don't want pool traversal filling up memory, but
2282789Sahrens  * if the ARC already has the data anyway, we shouldn't pay for the I/O.
2283789Sahrens  */
2284789Sahrens int
2285789Sahrens arc_tryread(spa_t *spa, blkptr_t *bp, void *data)
2286789Sahrens {
2287789Sahrens 	arc_buf_hdr_t *hdr;
2288789Sahrens 	kmutex_t *hash_mtx;
2289789Sahrens 	int rc = 0;
2290789Sahrens 
2291789Sahrens 	hdr = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_mtx);
2292789Sahrens 
22931544Seschrock 	if (hdr && hdr->b_datacnt > 0 && !HDR_IO_IN_PROGRESS(hdr)) {
22941544Seschrock 		arc_buf_t *buf = hdr->b_buf;
22951544Seschrock 
22961544Seschrock 		ASSERT(buf);
22971544Seschrock 		while (buf->b_data == NULL) {
22981544Seschrock 			buf = buf->b_next;
22991544Seschrock 			ASSERT(buf);
23001544Seschrock 		}
23011544Seschrock 		bcopy(buf->b_data, data, hdr->b_size);
23021544Seschrock 	} else {
2303789Sahrens 		rc = ENOENT;
23041544Seschrock 	}
2305789Sahrens 
2306789Sahrens 	if (hash_mtx)
2307789Sahrens 		mutex_exit(hash_mtx);
2308789Sahrens 
2309789Sahrens 	return (rc);
2310789Sahrens }
2311789Sahrens 
23121544Seschrock void
23131544Seschrock arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private)
23141544Seschrock {
23151544Seschrock 	ASSERT(buf->b_hdr != NULL);
23163403Sbmc 	ASSERT(buf->b_hdr->b_state != arc_anon);
23171544Seschrock 	ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt) || func == NULL);
23181544Seschrock 	buf->b_efunc = func;
23191544Seschrock 	buf->b_private = private;
23201544Seschrock }
23211544Seschrock 
23221544Seschrock /*
23231544Seschrock  * This is used by the DMU to let the ARC know that a buffer is
23241544Seschrock  * being evicted, so the ARC should clean up.  If this arc buf
23251544Seschrock  * is not yet in the evicted state, it will be put there.
23261544Seschrock  */
23271544Seschrock int
23281544Seschrock arc_buf_evict(arc_buf_t *buf)
23291544Seschrock {
23302887Smaybee 	arc_buf_hdr_t *hdr;
23311544Seschrock 	kmutex_t *hash_lock;
23321544Seschrock 	arc_buf_t **bufp;
23331544Seschrock 
23342887Smaybee 	mutex_enter(&arc_eviction_mtx);
23352887Smaybee 	hdr = buf->b_hdr;
23361544Seschrock 	if (hdr == NULL) {
23371544Seschrock 		/*
23381544Seschrock 		 * We are in arc_do_user_evicts().
23391544Seschrock 		 */
23401544Seschrock 		ASSERT(buf->b_data == NULL);
23412887Smaybee 		mutex_exit(&arc_eviction_mtx);
23421544Seschrock 		return (0);
23431544Seschrock 	}
23442887Smaybee 	hash_lock = HDR_LOCK(hdr);
23452887Smaybee 	mutex_exit(&arc_eviction_mtx);
23461544Seschrock 
23471544Seschrock 	mutex_enter(hash_lock);
23481544Seschrock 
23492724Smaybee 	if (buf->b_data == NULL) {
23502724Smaybee 		/*
23512724Smaybee 		 * We are on the eviction list.
23522724Smaybee 		 */
23532724Smaybee 		mutex_exit(hash_lock);
23542724Smaybee 		mutex_enter(&arc_eviction_mtx);
23552724Smaybee 		if (buf->b_hdr == NULL) {
23562724Smaybee 			/*
23572724Smaybee 			 * We are already in arc_do_user_evicts().
23582724Smaybee 			 */
23592724Smaybee 			mutex_exit(&arc_eviction_mtx);
23602724Smaybee 			return (0);
23612724Smaybee 		} else {
23622724Smaybee 			arc_buf_t copy = *buf; /* structure assignment */
23632724Smaybee 			/*
23642724Smaybee 			 * Process this buffer now
23652724Smaybee 			 * but let arc_do_user_evicts() do the reaping.
23662724Smaybee 			 */
23672724Smaybee 			buf->b_efunc = NULL;
23682724Smaybee 			mutex_exit(&arc_eviction_mtx);
23692724Smaybee 			VERIFY(copy.b_efunc(&copy) == 0);
23702724Smaybee 			return (1);
23712724Smaybee 		}
23722724Smaybee 	}
23732724Smaybee 
23742724Smaybee 	ASSERT(buf->b_hdr == hdr);
23752724Smaybee 	ASSERT3U(refcount_count(&hdr->b_refcnt), <, hdr->b_datacnt);
23763403Sbmc 	ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
23771544Seschrock 
23781544Seschrock 	/*
23791544Seschrock 	 * Pull this buffer off of the hdr
23801544Seschrock 	 */
23811544Seschrock 	bufp = &hdr->b_buf;
23821544Seschrock 	while (*bufp != buf)
23831544Seschrock 		bufp = &(*bufp)->b_next;
23841544Seschrock 	*bufp = buf->b_next;
23851544Seschrock 
23861544Seschrock 	ASSERT(buf->b_data != NULL);
23872688Smaybee 	arc_buf_destroy(buf, FALSE, FALSE);
23881544Seschrock 
23891544Seschrock 	if (hdr->b_datacnt == 0) {
23901544Seschrock 		arc_state_t *old_state = hdr->b_state;
23911544Seschrock 		arc_state_t *evicted_state;
23921544Seschrock 
23931544Seschrock 		ASSERT(refcount_is_zero(&hdr->b_refcnt));
23941544Seschrock 
23951544Seschrock 		evicted_state =
23963403Sbmc 		    (old_state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
23971544Seschrock 
23983403Sbmc 		mutex_enter(&old_state->arcs_mtx);
23993403Sbmc 		mutex_enter(&evicted_state->arcs_mtx);
24001544Seschrock 
24011544Seschrock 		arc_change_state(evicted_state, hdr, hash_lock);
24021544Seschrock 		ASSERT(HDR_IN_HASH_TABLE(hdr));
24031544Seschrock 		hdr->b_flags = ARC_IN_HASH_TABLE;
24041544Seschrock 
24053403Sbmc 		mutex_exit(&evicted_state->arcs_mtx);
24063403Sbmc 		mutex_exit(&old_state->arcs_mtx);
24071544Seschrock 	}
24081544Seschrock 	mutex_exit(hash_lock);
24091819Smaybee 
24101544Seschrock 	VERIFY(buf->b_efunc(buf) == 0);
24111544Seschrock 	buf->b_efunc = NULL;
24121544Seschrock 	buf->b_private = NULL;
24131544Seschrock 	buf->b_hdr = NULL;
24141544Seschrock 	kmem_cache_free(buf_cache, buf);
24151544Seschrock 	return (1);
24161544Seschrock }
24171544Seschrock 
2418789Sahrens /*
2419789Sahrens  * Release this buffer from the cache.  This must be done
2420789Sahrens  * after a read and prior to modifying the buffer contents.
2421789Sahrens  * If the buffer has more than one reference, we must make
2422789Sahrens  * make a new hdr for the buffer.
2423789Sahrens  */
2424789Sahrens void
2425789Sahrens arc_release(arc_buf_t *buf, void *tag)
2426789Sahrens {
2427789Sahrens 	arc_buf_hdr_t *hdr = buf->b_hdr;
2428789Sahrens 	kmutex_t *hash_lock = HDR_LOCK(hdr);
2429789Sahrens 
2430789Sahrens 	/* this buffer is not on any list */
2431789Sahrens 	ASSERT(refcount_count(&hdr->b_refcnt) > 0);
2432789Sahrens 
24333403Sbmc 	if (hdr->b_state == arc_anon) {
2434789Sahrens 		/* this buffer is already released */
2435789Sahrens 		ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 1);
2436789Sahrens 		ASSERT(BUF_EMPTY(hdr));
24371544Seschrock 		ASSERT(buf->b_efunc == NULL);
24383093Sahrens 		arc_buf_thaw(buf);
2439789Sahrens 		return;
2440789Sahrens 	}
2441789Sahrens 
2442789Sahrens 	mutex_enter(hash_lock);
2443789Sahrens 
24441544Seschrock 	/*
24451544Seschrock 	 * Do we have more than one buf?
24461544Seschrock 	 */
24471544Seschrock 	if (hdr->b_buf != buf || buf->b_next != NULL) {
2448789Sahrens 		arc_buf_hdr_t *nhdr;
2449789Sahrens 		arc_buf_t **bufp;
2450789Sahrens 		uint64_t blksz = hdr->b_size;
2451789Sahrens 		spa_t *spa = hdr->b_spa;
24523290Sjohansen 		arc_buf_contents_t type = hdr->b_type;
2453789Sahrens 
24541544Seschrock 		ASSERT(hdr->b_datacnt > 1);
2455789Sahrens 		/*
2456789Sahrens 		 * Pull the data off of this buf and attach it to
2457789Sahrens 		 * a new anonymous buf.
2458789Sahrens 		 */
24591544Seschrock 		(void) remove_reference(hdr, hash_lock, tag);
2460789Sahrens 		bufp = &hdr->b_buf;
24611544Seschrock 		while (*bufp != buf)
2462789Sahrens 			bufp = &(*bufp)->b_next;
2463789Sahrens 		*bufp = (*bufp)->b_next;
24643897Smaybee 		buf->b_next = NULL;
24651544Seschrock 
24663403Sbmc 		ASSERT3U(hdr->b_state->arcs_size, >=, hdr->b_size);
24673403Sbmc 		atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size);
24681544Seschrock 		if (refcount_is_zero(&hdr->b_refcnt)) {
24694309Smaybee 			uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type];
24704309Smaybee 			ASSERT3U(*size, >=, hdr->b_size);
24714309Smaybee 			atomic_add_64(size, -hdr->b_size);
24721544Seschrock 		}
24731544Seschrock 		hdr->b_datacnt -= 1;
24743547Smaybee 		arc_cksum_verify(buf);
24751544Seschrock 
2476789Sahrens 		mutex_exit(hash_lock);
2477789Sahrens 
2478789Sahrens 		nhdr = kmem_cache_alloc(hdr_cache, KM_SLEEP);
2479789Sahrens 		nhdr->b_size = blksz;
2480789Sahrens 		nhdr->b_spa = spa;
24813290Sjohansen 		nhdr->b_type = type;
2482789Sahrens 		nhdr->b_buf = buf;
24833403Sbmc 		nhdr->b_state = arc_anon;
2484789Sahrens 		nhdr->b_arc_access = 0;
2485789Sahrens 		nhdr->b_flags = 0;
24861544Seschrock 		nhdr->b_datacnt = 1;
24873547Smaybee 		nhdr->b_freeze_cksum = NULL;
24883897Smaybee 		(void) refcount_add(&nhdr->b_refcnt, tag);
2489789Sahrens 		buf->b_hdr = nhdr;
24903403Sbmc 		atomic_add_64(&arc_anon->arcs_size, blksz);
2491789Sahrens 
2492789Sahrens 		hdr = nhdr;
2493789Sahrens 	} else {
24941544Seschrock 		ASSERT(refcount_count(&hdr->b_refcnt) == 1);
2495789Sahrens 		ASSERT(!list_link_active(&hdr->b_arc_node));
2496789Sahrens 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
24973403Sbmc 		arc_change_state(arc_anon, hdr, hash_lock);
2498789Sahrens 		hdr->b_arc_access = 0;
2499789Sahrens 		mutex_exit(hash_lock);
2500789Sahrens 		bzero(&hdr->b_dva, sizeof (dva_t));
2501789Sahrens 		hdr->b_birth = 0;
2502789Sahrens 		hdr->b_cksum0 = 0;
25033547Smaybee 		arc_buf_thaw(buf);
2504789Sahrens 	}
25051544Seschrock 	buf->b_efunc = NULL;
25061544Seschrock 	buf->b_private = NULL;
2507789Sahrens }
2508789Sahrens 
2509789Sahrens int
2510789Sahrens arc_released(arc_buf_t *buf)
2511789Sahrens {
25123403Sbmc 	return (buf->b_data != NULL && buf->b_hdr->b_state == arc_anon);
25131544Seschrock }
25141544Seschrock 
25151544Seschrock int
25161544Seschrock arc_has_callback(arc_buf_t *buf)
25171544Seschrock {
25181544Seschrock 	return (buf->b_efunc != NULL);
2519789Sahrens }
2520789Sahrens 
25211544Seschrock #ifdef ZFS_DEBUG
25221544Seschrock int
25231544Seschrock arc_referenced(arc_buf_t *buf)
25241544Seschrock {
25251544Seschrock 	return (refcount_count(&buf->b_hdr->b_refcnt));
25261544Seschrock }
25271544Seschrock #endif
25281544Seschrock 
2529789Sahrens static void
25303547Smaybee arc_write_ready(zio_t *zio)
25313547Smaybee {
25323547Smaybee 	arc_write_callback_t *callback = zio->io_private;
25333547Smaybee 	arc_buf_t *buf = callback->awcb_buf;
25343547Smaybee 
25353547Smaybee 	if (callback->awcb_ready) {
25363547Smaybee 		ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt));
25373547Smaybee 		callback->awcb_ready(zio, buf, callback->awcb_private);
25383547Smaybee 	}
25393547Smaybee 	arc_cksum_compute(buf);
25403547Smaybee }
25413547Smaybee 
25423547Smaybee static void
2543789Sahrens arc_write_done(zio_t *zio)
2544789Sahrens {
25453547Smaybee 	arc_write_callback_t *callback = zio->io_private;
25463547Smaybee 	arc_buf_t *buf = callback->awcb_buf;
25473547Smaybee 	arc_buf_hdr_t *hdr = buf->b_hdr;
2548789Sahrens 
2549789Sahrens 	hdr->b_acb = NULL;
2550789Sahrens 
2551789Sahrens 	/* this buffer is on no lists and is not in the hash table */
25523403Sbmc 	ASSERT3P(hdr->b_state, ==, arc_anon);
2553789Sahrens 
2554789Sahrens 	hdr->b_dva = *BP_IDENTITY(zio->io_bp);
2555789Sahrens 	hdr->b_birth = zio->io_bp->blk_birth;
2556789Sahrens 	hdr->b_cksum0 = zio->io_bp->blk_cksum.zc_word[0];
25571544Seschrock 	/*
25581544Seschrock 	 * If the block to be written was all-zero, we may have
25591544Seschrock 	 * compressed it away.  In this case no write was performed
25601544Seschrock 	 * so there will be no dva/birth-date/checksum.  The buffer
25611544Seschrock 	 * must therefor remain anonymous (and uncached).
25621544Seschrock 	 */
2563789Sahrens 	if (!BUF_EMPTY(hdr)) {
2564789Sahrens 		arc_buf_hdr_t *exists;
2565789Sahrens 		kmutex_t *hash_lock;
2566789Sahrens 
25673093Sahrens 		arc_cksum_verify(buf);
25683093Sahrens 
2569789Sahrens 		exists = buf_hash_insert(hdr, &hash_lock);
2570789Sahrens 		if (exists) {
2571789Sahrens 			/*
2572789Sahrens 			 * This can only happen if we overwrite for
2573789Sahrens 			 * sync-to-convergence, because we remove
2574789Sahrens 			 * buffers from the hash table when we arc_free().
2575789Sahrens 			 */
2576789Sahrens 			ASSERT(DVA_EQUAL(BP_IDENTITY(&zio->io_bp_orig),
2577789Sahrens 			    BP_IDENTITY(zio->io_bp)));
2578789Sahrens 			ASSERT3U(zio->io_bp_orig.blk_birth, ==,
2579789Sahrens 			    zio->io_bp->blk_birth);
2580789Sahrens 
2581789Sahrens 			ASSERT(refcount_is_zero(&exists->b_refcnt));
25823403Sbmc 			arc_change_state(arc_anon, exists, hash_lock);
2583789Sahrens 			mutex_exit(hash_lock);
25841544Seschrock 			arc_hdr_destroy(exists);
2585789Sahrens 			exists = buf_hash_insert(hdr, &hash_lock);
2586789Sahrens 			ASSERT3P(exists, ==, NULL);
2587789Sahrens 		}
25881544Seschrock 		hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
25892688Smaybee 		arc_access(hdr, hash_lock);
25902688Smaybee 		mutex_exit(hash_lock);
25913547Smaybee 	} else if (callback->awcb_done == NULL) {
25921544Seschrock 		int destroy_hdr;
25931544Seschrock 		/*
25941544Seschrock 		 * This is an anonymous buffer with no user callback,
25951544Seschrock 		 * destroy it if there are no active references.
25961544Seschrock 		 */
25971544Seschrock 		mutex_enter(&arc_eviction_mtx);
25981544Seschrock 		destroy_hdr = refcount_is_zero(&hdr->b_refcnt);
25991544Seschrock 		hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
26001544Seschrock 		mutex_exit(&arc_eviction_mtx);
26011544Seschrock 		if (destroy_hdr)
26021544Seschrock 			arc_hdr_destroy(hdr);
26031544Seschrock 	} else {
26041544Seschrock 		hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
2605789Sahrens 	}
26061544Seschrock 
26073547Smaybee 	if (callback->awcb_done) {
2608789Sahrens 		ASSERT(!refcount_is_zero(&hdr->b_refcnt));
26093547Smaybee 		callback->awcb_done(zio, buf, callback->awcb_private);
2610789Sahrens 	}
2611789Sahrens 
26123547Smaybee 	kmem_free(callback, sizeof (arc_write_callback_t));
2613789Sahrens }
2614789Sahrens 
26153547Smaybee zio_t *
26161775Sbillm arc_write(zio_t *pio, spa_t *spa, int checksum, int compress, int ncopies,
2617789Sahrens     uint64_t txg, blkptr_t *bp, arc_buf_t *buf,
26183547Smaybee     arc_done_func_t *ready, arc_done_func_t *done, void *private, int priority,
26193547Smaybee     int flags, zbookmark_t *zb)
2620789Sahrens {
2621789Sahrens 	arc_buf_hdr_t *hdr = buf->b_hdr;
26223547Smaybee 	arc_write_callback_t *callback;
26233547Smaybee 	zio_t	*zio;
2624789Sahrens 
2625789Sahrens 	/* this is a private buffer - no locking required */
26263403Sbmc 	ASSERT3P(hdr->b_state, ==, arc_anon);
2627789Sahrens 	ASSERT(BUF_EMPTY(hdr));
2628789Sahrens 	ASSERT(!HDR_IO_ERROR(hdr));
26292237Smaybee 	ASSERT((hdr->b_flags & ARC_IO_IN_PROGRESS) == 0);
26302237Smaybee 	ASSERT(hdr->b_acb == 0);
26313547Smaybee 	callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP);
26323547Smaybee 	callback->awcb_ready = ready;
26333547Smaybee 	callback->awcb_done = done;
26343547Smaybee 	callback->awcb_private = private;
26353547Smaybee 	callback->awcb_buf = buf;
26361544Seschrock 	hdr->b_flags |= ARC_IO_IN_PROGRESS;
26373547Smaybee 	zio = zio_write(pio, spa, checksum, compress, ncopies, txg, bp,
26383547Smaybee 	    buf->b_data, hdr->b_size, arc_write_ready, arc_write_done, callback,
26393547Smaybee 	    priority, flags, zb);
2640789Sahrens 
26413547Smaybee 	return (zio);
2642789Sahrens }
2643789Sahrens 
2644789Sahrens int
2645789Sahrens arc_free(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp,
2646789Sahrens     zio_done_func_t *done, void *private, uint32_t arc_flags)
2647789Sahrens {
2648789Sahrens 	arc_buf_hdr_t *ab;
2649789Sahrens 	kmutex_t *hash_lock;
2650789Sahrens 	zio_t	*zio;
2651789Sahrens 
2652789Sahrens 	/*
2653789Sahrens 	 * If this buffer is in the cache, release it, so it
2654789Sahrens 	 * can be re-used.
2655789Sahrens 	 */
2656789Sahrens 	ab = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_lock);
2657789Sahrens 	if (ab != NULL) {
2658789Sahrens 		/*
2659789Sahrens 		 * The checksum of blocks to free is not always
2660789Sahrens 		 * preserved (eg. on the deadlist).  However, if it is
2661789Sahrens 		 * nonzero, it should match what we have in the cache.
2662789Sahrens 		 */
2663789Sahrens 		ASSERT(bp->blk_cksum.zc_word[0] == 0 ||
2664789Sahrens 		    ab->b_cksum0 == bp->blk_cksum.zc_word[0]);
26653403Sbmc 		if (ab->b_state != arc_anon)
26663403Sbmc 			arc_change_state(arc_anon, ab, hash_lock);
26672391Smaybee 		if (HDR_IO_IN_PROGRESS(ab)) {
26682391Smaybee 			/*
26692391Smaybee 			 * This should only happen when we prefetch.
26702391Smaybee 			 */
26712391Smaybee 			ASSERT(ab->b_flags & ARC_PREFETCH);
26722391Smaybee 			ASSERT3U(ab->b_datacnt, ==, 1);
26732391Smaybee 			ab->b_flags |= ARC_FREED_IN_READ;
26742391Smaybee 			if (HDR_IN_HASH_TABLE(ab))
26752391Smaybee 				buf_hash_remove(ab);
26762391Smaybee 			ab->b_arc_access = 0;
26772391Smaybee 			bzero(&ab->b_dva, sizeof (dva_t));
26782391Smaybee 			ab->b_birth = 0;
26792391Smaybee 			ab->b_cksum0 = 0;
26802391Smaybee 			ab->b_buf->b_efunc = NULL;
26812391Smaybee 			ab->b_buf->b_private = NULL;
26822391Smaybee 			mutex_exit(hash_lock);
26832391Smaybee 		} else if (refcount_is_zero(&ab->b_refcnt)) {
2684789Sahrens 			mutex_exit(hash_lock);
26851544Seschrock 			arc_hdr_destroy(ab);
26863403Sbmc 			ARCSTAT_BUMP(arcstat_deleted);
2687789Sahrens 		} else {
26881589Smaybee 			/*
26892391Smaybee 			 * We still have an active reference on this
26902391Smaybee 			 * buffer.  This can happen, e.g., from
26912391Smaybee 			 * dbuf_unoverride().
26921589Smaybee 			 */
26932391Smaybee 			ASSERT(!HDR_IN_HASH_TABLE(ab));
2694789Sahrens 			ab->b_arc_access = 0;
2695789Sahrens 			bzero(&ab->b_dva, sizeof (dva_t));
2696789Sahrens 			ab->b_birth = 0;
2697789Sahrens 			ab->b_cksum0 = 0;
26981544Seschrock 			ab->b_buf->b_efunc = NULL;
26991544Seschrock 			ab->b_buf->b_private = NULL;
2700789Sahrens 			mutex_exit(hash_lock);
2701789Sahrens 		}
2702789Sahrens 	}
2703789Sahrens 
2704789Sahrens 	zio = zio_free(pio, spa, txg, bp, done, private);
2705789Sahrens 
2706789Sahrens 	if (arc_flags & ARC_WAIT)
2707789Sahrens 		return (zio_wait(zio));
2708789Sahrens 
2709789Sahrens 	ASSERT(arc_flags & ARC_NOWAIT);
2710789Sahrens 	zio_nowait(zio);
2711789Sahrens 
2712789Sahrens 	return (0);
2713789Sahrens }
2714789Sahrens 
2715789Sahrens void
2716789Sahrens arc_tempreserve_clear(uint64_t tempreserve)
2717789Sahrens {
2718789Sahrens 	atomic_add_64(&arc_tempreserve, -tempreserve);
2719789Sahrens 	ASSERT((int64_t)arc_tempreserve >= 0);
2720789Sahrens }
2721789Sahrens 
2722789Sahrens int
2723789Sahrens arc_tempreserve_space(uint64_t tempreserve)
2724789Sahrens {
2725789Sahrens #ifdef ZFS_DEBUG
2726789Sahrens 	/*
2727789Sahrens 	 * Once in a while, fail for no reason.  Everything should cope.
2728789Sahrens 	 */
2729789Sahrens 	if (spa_get_random(10000) == 0) {
2730789Sahrens 		dprintf("forcing random failure\n");
2731789Sahrens 		return (ERESTART);
2732789Sahrens 	}
2733789Sahrens #endif
27343403Sbmc 	if (tempreserve > arc_c/4 && !arc_no_grow)
27353403Sbmc 		arc_c = MIN(arc_c_max, tempreserve * 4);
27363403Sbmc 	if (tempreserve > arc_c)
2737982Smaybee 		return (ENOMEM);
2738982Smaybee 
2739789Sahrens 	/*
2740982Smaybee 	 * Throttle writes when the amount of dirty data in the cache
2741982Smaybee 	 * gets too large.  We try to keep the cache less than half full
2742982Smaybee 	 * of dirty blocks so that our sync times don't grow too large.
2743982Smaybee 	 * Note: if two requests come in concurrently, we might let them
2744982Smaybee 	 * both succeed, when one of them should fail.  Not a huge deal.
2745982Smaybee 	 *
2746982Smaybee 	 * XXX The limit should be adjusted dynamically to keep the time
2747982Smaybee 	 * to sync a dataset fixed (around 1-5 seconds?).
2748789Sahrens 	 */
2749789Sahrens 
27503403Sbmc 	if (tempreserve + arc_tempreserve + arc_anon->arcs_size > arc_c / 2 &&
27513403Sbmc 	    arc_tempreserve + arc_anon->arcs_size > arc_c / 4) {
27524309Smaybee 		dprintf("failing, arc_tempreserve=%lluK anon_meta=%lluK "
27534309Smaybee 		    "anon_data=%lluK tempreserve=%lluK arc_c=%lluK\n",
27544309Smaybee 		    arc_tempreserve>>10,
27554309Smaybee 		    arc_anon->arcs_lsize[ARC_BUFC_METADATA]>>10,
27564309Smaybee 		    arc_anon->arcs_lsize[ARC_BUFC_DATA]>>10,
27573403Sbmc 		    tempreserve>>10, arc_c>>10);
2758789Sahrens 		return (ERESTART);
2759789Sahrens 	}
2760789Sahrens 	atomic_add_64(&arc_tempreserve, tempreserve);
2761789Sahrens 	return (0);
2762789Sahrens }
2763789Sahrens 
2764789Sahrens void
2765789Sahrens arc_init(void)
2766789Sahrens {
2767789Sahrens 	mutex_init(&arc_reclaim_thr_lock, NULL, MUTEX_DEFAULT, NULL);
2768789Sahrens 	cv_init(&arc_reclaim_thr_cv, NULL, CV_DEFAULT, NULL);
2769789Sahrens 
27702391Smaybee 	/* Convert seconds to clock ticks */
27712638Sperrin 	arc_min_prefetch_lifespan = 1 * hz;
27722391Smaybee 
2773789Sahrens 	/* Start out with 1/8 of all memory */
27743403Sbmc 	arc_c = physmem * PAGESIZE / 8;
2775789Sahrens 
2776789Sahrens #ifdef _KERNEL
2777789Sahrens 	/*
2778789Sahrens 	 * On architectures where the physical memory can be larger
2779789Sahrens 	 * than the addressable space (intel in 32-bit mode), we may
2780789Sahrens 	 * need to limit the cache to 1/8 of VM size.
2781789Sahrens 	 */
27823403Sbmc 	arc_c = MIN(arc_c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8);
2783789Sahrens #endif
2784789Sahrens 
2785982Smaybee 	/* set min cache to 1/32 of all memory, or 64MB, whichever is more */
27863403Sbmc 	arc_c_min = MAX(arc_c / 4, 64<<20);
2787982Smaybee 	/* set max to 3/4 of all memory, or all but 1GB, whichever is more */
27883403Sbmc 	if (arc_c * 8 >= 1<<30)
27893403Sbmc 		arc_c_max = (arc_c * 8) - (1<<30);
2790789Sahrens 	else
27913403Sbmc 		arc_c_max = arc_c_min;
27923403Sbmc 	arc_c_max = MAX(arc_c * 6, arc_c_max);
27932885Sahrens 
27942885Sahrens 	/*
27952885Sahrens 	 * Allow the tunables to override our calculations if they are
27962885Sahrens 	 * reasonable (ie. over 64MB)
27972885Sahrens 	 */
27982885Sahrens 	if (zfs_arc_max > 64<<20 && zfs_arc_max < physmem * PAGESIZE)
27993403Sbmc 		arc_c_max = zfs_arc_max;
28003403Sbmc 	if (zfs_arc_min > 64<<20 && zfs_arc_min <= arc_c_max)
28013403Sbmc 		arc_c_min = zfs_arc_min;
28022885Sahrens 
28033403Sbmc 	arc_c = arc_c_max;
28043403Sbmc 	arc_p = (arc_c >> 1);
2805789Sahrens 
28064309Smaybee 	/* limit meta-data to 1/4 of the arc capacity */
28074309Smaybee 	arc_meta_limit = arc_c_max / 4;
28084645Sek110237 
28094645Sek110237 	/* Allow the tunable to override if it is reasonable */
28104645Sek110237 	if (zfs_arc_meta_limit > 0 && zfs_arc_meta_limit <= arc_c_max)
28114645Sek110237 		arc_meta_limit = zfs_arc_meta_limit;
28124645Sek110237 
28134309Smaybee 	if (arc_c_min < arc_meta_limit / 2 && zfs_arc_min == 0)
28144309Smaybee 		arc_c_min = arc_meta_limit / 2;
28154309Smaybee 
2816789Sahrens 	/* if kmem_flags are set, lets try to use less memory */
2817789Sahrens 	if (kmem_debugging())
28183403Sbmc 		arc_c = arc_c / 2;
28193403Sbmc 	if (arc_c < arc_c_min)
28203403Sbmc 		arc_c = arc_c_min;
2821789Sahrens 
28223403Sbmc 	arc_anon = &ARC_anon;
28233403Sbmc 	arc_mru = &ARC_mru;
28243403Sbmc 	arc_mru_ghost = &ARC_mru_ghost;
28253403Sbmc 	arc_mfu = &ARC_mfu;
28263403Sbmc 	arc_mfu_ghost = &ARC_mfu_ghost;
28273403Sbmc 	arc_size = 0;
2828789Sahrens 
28293403Sbmc 	mutex_init(&arc_anon->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
28303403Sbmc 	mutex_init(&arc_mru->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
28313403Sbmc 	mutex_init(&arc_mru_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
28323403Sbmc 	mutex_init(&arc_mfu->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
28333403Sbmc 	mutex_init(&arc_mfu_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
28342688Smaybee 
28354309Smaybee 	list_create(&arc_mru->arcs_list[ARC_BUFC_METADATA],
28364309Smaybee 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
28374309Smaybee 	list_create(&arc_mru->arcs_list[ARC_BUFC_DATA],
28384309Smaybee 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
28394309Smaybee 	list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
28404309Smaybee 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
28414309Smaybee 	list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
28424309Smaybee 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
28434309Smaybee 	list_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
28444309Smaybee 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
28454309Smaybee 	list_create(&arc_mfu->arcs_list[ARC_BUFC_DATA],
28464309Smaybee 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
28474309Smaybee 	list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
28484309Smaybee 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
28494309Smaybee 	list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
28504309Smaybee 	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
2851789Sahrens 
2852789Sahrens 	buf_init();
2853789Sahrens 
2854789Sahrens 	arc_thread_exit = 0;
28551544Seschrock 	arc_eviction_list = NULL;
28561544Seschrock 	mutex_init(&arc_eviction_mtx, NULL, MUTEX_DEFAULT, NULL);
28572887Smaybee 	bzero(&arc_eviction_hdr, sizeof (arc_buf_hdr_t));
2858789Sahrens 
28593403Sbmc 	arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED,
28603403Sbmc 	    sizeof (arc_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
28613403Sbmc 
28623403Sbmc 	if (arc_ksp != NULL) {
28633403Sbmc 		arc_ksp->ks_data = &arc_stats;
28643403Sbmc 		kstat_install(arc_ksp);
28653403Sbmc 	}
28663403Sbmc 
2867789Sahrens 	(void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0,
2868789Sahrens 	    TS_RUN, minclsyspri);
28693158Smaybee 
28703158Smaybee 	arc_dead = FALSE;
2871789Sahrens }
2872789Sahrens 
2873789Sahrens void
2874789Sahrens arc_fini(void)
2875789Sahrens {
2876789Sahrens 	mutex_enter(&arc_reclaim_thr_lock);
2877789Sahrens 	arc_thread_exit = 1;
2878789Sahrens 	while (arc_thread_exit != 0)
2879789Sahrens 		cv_wait(&arc_reclaim_thr_cv, &arc_reclaim_thr_lock);
2880789Sahrens 	mutex_exit(&arc_reclaim_thr_lock);
2881789Sahrens 
2882789Sahrens 	arc_flush();
2883789Sahrens 
2884789Sahrens 	arc_dead = TRUE;
2885789Sahrens 
28863403Sbmc 	if (arc_ksp != NULL) {
28873403Sbmc 		kstat_delete(arc_ksp);
28883403Sbmc 		arc_ksp = NULL;
28893403Sbmc 	}
28903403Sbmc 
28911544Seschrock 	mutex_destroy(&arc_eviction_mtx);
2892789Sahrens 	mutex_destroy(&arc_reclaim_thr_lock);
2893789Sahrens 	cv_destroy(&arc_reclaim_thr_cv);
2894789Sahrens 
28954309Smaybee 	list_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]);
28964309Smaybee 	list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
28974309Smaybee 	list_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]);
28984309Smaybee 	list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
28994309Smaybee 	list_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]);
29004309Smaybee 	list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
29014309Smaybee 	list_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]);
29024309Smaybee 	list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
2903789Sahrens 
29043403Sbmc 	mutex_destroy(&arc_anon->arcs_mtx);
29053403Sbmc 	mutex_destroy(&arc_mru->arcs_mtx);
29063403Sbmc 	mutex_destroy(&arc_mru_ghost->arcs_mtx);
29073403Sbmc 	mutex_destroy(&arc_mfu->arcs_mtx);
29083403Sbmc 	mutex_destroy(&arc_mfu_ghost->arcs_mtx);
29092856Snd150628 
2910789Sahrens 	buf_fini();
2911789Sahrens }
2912