xref: /onnv-gate/usr/src/uts/common/fs/zfs/arc.c (revision 2856)
1789Sahrens /*
2789Sahrens  * CDDL HEADER START
3789Sahrens  *
4789Sahrens  * The contents of this file are subject to the terms of the
51484Sek110237  * Common Development and Distribution License (the "License").
61484Sek110237  * You may not use this file except in compliance with the License.
7789Sahrens  *
8789Sahrens  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9789Sahrens  * or http://www.opensolaris.org/os/licensing.
10789Sahrens  * See the License for the specific language governing permissions
11789Sahrens  * and limitations under the License.
12789Sahrens  *
13789Sahrens  * When distributing Covered Code, include this CDDL HEADER in each
14789Sahrens  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15789Sahrens  * If applicable, add the following below this CDDL HEADER, with the
16789Sahrens  * fields enclosed by brackets "[]" replaced with your own identifying
17789Sahrens  * information: Portions Copyright [yyyy] [name of copyright owner]
18789Sahrens  *
19789Sahrens  * CDDL HEADER END
20789Sahrens  */
21789Sahrens /*
221484Sek110237  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23789Sahrens  * Use is subject to license terms.
24789Sahrens  */
25789Sahrens 
26789Sahrens #pragma ident	"%Z%%M%	%I%	%E% SMI"
27789Sahrens 
28789Sahrens /*
29789Sahrens  * DVA-based Adjustable Relpacement Cache
30789Sahrens  *
311544Seschrock  * While much of the theory of operation used here is
321544Seschrock  * based on the self-tuning, low overhead replacement cache
33789Sahrens  * presented by Megiddo and Modha at FAST 2003, there are some
34789Sahrens  * significant differences:
35789Sahrens  *
36789Sahrens  * 1. The Megiddo and Modha model assumes any page is evictable.
37789Sahrens  * Pages in its cache cannot be "locked" into memory.  This makes
38789Sahrens  * the eviction algorithm simple: evict the last page in the list.
39789Sahrens  * This also make the performance characteristics easy to reason
40789Sahrens  * about.  Our cache is not so simple.  At any given moment, some
41789Sahrens  * subset of the blocks in the cache are un-evictable because we
42789Sahrens  * have handed out a reference to them.  Blocks are only evictable
43789Sahrens  * when there are no external references active.  This makes
44789Sahrens  * eviction far more problematic:  we choose to evict the evictable
45789Sahrens  * blocks that are the "lowest" in the list.
46789Sahrens  *
47789Sahrens  * There are times when it is not possible to evict the requested
48789Sahrens  * space.  In these circumstances we are unable to adjust the cache
49789Sahrens  * size.  To prevent the cache growing unbounded at these times we
50789Sahrens  * implement a "cache throttle" that slowes the flow of new data
51789Sahrens  * into the cache until we can make space avaiable.
52789Sahrens  *
53789Sahrens  * 2. The Megiddo and Modha model assumes a fixed cache size.
54789Sahrens  * Pages are evicted when the cache is full and there is a cache
55789Sahrens  * miss.  Our model has a variable sized cache.  It grows with
56789Sahrens  * high use, but also tries to react to memory preasure from the
57789Sahrens  * operating system: decreasing its size when system memory is
58789Sahrens  * tight.
59789Sahrens  *
60789Sahrens  * 3. The Megiddo and Modha model assumes a fixed page size. All
61789Sahrens  * elements of the cache are therefor exactly the same size.  So
62789Sahrens  * when adjusting the cache size following a cache miss, its simply
63789Sahrens  * a matter of choosing a single page to evict.  In our model, we
64789Sahrens  * have variable sized cache blocks (rangeing from 512 bytes to
65789Sahrens  * 128K bytes).  We therefor choose a set of blocks to evict to make
66789Sahrens  * space for a cache miss that approximates as closely as possible
67789Sahrens  * the space used by the new block.
68789Sahrens  *
69789Sahrens  * See also:  "ARC: A Self-Tuning, Low Overhead Replacement Cache"
70789Sahrens  * by N. Megiddo & D. Modha, FAST 2003
71789Sahrens  */
72789Sahrens 
73789Sahrens /*
74789Sahrens  * The locking model:
75789Sahrens  *
76789Sahrens  * A new reference to a cache buffer can be obtained in two
77789Sahrens  * ways: 1) via a hash table lookup using the DVA as a key,
78789Sahrens  * or 2) via one of the ARC lists.  The arc_read() inerface
79789Sahrens  * uses method 1, while the internal arc algorithms for
80789Sahrens  * adjusting the cache use method 2.  We therefor provide two
81789Sahrens  * types of locks: 1) the hash table lock array, and 2) the
82789Sahrens  * arc list locks.
83789Sahrens  *
84789Sahrens  * Buffers do not have their own mutexs, rather they rely on the
85789Sahrens  * hash table mutexs for the bulk of their protection (i.e. most
86789Sahrens  * fields in the arc_buf_hdr_t are protected by these mutexs).
87789Sahrens  *
88789Sahrens  * buf_hash_find() returns the appropriate mutex (held) when it
89789Sahrens  * locates the requested buffer in the hash table.  It returns
90789Sahrens  * NULL for the mutex if the buffer was not in the table.
91789Sahrens  *
92789Sahrens  * buf_hash_remove() expects the appropriate hash mutex to be
93789Sahrens  * already held before it is invoked.
94789Sahrens  *
95789Sahrens  * Each arc state also has a mutex which is used to protect the
96789Sahrens  * buffer list associated with the state.  When attempting to
97789Sahrens  * obtain a hash table lock while holding an arc list lock you
98789Sahrens  * must use: mutex_tryenter() to avoid deadlock.  Also note that
992688Smaybee  * the active state mutex must be held before the ghost state mutex.
100789Sahrens  *
1011544Seschrock  * Arc buffers may have an associated eviction callback function.
1021544Seschrock  * This function will be invoked prior to removing the buffer (e.g.
1031544Seschrock  * in arc_do_user_evicts()).  Note however that the data associated
1041544Seschrock  * with the buffer may be evicted prior to the callback.  The callback
1051544Seschrock  * must be made with *no locks held* (to prevent deadlock).  Additionally,
1061544Seschrock  * the users of callbacks must ensure that their private data is
1071544Seschrock  * protected from simultaneous callbacks from arc_buf_evict()
1081544Seschrock  * and arc_do_user_evicts().
1091544Seschrock  *
110789Sahrens  * Note that the majority of the performance stats are manipulated
111789Sahrens  * with atomic operations.
112789Sahrens  */
113789Sahrens 
114789Sahrens #include <sys/spa.h>
115789Sahrens #include <sys/zio.h>
116789Sahrens #include <sys/zfs_context.h>
117789Sahrens #include <sys/arc.h>
118789Sahrens #include <sys/refcount.h>
119789Sahrens #ifdef _KERNEL
120789Sahrens #include <sys/vmsystm.h>
121789Sahrens #include <vm/anon.h>
122789Sahrens #include <sys/fs/swapnode.h>
1231484Sek110237 #include <sys/dnlc.h>
124789Sahrens #endif
125789Sahrens #include <sys/callb.h>
126789Sahrens 
127789Sahrens static kmutex_t		arc_reclaim_thr_lock;
128789Sahrens static kcondvar_t	arc_reclaim_thr_cv;	/* used to signal reclaim thr */
129789Sahrens static uint8_t		arc_thread_exit;
130789Sahrens 
1311484Sek110237 #define	ARC_REDUCE_DNLC_PERCENT	3
1321484Sek110237 uint_t arc_reduce_dnlc_percent = ARC_REDUCE_DNLC_PERCENT;
1331484Sek110237 
134789Sahrens typedef enum arc_reclaim_strategy {
135789Sahrens 	ARC_RECLAIM_AGGR,		/* Aggressive reclaim strategy */
136789Sahrens 	ARC_RECLAIM_CONS		/* Conservative reclaim strategy */
137789Sahrens } arc_reclaim_strategy_t;
138789Sahrens 
139789Sahrens /* number of seconds before growing cache again */
140789Sahrens static int		arc_grow_retry = 60;
141789Sahrens 
1422391Smaybee /*
1432638Sperrin  * minimum lifespan of a prefetch block in clock ticks
1442638Sperrin  * (initialized in arc_init())
1452391Smaybee  */
1462638Sperrin static int		arc_min_prefetch_lifespan;
1472391Smaybee 
148789Sahrens static kmutex_t arc_reclaim_lock;
149789Sahrens static int arc_dead;
150789Sahrens 
151789Sahrens /*
152789Sahrens  * Note that buffers can be on one of 5 states:
153789Sahrens  *	ARC_anon	- anonymous (discussed below)
1541544Seschrock  *	ARC_mru		- recently used, currently cached
1551544Seschrock  *	ARC_mru_ghost	- recentely used, no longer in cache
1561544Seschrock  *	ARC_mfu		- frequently used, currently cached
1571544Seschrock  *	ARC_mfu_ghost	- frequently used, no longer in cache
158789Sahrens  * When there are no active references to the buffer, they
159789Sahrens  * are linked onto one of the lists in arc.  These are the
160789Sahrens  * only buffers that can be evicted or deleted.
161789Sahrens  *
162789Sahrens  * Anonymous buffers are buffers that are not associated with
163789Sahrens  * a DVA.  These are buffers that hold dirty block copies
164789Sahrens  * before they are written to stable storage.  By definition,
1651544Seschrock  * they are "ref'd" and are considered part of arc_mru
166789Sahrens  * that cannot be freed.  Generally, they will aquire a DVA
1671544Seschrock  * as they are written and migrate onto the arc_mru list.
168789Sahrens  */
169789Sahrens 
170789Sahrens typedef struct arc_state {
171789Sahrens 	list_t	list;	/* linked list of evictable buffer in state */
172789Sahrens 	uint64_t lsize;	/* total size of buffers in the linked list */
173789Sahrens 	uint64_t size;	/* total size of all buffers in this state */
174789Sahrens 	uint64_t hits;
175789Sahrens 	kmutex_t mtx;
176789Sahrens } arc_state_t;
177789Sahrens 
178789Sahrens /* The 5 states: */
179789Sahrens static arc_state_t ARC_anon;
1801544Seschrock static arc_state_t ARC_mru;
1811544Seschrock static arc_state_t ARC_mru_ghost;
1821544Seschrock static arc_state_t ARC_mfu;
1831544Seschrock static arc_state_t ARC_mfu_ghost;
184789Sahrens 
185789Sahrens static struct arc {
186789Sahrens 	arc_state_t 	*anon;
1871544Seschrock 	arc_state_t	*mru;
1881544Seschrock 	arc_state_t	*mru_ghost;
1891544Seschrock 	arc_state_t	*mfu;
1901544Seschrock 	arc_state_t	*mfu_ghost;
191789Sahrens 	uint64_t	size;		/* Actual total arc size */
1921544Seschrock 	uint64_t	p;		/* Target size (in bytes) of mru */
193789Sahrens 	uint64_t	c;		/* Target size of cache (in bytes) */
194789Sahrens 	uint64_t	c_min;		/* Minimum target cache size */
195789Sahrens 	uint64_t	c_max;		/* Maximum target cache size */
196789Sahrens 
197789Sahrens 	/* performance stats */
198789Sahrens 	uint64_t	hits;
199789Sahrens 	uint64_t	misses;
200789Sahrens 	uint64_t	deleted;
2012688Smaybee 	uint64_t	recycle_miss;
2022688Smaybee 	uint64_t	mutex_miss;
2032688Smaybee 	uint64_t	evict_skip;
204789Sahrens 	uint64_t	hash_elements;
205789Sahrens 	uint64_t	hash_elements_max;
206789Sahrens 	uint64_t	hash_collisions;
207789Sahrens 	uint64_t	hash_chains;
208789Sahrens 	uint32_t	hash_chain_max;
209789Sahrens 
210789Sahrens 	int		no_grow;	/* Don't try to grow cache size */
211789Sahrens } arc;
212789Sahrens 
213789Sahrens static uint64_t arc_tempreserve;
214789Sahrens 
215789Sahrens typedef struct arc_callback arc_callback_t;
216789Sahrens 
217789Sahrens struct arc_callback {
218789Sahrens 	arc_done_func_t		*acb_done;
219789Sahrens 	void			*acb_private;
220789Sahrens 	arc_byteswap_func_t	*acb_byteswap;
221789Sahrens 	arc_buf_t		*acb_buf;
222789Sahrens 	zio_t			*acb_zio_dummy;
223789Sahrens 	arc_callback_t		*acb_next;
224789Sahrens };
225789Sahrens 
226789Sahrens struct arc_buf_hdr {
227789Sahrens 	/* immutable */
228789Sahrens 	uint64_t		b_size;
229789Sahrens 	spa_t			*b_spa;
230789Sahrens 
231789Sahrens 	/* protected by hash lock */
232789Sahrens 	dva_t			b_dva;
233789Sahrens 	uint64_t		b_birth;
234789Sahrens 	uint64_t		b_cksum0;
235789Sahrens 
236789Sahrens 	arc_buf_hdr_t		*b_hash_next;
237789Sahrens 	arc_buf_t		*b_buf;
238789Sahrens 	uint32_t		b_flags;
2391544Seschrock 	uint32_t		b_datacnt;
240789Sahrens 
241789Sahrens 	kcondvar_t		b_cv;
242789Sahrens 	arc_callback_t		*b_acb;
243789Sahrens 
244789Sahrens 	/* protected by arc state mutex */
245789Sahrens 	arc_state_t		*b_state;
246789Sahrens 	list_node_t		b_arc_node;
247789Sahrens 
248789Sahrens 	/* updated atomically */
249789Sahrens 	clock_t			b_arc_access;
250789Sahrens 
251789Sahrens 	/* self protecting */
252789Sahrens 	refcount_t		b_refcnt;
253789Sahrens };
254789Sahrens 
2551544Seschrock static arc_buf_t *arc_eviction_list;
2561544Seschrock static kmutex_t arc_eviction_mtx;
2572688Smaybee static void arc_get_data_buf(arc_buf_t *buf);
2582688Smaybee static void arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock);
2591544Seschrock 
2601544Seschrock #define	GHOST_STATE(state)	\
2611544Seschrock 	((state) == arc.mru_ghost || (state) == arc.mfu_ghost)
2621544Seschrock 
263789Sahrens /*
264789Sahrens  * Private ARC flags.  These flags are private ARC only flags that will show up
265789Sahrens  * in b_flags in the arc_hdr_buf_t.  Some flags are publicly declared, and can
266789Sahrens  * be passed in as arc_flags in things like arc_read.  However, these flags
267789Sahrens  * should never be passed and should only be set by ARC code.  When adding new
268789Sahrens  * public flags, make sure not to smash the private ones.
269789Sahrens  */
270789Sahrens 
2711544Seschrock #define	ARC_IN_HASH_TABLE	(1 << 9)	/* this buffer is hashed */
272789Sahrens #define	ARC_IO_IN_PROGRESS	(1 << 10)	/* I/O in progress for buf */
273789Sahrens #define	ARC_IO_ERROR		(1 << 11)	/* I/O failed for buf */
274789Sahrens #define	ARC_FREED_IN_READ	(1 << 12)	/* buf freed while in read */
2751544Seschrock #define	ARC_BUF_AVAILABLE	(1 << 13)	/* block not in active use */
2762391Smaybee #define	ARC_INDIRECT		(1 << 14)	/* this is an indirect block */
277789Sahrens 
2781544Seschrock #define	HDR_IN_HASH_TABLE(hdr)	((hdr)->b_flags & ARC_IN_HASH_TABLE)
279789Sahrens #define	HDR_IO_IN_PROGRESS(hdr)	((hdr)->b_flags & ARC_IO_IN_PROGRESS)
280789Sahrens #define	HDR_IO_ERROR(hdr)	((hdr)->b_flags & ARC_IO_ERROR)
281789Sahrens #define	HDR_FREED_IN_READ(hdr)	((hdr)->b_flags & ARC_FREED_IN_READ)
2821544Seschrock #define	HDR_BUF_AVAILABLE(hdr)	((hdr)->b_flags & ARC_BUF_AVAILABLE)
283789Sahrens 
284789Sahrens /*
285789Sahrens  * Hash table routines
286789Sahrens  */
287789Sahrens 
288789Sahrens #define	HT_LOCK_PAD	64
289789Sahrens 
290789Sahrens struct ht_lock {
291789Sahrens 	kmutex_t	ht_lock;
292789Sahrens #ifdef _KERNEL
293789Sahrens 	unsigned char	pad[(HT_LOCK_PAD - sizeof (kmutex_t))];
294789Sahrens #endif
295789Sahrens };
296789Sahrens 
297789Sahrens #define	BUF_LOCKS 256
298789Sahrens typedef struct buf_hash_table {
299789Sahrens 	uint64_t ht_mask;
300789Sahrens 	arc_buf_hdr_t **ht_table;
301789Sahrens 	struct ht_lock ht_locks[BUF_LOCKS];
302789Sahrens } buf_hash_table_t;
303789Sahrens 
304789Sahrens static buf_hash_table_t buf_hash_table;
305789Sahrens 
306789Sahrens #define	BUF_HASH_INDEX(spa, dva, birth) \
307789Sahrens 	(buf_hash(spa, dva, birth) & buf_hash_table.ht_mask)
308789Sahrens #define	BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)])
309789Sahrens #define	BUF_HASH_LOCK(idx)	(&(BUF_HASH_LOCK_NTRY(idx).ht_lock))
310789Sahrens #define	HDR_LOCK(buf) \
311789Sahrens 	(BUF_HASH_LOCK(BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth)))
312789Sahrens 
313789Sahrens uint64_t zfs_crc64_table[256];
314789Sahrens 
315789Sahrens static uint64_t
316789Sahrens buf_hash(spa_t *spa, dva_t *dva, uint64_t birth)
317789Sahrens {
318789Sahrens 	uintptr_t spav = (uintptr_t)spa;
319789Sahrens 	uint8_t *vdva = (uint8_t *)dva;
320789Sahrens 	uint64_t crc = -1ULL;
321789Sahrens 	int i;
322789Sahrens 
323789Sahrens 	ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
324789Sahrens 
325789Sahrens 	for (i = 0; i < sizeof (dva_t); i++)
326789Sahrens 		crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF];
327789Sahrens 
328789Sahrens 	crc ^= (spav>>8) ^ birth;
329789Sahrens 
330789Sahrens 	return (crc);
331789Sahrens }
332789Sahrens 
333789Sahrens #define	BUF_EMPTY(buf)						\
334789Sahrens 	((buf)->b_dva.dva_word[0] == 0 &&			\
335789Sahrens 	(buf)->b_dva.dva_word[1] == 0 &&			\
336789Sahrens 	(buf)->b_birth == 0)
337789Sahrens 
338789Sahrens #define	BUF_EQUAL(spa, dva, birth, buf)				\
339789Sahrens 	((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) &&	\
340789Sahrens 	((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) &&	\
341789Sahrens 	((buf)->b_birth == birth) && ((buf)->b_spa == spa)
342789Sahrens 
343789Sahrens static arc_buf_hdr_t *
344789Sahrens buf_hash_find(spa_t *spa, dva_t *dva, uint64_t birth, kmutex_t **lockp)
345789Sahrens {
346789Sahrens 	uint64_t idx = BUF_HASH_INDEX(spa, dva, birth);
347789Sahrens 	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
348789Sahrens 	arc_buf_hdr_t *buf;
349789Sahrens 
350789Sahrens 	mutex_enter(hash_lock);
351789Sahrens 	for (buf = buf_hash_table.ht_table[idx]; buf != NULL;
352789Sahrens 	    buf = buf->b_hash_next) {
353789Sahrens 		if (BUF_EQUAL(spa, dva, birth, buf)) {
354789Sahrens 			*lockp = hash_lock;
355789Sahrens 			return (buf);
356789Sahrens 		}
357789Sahrens 	}
358789Sahrens 	mutex_exit(hash_lock);
359789Sahrens 	*lockp = NULL;
360789Sahrens 	return (NULL);
361789Sahrens }
362789Sahrens 
363789Sahrens /*
364789Sahrens  * Insert an entry into the hash table.  If there is already an element
365789Sahrens  * equal to elem in the hash table, then the already existing element
366789Sahrens  * will be returned and the new element will not be inserted.
367789Sahrens  * Otherwise returns NULL.
368789Sahrens  */
369789Sahrens static arc_buf_hdr_t *
370789Sahrens buf_hash_insert(arc_buf_hdr_t *buf, kmutex_t **lockp)
371789Sahrens {
372789Sahrens 	uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth);
373789Sahrens 	kmutex_t *hash_lock = BUF_HASH_LOCK(idx);
374789Sahrens 	arc_buf_hdr_t *fbuf;
375789Sahrens 	uint32_t max, i;
376789Sahrens 
3771544Seschrock 	ASSERT(!HDR_IN_HASH_TABLE(buf));
378789Sahrens 	*lockp = hash_lock;
379789Sahrens 	mutex_enter(hash_lock);
380789Sahrens 	for (fbuf = buf_hash_table.ht_table[idx], i = 0; fbuf != NULL;
381789Sahrens 	    fbuf = fbuf->b_hash_next, i++) {
382789Sahrens 		if (BUF_EQUAL(buf->b_spa, &buf->b_dva, buf->b_birth, fbuf))
383789Sahrens 			return (fbuf);
384789Sahrens 	}
385789Sahrens 
386789Sahrens 	buf->b_hash_next = buf_hash_table.ht_table[idx];
387789Sahrens 	buf_hash_table.ht_table[idx] = buf;
3881544Seschrock 	buf->b_flags |= ARC_IN_HASH_TABLE;
389789Sahrens 
390789Sahrens 	/* collect some hash table performance data */
391789Sahrens 	if (i > 0) {
392789Sahrens 		atomic_add_64(&arc.hash_collisions, 1);
393789Sahrens 		if (i == 1)
394789Sahrens 			atomic_add_64(&arc.hash_chains, 1);
395789Sahrens 	}
396789Sahrens 	while (i > (max = arc.hash_chain_max) &&
397789Sahrens 	    max != atomic_cas_32(&arc.hash_chain_max, max, i)) {
398789Sahrens 		continue;
399789Sahrens 	}
400789Sahrens 	atomic_add_64(&arc.hash_elements, 1);
401789Sahrens 	if (arc.hash_elements > arc.hash_elements_max)
402789Sahrens 		atomic_add_64(&arc.hash_elements_max, 1);
403789Sahrens 
404789Sahrens 	return (NULL);
405789Sahrens }
406789Sahrens 
407789Sahrens static void
408789Sahrens buf_hash_remove(arc_buf_hdr_t *buf)
409789Sahrens {
410789Sahrens 	arc_buf_hdr_t *fbuf, **bufp;
411789Sahrens 	uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth);
412789Sahrens 
413789Sahrens 	ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx)));
4141544Seschrock 	ASSERT(HDR_IN_HASH_TABLE(buf));
415789Sahrens 
416789Sahrens 	bufp = &buf_hash_table.ht_table[idx];
417789Sahrens 	while ((fbuf = *bufp) != buf) {
418789Sahrens 		ASSERT(fbuf != NULL);
419789Sahrens 		bufp = &fbuf->b_hash_next;
420789Sahrens 	}
421789Sahrens 	*bufp = buf->b_hash_next;
422789Sahrens 	buf->b_hash_next = NULL;
4231544Seschrock 	buf->b_flags &= ~ARC_IN_HASH_TABLE;
424789Sahrens 
425789Sahrens 	/* collect some hash table performance data */
426789Sahrens 	atomic_add_64(&arc.hash_elements, -1);
427789Sahrens 	if (buf_hash_table.ht_table[idx] &&
428789Sahrens 	    buf_hash_table.ht_table[idx]->b_hash_next == NULL)
429789Sahrens 		atomic_add_64(&arc.hash_chains, -1);
430789Sahrens }
431789Sahrens 
432789Sahrens /*
433789Sahrens  * Global data structures and functions for the buf kmem cache.
434789Sahrens  */
435789Sahrens static kmem_cache_t *hdr_cache;
436789Sahrens static kmem_cache_t *buf_cache;
437789Sahrens 
438789Sahrens static void
439789Sahrens buf_fini(void)
440789Sahrens {
441789Sahrens 	int i;
442789Sahrens 
443789Sahrens 	kmem_free(buf_hash_table.ht_table,
444789Sahrens 	    (buf_hash_table.ht_mask + 1) * sizeof (void *));
445789Sahrens 	for (i = 0; i < BUF_LOCKS; i++)
446789Sahrens 		mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock);
447789Sahrens 	kmem_cache_destroy(hdr_cache);
448789Sahrens 	kmem_cache_destroy(buf_cache);
449789Sahrens }
450789Sahrens 
451789Sahrens /*
452789Sahrens  * Constructor callback - called when the cache is empty
453789Sahrens  * and a new buf is requested.
454789Sahrens  */
455789Sahrens /* ARGSUSED */
456789Sahrens static int
457789Sahrens hdr_cons(void *vbuf, void *unused, int kmflag)
458789Sahrens {
459789Sahrens 	arc_buf_hdr_t *buf = vbuf;
460789Sahrens 
461789Sahrens 	bzero(buf, sizeof (arc_buf_hdr_t));
462789Sahrens 	refcount_create(&buf->b_refcnt);
463789Sahrens 	cv_init(&buf->b_cv, NULL, CV_DEFAULT, NULL);
464789Sahrens 	return (0);
465789Sahrens }
466789Sahrens 
467789Sahrens /*
468789Sahrens  * Destructor callback - called when a cached buf is
469789Sahrens  * no longer required.
470789Sahrens  */
471789Sahrens /* ARGSUSED */
472789Sahrens static void
473789Sahrens hdr_dest(void *vbuf, void *unused)
474789Sahrens {
475789Sahrens 	arc_buf_hdr_t *buf = vbuf;
476789Sahrens 
477789Sahrens 	refcount_destroy(&buf->b_refcnt);
478789Sahrens 	cv_destroy(&buf->b_cv);
479789Sahrens }
480789Sahrens 
4811544Seschrock static int arc_reclaim_needed(void);
482789Sahrens void arc_kmem_reclaim(void);
483789Sahrens 
484789Sahrens /*
485789Sahrens  * Reclaim callback -- invoked when memory is low.
486789Sahrens  */
487789Sahrens /* ARGSUSED */
488789Sahrens static void
489789Sahrens hdr_recl(void *unused)
490789Sahrens {
491789Sahrens 	dprintf("hdr_recl called\n");
4921544Seschrock 	if (arc_reclaim_needed())
4931544Seschrock 		arc_kmem_reclaim();
494789Sahrens }
495789Sahrens 
496789Sahrens static void
497789Sahrens buf_init(void)
498789Sahrens {
499789Sahrens 	uint64_t *ct;
5001544Seschrock 	uint64_t hsize = 1ULL << 12;
501789Sahrens 	int i, j;
502789Sahrens 
503789Sahrens 	/*
504789Sahrens 	 * The hash table is big enough to fill all of physical memory
5051544Seschrock 	 * with an average 64K block size.  The table will take up
5061544Seschrock 	 * totalmem*sizeof(void*)/64K (eg. 128KB/GB with 8-byte pointers).
507789Sahrens 	 */
5081544Seschrock 	while (hsize * 65536 < physmem * PAGESIZE)
509789Sahrens 		hsize <<= 1;
5101544Seschrock retry:
511789Sahrens 	buf_hash_table.ht_mask = hsize - 1;
5121544Seschrock 	buf_hash_table.ht_table =
5131544Seschrock 	    kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP);
5141544Seschrock 	if (buf_hash_table.ht_table == NULL) {
5151544Seschrock 		ASSERT(hsize > (1ULL << 8));
5161544Seschrock 		hsize >>= 1;
5171544Seschrock 		goto retry;
5181544Seschrock 	}
519789Sahrens 
520789Sahrens 	hdr_cache = kmem_cache_create("arc_buf_hdr_t", sizeof (arc_buf_hdr_t),
521789Sahrens 	    0, hdr_cons, hdr_dest, hdr_recl, NULL, NULL, 0);
522789Sahrens 	buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t),
523789Sahrens 	    0, NULL, NULL, NULL, NULL, NULL, 0);
524789Sahrens 
525789Sahrens 	for (i = 0; i < 256; i++)
526789Sahrens 		for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--)
527789Sahrens 			*ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY);
528789Sahrens 
529789Sahrens 	for (i = 0; i < BUF_LOCKS; i++) {
530789Sahrens 		mutex_init(&buf_hash_table.ht_locks[i].ht_lock,
531789Sahrens 		    NULL, MUTEX_DEFAULT, NULL);
532789Sahrens 	}
533789Sahrens }
534789Sahrens 
535789Sahrens #define	ARC_MINTIME	(hz>>4) /* 62 ms */
536789Sahrens 
537789Sahrens static void
538789Sahrens add_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag)
539789Sahrens {
540789Sahrens 	ASSERT(MUTEX_HELD(hash_lock));
541789Sahrens 
542789Sahrens 	if ((refcount_add(&ab->b_refcnt, tag) == 1) &&
543789Sahrens 	    (ab->b_state != arc.anon)) {
5441544Seschrock 		int delta = ab->b_size * ab->b_datacnt;
545789Sahrens 
546789Sahrens 		ASSERT(!MUTEX_HELD(&ab->b_state->mtx));
547789Sahrens 		mutex_enter(&ab->b_state->mtx);
548789Sahrens 		ASSERT(list_link_active(&ab->b_arc_node));
549789Sahrens 		list_remove(&ab->b_state->list, ab);
5501544Seschrock 		if (GHOST_STATE(ab->b_state)) {
5511544Seschrock 			ASSERT3U(ab->b_datacnt, ==, 0);
5521544Seschrock 			ASSERT3P(ab->b_buf, ==, NULL);
5531544Seschrock 			delta = ab->b_size;
5541544Seschrock 		}
5551544Seschrock 		ASSERT(delta > 0);
5561544Seschrock 		ASSERT3U(ab->b_state->lsize, >=, delta);
5571544Seschrock 		atomic_add_64(&ab->b_state->lsize, -delta);
558789Sahrens 		mutex_exit(&ab->b_state->mtx);
5592391Smaybee 		/* remove the prefetch flag is we get a reference */
5602391Smaybee 		if (ab->b_flags & ARC_PREFETCH)
5612391Smaybee 			ab->b_flags &= ~ARC_PREFETCH;
562789Sahrens 	}
563789Sahrens }
564789Sahrens 
565789Sahrens static int
566789Sahrens remove_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag)
567789Sahrens {
568789Sahrens 	int cnt;
569789Sahrens 
5701544Seschrock 	ASSERT(ab->b_state == arc.anon || MUTEX_HELD(hash_lock));
5711544Seschrock 	ASSERT(!GHOST_STATE(ab->b_state));
572789Sahrens 
573789Sahrens 	if (((cnt = refcount_remove(&ab->b_refcnt, tag)) == 0) &&
574789Sahrens 	    (ab->b_state != arc.anon)) {
575789Sahrens 
576789Sahrens 		ASSERT(!MUTEX_HELD(&ab->b_state->mtx));
577789Sahrens 		mutex_enter(&ab->b_state->mtx);
578789Sahrens 		ASSERT(!list_link_active(&ab->b_arc_node));
579789Sahrens 		list_insert_head(&ab->b_state->list, ab);
5801544Seschrock 		ASSERT(ab->b_datacnt > 0);
5811544Seschrock 		atomic_add_64(&ab->b_state->lsize, ab->b_size * ab->b_datacnt);
5821544Seschrock 		ASSERT3U(ab->b_state->size, >=, ab->b_state->lsize);
583789Sahrens 		mutex_exit(&ab->b_state->mtx);
584789Sahrens 	}
585789Sahrens 	return (cnt);
586789Sahrens }
587789Sahrens 
588789Sahrens /*
589789Sahrens  * Move the supplied buffer to the indicated state.  The mutex
590789Sahrens  * for the buffer must be held by the caller.
591789Sahrens  */
592789Sahrens static void
5931544Seschrock arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *ab, kmutex_t *hash_lock)
594789Sahrens {
5951544Seschrock 	arc_state_t *old_state = ab->b_state;
5961544Seschrock 	int refcnt = refcount_count(&ab->b_refcnt);
5971544Seschrock 	int from_delta, to_delta;
598789Sahrens 
599789Sahrens 	ASSERT(MUTEX_HELD(hash_lock));
6001544Seschrock 	ASSERT(new_state != old_state);
6011544Seschrock 	ASSERT(refcnt == 0 || ab->b_datacnt > 0);
6021544Seschrock 	ASSERT(ab->b_datacnt == 0 || !GHOST_STATE(new_state));
6031544Seschrock 
6041544Seschrock 	from_delta = to_delta = ab->b_datacnt * ab->b_size;
605789Sahrens 
606789Sahrens 	/*
607789Sahrens 	 * If this buffer is evictable, transfer it from the
608789Sahrens 	 * old state list to the new state list.
609789Sahrens 	 */
6101544Seschrock 	if (refcnt == 0) {
6111544Seschrock 		if (old_state != arc.anon) {
6121544Seschrock 			int use_mutex = !MUTEX_HELD(&old_state->mtx);
6131544Seschrock 
6141544Seschrock 			if (use_mutex)
6151544Seschrock 				mutex_enter(&old_state->mtx);
6161544Seschrock 
6171544Seschrock 			ASSERT(list_link_active(&ab->b_arc_node));
6181544Seschrock 			list_remove(&old_state->list, ab);
619789Sahrens 
6202391Smaybee 			/*
6212391Smaybee 			 * If prefetching out of the ghost cache,
6222391Smaybee 			 * we will have a non-null datacnt.
6232391Smaybee 			 */
6242391Smaybee 			if (GHOST_STATE(old_state) && ab->b_datacnt == 0) {
6252391Smaybee 				/* ghost elements have a ghost size */
6261544Seschrock 				ASSERT(ab->b_buf == NULL);
6271544Seschrock 				from_delta = ab->b_size;
628789Sahrens 			}
6291544Seschrock 			ASSERT3U(old_state->lsize, >=, from_delta);
6301544Seschrock 			atomic_add_64(&old_state->lsize, -from_delta);
6311544Seschrock 
6321544Seschrock 			if (use_mutex)
6331544Seschrock 				mutex_exit(&old_state->mtx);
634789Sahrens 		}
635789Sahrens 		if (new_state != arc.anon) {
6361544Seschrock 			int use_mutex = !MUTEX_HELD(&new_state->mtx);
637789Sahrens 
6381544Seschrock 			if (use_mutex)
639789Sahrens 				mutex_enter(&new_state->mtx);
6401544Seschrock 
641789Sahrens 			list_insert_head(&new_state->list, ab);
6421544Seschrock 
6431544Seschrock 			/* ghost elements have a ghost size */
6441544Seschrock 			if (GHOST_STATE(new_state)) {
6451544Seschrock 				ASSERT(ab->b_datacnt == 0);
6461544Seschrock 				ASSERT(ab->b_buf == NULL);
6471544Seschrock 				to_delta = ab->b_size;
6481544Seschrock 			}
6491544Seschrock 			atomic_add_64(&new_state->lsize, to_delta);
6501544Seschrock 			ASSERT3U(new_state->size + to_delta, >=,
6511544Seschrock 			    new_state->lsize);
6521544Seschrock 
6531544Seschrock 			if (use_mutex)
654789Sahrens 				mutex_exit(&new_state->mtx);
655789Sahrens 		}
656789Sahrens 	}
657789Sahrens 
658789Sahrens 	ASSERT(!BUF_EMPTY(ab));
6591544Seschrock 	if (new_state == arc.anon && old_state != arc.anon) {
660789Sahrens 		buf_hash_remove(ab);
661789Sahrens 	}
662789Sahrens 
6631544Seschrock 	/* adjust state sizes */
6641544Seschrock 	if (to_delta)
6651544Seschrock 		atomic_add_64(&new_state->size, to_delta);
6661544Seschrock 	if (from_delta) {
6671544Seschrock 		ASSERT3U(old_state->size, >=, from_delta);
6681544Seschrock 		atomic_add_64(&old_state->size, -from_delta);
669789Sahrens 	}
670789Sahrens 	ab->b_state = new_state;
671789Sahrens }
672789Sahrens 
673789Sahrens arc_buf_t *
674789Sahrens arc_buf_alloc(spa_t *spa, int size, void *tag)
675789Sahrens {
676789Sahrens 	arc_buf_hdr_t *hdr;
677789Sahrens 	arc_buf_t *buf;
678789Sahrens 
679789Sahrens 	ASSERT3U(size, >, 0);
680789Sahrens 	hdr = kmem_cache_alloc(hdr_cache, KM_SLEEP);
681789Sahrens 	ASSERT(BUF_EMPTY(hdr));
682789Sahrens 	hdr->b_size = size;
683789Sahrens 	hdr->b_spa = spa;
684789Sahrens 	hdr->b_state = arc.anon;
685789Sahrens 	hdr->b_arc_access = 0;
686789Sahrens 	buf = kmem_cache_alloc(buf_cache, KM_SLEEP);
687789Sahrens 	buf->b_hdr = hdr;
6882688Smaybee 	buf->b_data = NULL;
6891544Seschrock 	buf->b_efunc = NULL;
6901544Seschrock 	buf->b_private = NULL;
691789Sahrens 	buf->b_next = NULL;
692789Sahrens 	hdr->b_buf = buf;
6932688Smaybee 	arc_get_data_buf(buf);
6941544Seschrock 	hdr->b_datacnt = 1;
695789Sahrens 	hdr->b_flags = 0;
696789Sahrens 	ASSERT(refcount_is_zero(&hdr->b_refcnt));
697789Sahrens 	(void) refcount_add(&hdr->b_refcnt, tag);
698789Sahrens 
699789Sahrens 	return (buf);
700789Sahrens }
701789Sahrens 
7022688Smaybee static arc_buf_t *
7032688Smaybee arc_buf_clone(arc_buf_t *from)
7041544Seschrock {
7052688Smaybee 	arc_buf_t *buf;
7062688Smaybee 	arc_buf_hdr_t *hdr = from->b_hdr;
7072688Smaybee 	uint64_t size = hdr->b_size;
7081544Seschrock 
7092688Smaybee 	buf = kmem_cache_alloc(buf_cache, KM_SLEEP);
7102688Smaybee 	buf->b_hdr = hdr;
7112688Smaybee 	buf->b_data = NULL;
7122688Smaybee 	buf->b_efunc = NULL;
7132688Smaybee 	buf->b_private = NULL;
7142688Smaybee 	buf->b_next = hdr->b_buf;
7152688Smaybee 	hdr->b_buf = buf;
7162688Smaybee 	arc_get_data_buf(buf);
7172688Smaybee 	bcopy(from->b_data, buf->b_data, size);
7182688Smaybee 	hdr->b_datacnt += 1;
7192688Smaybee 	return (buf);
7201544Seschrock }
7211544Seschrock 
7221544Seschrock void
7231544Seschrock arc_buf_add_ref(arc_buf_t *buf, void* tag)
7241544Seschrock {
7252724Smaybee 	arc_buf_hdr_t *hdr = buf->b_hdr;
7261544Seschrock 	kmutex_t *hash_lock;
7271544Seschrock 
7282724Smaybee 	/*
7292724Smaybee 	 * Check to see if this buffer is currently being evicted via
7302724Smaybee 	 * arc_do_user_evicts().  We can do this without holding any
7312724Smaybee 	 * locks because if we happen to obtain the header before its
7322724Smaybee 	 * cleared, we will find b_data is NULL later.
7332724Smaybee 	 */
7342724Smaybee 	if (hdr == NULL)
7352724Smaybee 		return;
7362724Smaybee 
7372724Smaybee 	hash_lock = HDR_LOCK(hdr);
7382724Smaybee 	mutex_enter(hash_lock);
7391544Seschrock 	if (buf->b_data == NULL) {
7401544Seschrock 		/*
7411544Seschrock 		 * This buffer is evicted.
7421544Seschrock 		 */
7432724Smaybee 		mutex_exit(hash_lock);
7441544Seschrock 		return;
7451544Seschrock 	}
7461544Seschrock 
7472724Smaybee 	ASSERT(buf->b_hdr == hdr);
7482724Smaybee 	ASSERT(hdr->b_state == arc.mru || hdr->b_state == arc.mfu);
7491544Seschrock 	add_reference(hdr, hash_lock, tag);
7502688Smaybee 	arc_access(hdr, hash_lock);
7512688Smaybee 	mutex_exit(hash_lock);
7521544Seschrock 	atomic_add_64(&arc.hits, 1);
7531544Seschrock }
7541544Seschrock 
755789Sahrens static void
7562688Smaybee arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t all)
7571544Seschrock {
7581544Seschrock 	arc_buf_t **bufp;
7591544Seschrock 
7601544Seschrock 	/* free up data associated with the buf */
7611544Seschrock 	if (buf->b_data) {
7621544Seschrock 		arc_state_t *state = buf->b_hdr->b_state;
7631544Seschrock 		uint64_t size = buf->b_hdr->b_size;
7641544Seschrock 
7652688Smaybee 		if (!recycle) {
7662688Smaybee 			zio_buf_free(buf->b_data, size);
7672688Smaybee 			atomic_add_64(&arc.size, -size);
7682688Smaybee 		}
7691544Seschrock 		if (list_link_active(&buf->b_hdr->b_arc_node)) {
7701544Seschrock 			ASSERT(refcount_is_zero(&buf->b_hdr->b_refcnt));
7711544Seschrock 			ASSERT(state != arc.anon);
7721544Seschrock 			ASSERT3U(state->lsize, >=, size);
7731544Seschrock 			atomic_add_64(&state->lsize, -size);
7741544Seschrock 		}
7751544Seschrock 		ASSERT3U(state->size, >=, size);
7761544Seschrock 		atomic_add_64(&state->size, -size);
7771544Seschrock 		buf->b_data = NULL;
7781544Seschrock 		ASSERT(buf->b_hdr->b_datacnt > 0);
7791544Seschrock 		buf->b_hdr->b_datacnt -= 1;
7801544Seschrock 	}
7811544Seschrock 
7821544Seschrock 	/* only remove the buf if requested */
7831544Seschrock 	if (!all)
7841544Seschrock 		return;
7851544Seschrock 
7861544Seschrock 	/* remove the buf from the hdr list */
7871544Seschrock 	for (bufp = &buf->b_hdr->b_buf; *bufp != buf; bufp = &(*bufp)->b_next)
7881544Seschrock 		continue;
7891544Seschrock 	*bufp = buf->b_next;
7901544Seschrock 
7911544Seschrock 	ASSERT(buf->b_efunc == NULL);
7921544Seschrock 
7931544Seschrock 	/* clean up the buf */
7941544Seschrock 	buf->b_hdr = NULL;
7951544Seschrock 	kmem_cache_free(buf_cache, buf);
7961544Seschrock }
7971544Seschrock 
7981544Seschrock static void
7991544Seschrock arc_hdr_destroy(arc_buf_hdr_t *hdr)
800789Sahrens {
801789Sahrens 	ASSERT(refcount_is_zero(&hdr->b_refcnt));
802789Sahrens 	ASSERT3P(hdr->b_state, ==, arc.anon);
8031544Seschrock 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
804789Sahrens 
805789Sahrens 	if (!BUF_EMPTY(hdr)) {
8061544Seschrock 		ASSERT(!HDR_IN_HASH_TABLE(hdr));
807789Sahrens 		bzero(&hdr->b_dva, sizeof (dva_t));
808789Sahrens 		hdr->b_birth = 0;
809789Sahrens 		hdr->b_cksum0 = 0;
810789Sahrens 	}
8111544Seschrock 	while (hdr->b_buf) {
812789Sahrens 		arc_buf_t *buf = hdr->b_buf;
813789Sahrens 
8141544Seschrock 		if (buf->b_efunc) {
8151544Seschrock 			mutex_enter(&arc_eviction_mtx);
8161544Seschrock 			ASSERT(buf->b_hdr != NULL);
8172688Smaybee 			arc_buf_destroy(hdr->b_buf, FALSE, FALSE);
8181544Seschrock 			hdr->b_buf = buf->b_next;
8191544Seschrock 			buf->b_next = arc_eviction_list;
8201544Seschrock 			arc_eviction_list = buf;
8211544Seschrock 			mutex_exit(&arc_eviction_mtx);
8221544Seschrock 		} else {
8232688Smaybee 			arc_buf_destroy(hdr->b_buf, FALSE, TRUE);
8241544Seschrock 		}
825789Sahrens 	}
8261544Seschrock 
827789Sahrens 	ASSERT(!list_link_active(&hdr->b_arc_node));
828789Sahrens 	ASSERT3P(hdr->b_hash_next, ==, NULL);
829789Sahrens 	ASSERT3P(hdr->b_acb, ==, NULL);
830789Sahrens 	kmem_cache_free(hdr_cache, hdr);
831789Sahrens }
832789Sahrens 
833789Sahrens void
834789Sahrens arc_buf_free(arc_buf_t *buf, void *tag)
835789Sahrens {
836789Sahrens 	arc_buf_hdr_t *hdr = buf->b_hdr;
8371544Seschrock 	int hashed = hdr->b_state != arc.anon;
8381544Seschrock 
8391544Seschrock 	ASSERT(buf->b_efunc == NULL);
8401544Seschrock 	ASSERT(buf->b_data != NULL);
8411544Seschrock 
8421544Seschrock 	if (hashed) {
8431544Seschrock 		kmutex_t *hash_lock = HDR_LOCK(hdr);
8441544Seschrock 
8451544Seschrock 		mutex_enter(hash_lock);
8461544Seschrock 		(void) remove_reference(hdr, hash_lock, tag);
8471544Seschrock 		if (hdr->b_datacnt > 1)
8482688Smaybee 			arc_buf_destroy(buf, FALSE, TRUE);
8491544Seschrock 		else
8501544Seschrock 			hdr->b_flags |= ARC_BUF_AVAILABLE;
8511544Seschrock 		mutex_exit(hash_lock);
8521544Seschrock 	} else if (HDR_IO_IN_PROGRESS(hdr)) {
8531544Seschrock 		int destroy_hdr;
8541544Seschrock 		/*
8551544Seschrock 		 * We are in the middle of an async write.  Don't destroy
8561544Seschrock 		 * this buffer unless the write completes before we finish
8571544Seschrock 		 * decrementing the reference count.
8581544Seschrock 		 */
8591544Seschrock 		mutex_enter(&arc_eviction_mtx);
8601544Seschrock 		(void) remove_reference(hdr, NULL, tag);
8611544Seschrock 		ASSERT(refcount_is_zero(&hdr->b_refcnt));
8621544Seschrock 		destroy_hdr = !HDR_IO_IN_PROGRESS(hdr);
8631544Seschrock 		mutex_exit(&arc_eviction_mtx);
8641544Seschrock 		if (destroy_hdr)
8651544Seschrock 			arc_hdr_destroy(hdr);
8661544Seschrock 	} else {
8671544Seschrock 		if (remove_reference(hdr, NULL, tag) > 0) {
8681544Seschrock 			ASSERT(HDR_IO_ERROR(hdr));
8692688Smaybee 			arc_buf_destroy(buf, FALSE, TRUE);
8701544Seschrock 		} else {
8711544Seschrock 			arc_hdr_destroy(hdr);
8721544Seschrock 		}
8731544Seschrock 	}
8741544Seschrock }
8751544Seschrock 
8761544Seschrock int
8771544Seschrock arc_buf_remove_ref(arc_buf_t *buf, void* tag)
8781544Seschrock {
8791544Seschrock 	arc_buf_hdr_t *hdr = buf->b_hdr;
880789Sahrens 	kmutex_t *hash_lock = HDR_LOCK(hdr);
8811544Seschrock 	int no_callback = (buf->b_efunc == NULL);
8821544Seschrock 
8831544Seschrock 	if (hdr->b_state == arc.anon) {
8841544Seschrock 		arc_buf_free(buf, tag);
8851544Seschrock 		return (no_callback);
8861544Seschrock 	}
887789Sahrens 
888789Sahrens 	mutex_enter(hash_lock);
8891544Seschrock 	ASSERT(hdr->b_state != arc.anon);
8901544Seschrock 	ASSERT(buf->b_data != NULL);
891789Sahrens 
8921544Seschrock 	(void) remove_reference(hdr, hash_lock, tag);
8931544Seschrock 	if (hdr->b_datacnt > 1) {
8941544Seschrock 		if (no_callback)
8952688Smaybee 			arc_buf_destroy(buf, FALSE, TRUE);
8961544Seschrock 	} else if (no_callback) {
8971544Seschrock 		ASSERT(hdr->b_buf == buf && buf->b_next == NULL);
8981544Seschrock 		hdr->b_flags |= ARC_BUF_AVAILABLE;
899789Sahrens 	}
9001544Seschrock 	ASSERT(no_callback || hdr->b_datacnt > 1 ||
9011544Seschrock 	    refcount_is_zero(&hdr->b_refcnt));
902789Sahrens 	mutex_exit(hash_lock);
9031544Seschrock 	return (no_callback);
904789Sahrens }
905789Sahrens 
906789Sahrens int
907789Sahrens arc_buf_size(arc_buf_t *buf)
908789Sahrens {
909789Sahrens 	return (buf->b_hdr->b_size);
910789Sahrens }
911789Sahrens 
912789Sahrens /*
913789Sahrens  * Evict buffers from list until we've removed the specified number of
914789Sahrens  * bytes.  Move the removed buffers to the appropriate evict state.
9152688Smaybee  * If the recycle flag is set, then attempt to "recycle" a buffer:
9162688Smaybee  * - look for a buffer to evict that is `bytes' long.
9172688Smaybee  * - return the data block from this buffer rather than freeing it.
9182688Smaybee  * This flag is used by callers that are trying to make space for a
9192688Smaybee  * new buffer in a full arc cache.
920789Sahrens  */
9212688Smaybee static void *
9222688Smaybee arc_evict(arc_state_t *state, int64_t bytes, boolean_t recycle)
923789Sahrens {
924789Sahrens 	arc_state_t *evicted_state;
9252688Smaybee 	uint64_t bytes_evicted = 0, skipped = 0, missed = 0;
926789Sahrens 	arc_buf_hdr_t *ab, *ab_prev;
927789Sahrens 	kmutex_t *hash_lock;
9282688Smaybee 	boolean_t have_lock;
9292688Smaybee 	void *steal = NULL;
930789Sahrens 
9311544Seschrock 	ASSERT(state == arc.mru || state == arc.mfu);
932789Sahrens 
9331544Seschrock 	evicted_state = (state == arc.mru) ? arc.mru_ghost : arc.mfu_ghost;
934789Sahrens 
935789Sahrens 	mutex_enter(&state->mtx);
936789Sahrens 	mutex_enter(&evicted_state->mtx);
937789Sahrens 
938789Sahrens 	for (ab = list_tail(&state->list); ab; ab = ab_prev) {
939789Sahrens 		ab_prev = list_prev(&state->list, ab);
9402391Smaybee 		/* prefetch buffers have a minimum lifespan */
9412688Smaybee 		if (HDR_IO_IN_PROGRESS(ab) ||
9422688Smaybee 		    (ab->b_flags & (ARC_PREFETCH|ARC_INDIRECT) &&
9432688Smaybee 		    lbolt - ab->b_arc_access < arc_min_prefetch_lifespan)) {
9442391Smaybee 			skipped++;
9452391Smaybee 			continue;
9462391Smaybee 		}
9472688Smaybee 		if (recycle && (ab->b_size != bytes || ab->b_datacnt > 1))
9482688Smaybee 			continue;
949789Sahrens 		hash_lock = HDR_LOCK(ab);
9502688Smaybee 		have_lock = MUTEX_HELD(hash_lock);
9512688Smaybee 		if (have_lock || mutex_tryenter(hash_lock)) {
952789Sahrens 			ASSERT3U(refcount_count(&ab->b_refcnt), ==, 0);
9531544Seschrock 			ASSERT(ab->b_datacnt > 0);
9541544Seschrock 			while (ab->b_buf) {
9551544Seschrock 				arc_buf_t *buf = ab->b_buf;
9562688Smaybee 				if (buf->b_data) {
9571544Seschrock 					bytes_evicted += ab->b_size;
9582688Smaybee 					if (recycle)
9592688Smaybee 						steal = buf->b_data;
9602688Smaybee 				}
9611544Seschrock 				if (buf->b_efunc) {
9621544Seschrock 					mutex_enter(&arc_eviction_mtx);
9632688Smaybee 					arc_buf_destroy(buf, recycle, FALSE);
9641544Seschrock 					ab->b_buf = buf->b_next;
9651544Seschrock 					buf->b_next = arc_eviction_list;
9661544Seschrock 					arc_eviction_list = buf;
9671544Seschrock 					mutex_exit(&arc_eviction_mtx);
9681544Seschrock 				} else {
9692688Smaybee 					arc_buf_destroy(buf, recycle, TRUE);
9701544Seschrock 				}
9711544Seschrock 			}
9721544Seschrock 			ASSERT(ab->b_datacnt == 0);
973789Sahrens 			arc_change_state(evicted_state, ab, hash_lock);
9741544Seschrock 			ASSERT(HDR_IN_HASH_TABLE(ab));
9751544Seschrock 			ab->b_flags = ARC_IN_HASH_TABLE;
976789Sahrens 			DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, ab);
9772688Smaybee 			if (!have_lock)
9782688Smaybee 				mutex_exit(hash_lock);
9791544Seschrock 			if (bytes >= 0 && bytes_evicted >= bytes)
980789Sahrens 				break;
981789Sahrens 		} else {
9822688Smaybee 			missed += 1;
983789Sahrens 		}
984789Sahrens 	}
985789Sahrens 	mutex_exit(&evicted_state->mtx);
986789Sahrens 	mutex_exit(&state->mtx);
987789Sahrens 
988789Sahrens 	if (bytes_evicted < bytes)
989789Sahrens 		dprintf("only evicted %lld bytes from %x",
990789Sahrens 		    (longlong_t)bytes_evicted, state);
991789Sahrens 
9922688Smaybee 	if (skipped)
9932688Smaybee 		atomic_add_64(&arc.evict_skip, skipped);
9942688Smaybee 	if (missed)
9952688Smaybee 		atomic_add_64(&arc.mutex_miss, missed);
9962688Smaybee 	return (steal);
997789Sahrens }
998789Sahrens 
999789Sahrens /*
1000789Sahrens  * Remove buffers from list until we've removed the specified number of
1001789Sahrens  * bytes.  Destroy the buffers that are removed.
1002789Sahrens  */
1003789Sahrens static void
10041544Seschrock arc_evict_ghost(arc_state_t *state, int64_t bytes)
1005789Sahrens {
1006789Sahrens 	arc_buf_hdr_t *ab, *ab_prev;
1007789Sahrens 	kmutex_t *hash_lock;
10081544Seschrock 	uint64_t bytes_deleted = 0;
10091544Seschrock 	uint_t bufs_skipped = 0;
1010789Sahrens 
10111544Seschrock 	ASSERT(GHOST_STATE(state));
1012789Sahrens top:
1013789Sahrens 	mutex_enter(&state->mtx);
1014789Sahrens 	for (ab = list_tail(&state->list); ab; ab = ab_prev) {
1015789Sahrens 		ab_prev = list_prev(&state->list, ab);
1016789Sahrens 		hash_lock = HDR_LOCK(ab);
1017789Sahrens 		if (mutex_tryenter(hash_lock)) {
10182391Smaybee 			ASSERT(!HDR_IO_IN_PROGRESS(ab));
10191544Seschrock 			ASSERT(ab->b_buf == NULL);
1020789Sahrens 			arc_change_state(arc.anon, ab, hash_lock);
1021789Sahrens 			mutex_exit(hash_lock);
1022789Sahrens 			atomic_add_64(&arc.deleted, 1);
10231544Seschrock 			bytes_deleted += ab->b_size;
10241544Seschrock 			arc_hdr_destroy(ab);
1025789Sahrens 			DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, ab);
1026789Sahrens 			if (bytes >= 0 && bytes_deleted >= bytes)
1027789Sahrens 				break;
1028789Sahrens 		} else {
1029789Sahrens 			if (bytes < 0) {
1030789Sahrens 				mutex_exit(&state->mtx);
1031789Sahrens 				mutex_enter(hash_lock);
1032789Sahrens 				mutex_exit(hash_lock);
1033789Sahrens 				goto top;
1034789Sahrens 			}
1035789Sahrens 			bufs_skipped += 1;
1036789Sahrens 		}
1037789Sahrens 	}
1038789Sahrens 	mutex_exit(&state->mtx);
1039789Sahrens 
1040789Sahrens 	if (bufs_skipped) {
10412688Smaybee 		atomic_add_64(&arc.mutex_miss, bufs_skipped);
1042789Sahrens 		ASSERT(bytes >= 0);
1043789Sahrens 	}
1044789Sahrens 
1045789Sahrens 	if (bytes_deleted < bytes)
1046789Sahrens 		dprintf("only deleted %lld bytes from %p",
1047789Sahrens 		    (longlong_t)bytes_deleted, state);
1048789Sahrens }
1049789Sahrens 
1050789Sahrens static void
1051789Sahrens arc_adjust(void)
1052789Sahrens {
1053789Sahrens 	int64_t top_sz, mru_over, arc_over;
1054789Sahrens 
10551544Seschrock 	top_sz = arc.anon->size + arc.mru->size;
1056789Sahrens 
10571544Seschrock 	if (top_sz > arc.p && arc.mru->lsize > 0) {
10581544Seschrock 		int64_t toevict = MIN(arc.mru->lsize, top_sz-arc.p);
10592688Smaybee 		(void) arc_evict(arc.mru, toevict, FALSE);
10601544Seschrock 		top_sz = arc.anon->size + arc.mru->size;
1061789Sahrens 	}
1062789Sahrens 
10631544Seschrock 	mru_over = top_sz + arc.mru_ghost->size - arc.c;
1064789Sahrens 
1065789Sahrens 	if (mru_over > 0) {
10661544Seschrock 		if (arc.mru_ghost->lsize > 0) {
10671544Seschrock 			int64_t todelete = MIN(arc.mru_ghost->lsize, mru_over);
10681544Seschrock 			arc_evict_ghost(arc.mru_ghost, todelete);
1069789Sahrens 		}
1070789Sahrens 	}
1071789Sahrens 
1072789Sahrens 	if ((arc_over = arc.size - arc.c) > 0) {
10731544Seschrock 		int64_t tbl_over;
1074789Sahrens 
10751544Seschrock 		if (arc.mfu->lsize > 0) {
10761544Seschrock 			int64_t toevict = MIN(arc.mfu->lsize, arc_over);
10772688Smaybee 			(void) arc_evict(arc.mfu, toevict, FALSE);
1078789Sahrens 		}
1079789Sahrens 
10801544Seschrock 		tbl_over = arc.size + arc.mru_ghost->lsize +
10811544Seschrock 		    arc.mfu_ghost->lsize - arc.c*2;
1082789Sahrens 
10831544Seschrock 		if (tbl_over > 0 && arc.mfu_ghost->lsize > 0) {
10841544Seschrock 			int64_t todelete = MIN(arc.mfu_ghost->lsize, tbl_over);
10851544Seschrock 			arc_evict_ghost(arc.mfu_ghost, todelete);
1086789Sahrens 		}
1087789Sahrens 	}
1088789Sahrens }
1089789Sahrens 
10901544Seschrock static void
10911544Seschrock arc_do_user_evicts(void)
10921544Seschrock {
10931544Seschrock 	mutex_enter(&arc_eviction_mtx);
10941544Seschrock 	while (arc_eviction_list != NULL) {
10951544Seschrock 		arc_buf_t *buf = arc_eviction_list;
10961544Seschrock 		arc_eviction_list = buf->b_next;
10971544Seschrock 		buf->b_hdr = NULL;
10981544Seschrock 		mutex_exit(&arc_eviction_mtx);
10991544Seschrock 
11001819Smaybee 		if (buf->b_efunc != NULL)
11011819Smaybee 			VERIFY(buf->b_efunc(buf) == 0);
11021544Seschrock 
11031544Seschrock 		buf->b_efunc = NULL;
11041544Seschrock 		buf->b_private = NULL;
11051544Seschrock 		kmem_cache_free(buf_cache, buf);
11061544Seschrock 		mutex_enter(&arc_eviction_mtx);
11071544Seschrock 	}
11081544Seschrock 	mutex_exit(&arc_eviction_mtx);
11091544Seschrock }
11101544Seschrock 
1111789Sahrens /*
1112789Sahrens  * Flush all *evictable* data from the cache.
1113789Sahrens  * NOTE: this will not touch "active" (i.e. referenced) data.
1114789Sahrens  */
1115789Sahrens void
1116789Sahrens arc_flush(void)
1117789Sahrens {
11182688Smaybee 	while (list_head(&arc.mru->list))
11192688Smaybee 		(void) arc_evict(arc.mru, -1, FALSE);
11202688Smaybee 	while (list_head(&arc.mfu->list))
11212688Smaybee 		(void) arc_evict(arc.mfu, -1, FALSE);
1122789Sahrens 
11231544Seschrock 	arc_evict_ghost(arc.mru_ghost, -1);
11241544Seschrock 	arc_evict_ghost(arc.mfu_ghost, -1);
11251544Seschrock 
11261544Seschrock 	mutex_enter(&arc_reclaim_thr_lock);
11271544Seschrock 	arc_do_user_evicts();
11281544Seschrock 	mutex_exit(&arc_reclaim_thr_lock);
11291544Seschrock 	ASSERT(arc_eviction_list == NULL);
1130789Sahrens }
1131789Sahrens 
11322391Smaybee int arc_kmem_reclaim_shift = 5;		/* log2(fraction of arc to reclaim) */
11332391Smaybee 
1134789Sahrens void
1135789Sahrens arc_kmem_reclaim(void)
1136789Sahrens {
11372048Sstans 	uint64_t to_free;
11382048Sstans 
1139789Sahrens 	/*
1140789Sahrens 	 * We need arc_reclaim_lock because we don't want multiple
1141789Sahrens 	 * threads trying to reclaim concurrently.
1142789Sahrens 	 */
1143789Sahrens 
1144789Sahrens 	/*
1145789Sahrens 	 * umem calls the reclaim func when we destroy the buf cache,
1146789Sahrens 	 * which is after we do arc_fini().  So we set a flag to prevent
1147789Sahrens 	 * accessing the destroyed mutexes and lists.
1148789Sahrens 	 */
1149789Sahrens 	if (arc_dead)
1150789Sahrens 		return;
1151789Sahrens 
11521544Seschrock 	if (arc.c <= arc.c_min)
11531544Seschrock 		return;
11541544Seschrock 
1155789Sahrens 	mutex_enter(&arc_reclaim_lock);
1156789Sahrens 
11572048Sstans #ifdef _KERNEL
11582391Smaybee 	to_free = MAX(arc.c >> arc_kmem_reclaim_shift, ptob(needfree));
11592048Sstans #else
11602391Smaybee 	to_free = arc.c >> arc_kmem_reclaim_shift;
11612048Sstans #endif
11622048Sstans 	if (arc.c > to_free)
11632048Sstans 		atomic_add_64(&arc.c, -to_free);
11642048Sstans 	else
11652048Sstans 		arc.c = arc.c_min;
11662048Sstans 
11672391Smaybee 	atomic_add_64(&arc.p, -(arc.p >> arc_kmem_reclaim_shift));
11681544Seschrock 	if (arc.c > arc.size)
11691544Seschrock 		arc.c = arc.size;
1170789Sahrens 	if (arc.c < arc.c_min)
1171789Sahrens 		arc.c = arc.c_min;
11721544Seschrock 	if (arc.p > arc.c)
11731544Seschrock 		arc.p = (arc.c >> 1);
11741544Seschrock 	ASSERT((int64_t)arc.p >= 0);
1175789Sahrens 
1176789Sahrens 	arc_adjust();
1177789Sahrens 
1178789Sahrens 	mutex_exit(&arc_reclaim_lock);
1179789Sahrens }
1180789Sahrens 
1181789Sahrens static int
1182789Sahrens arc_reclaim_needed(void)
1183789Sahrens {
1184789Sahrens 	uint64_t extra;
1185789Sahrens 
1186789Sahrens #ifdef _KERNEL
11872048Sstans 
11882048Sstans 	if (needfree)
11892048Sstans 		return (1);
11902048Sstans 
1191789Sahrens 	/*
1192789Sahrens 	 * take 'desfree' extra pages, so we reclaim sooner, rather than later
1193789Sahrens 	 */
1194789Sahrens 	extra = desfree;
1195789Sahrens 
1196789Sahrens 	/*
1197789Sahrens 	 * check that we're out of range of the pageout scanner.  It starts to
1198789Sahrens 	 * schedule paging if freemem is less than lotsfree and needfree.
1199789Sahrens 	 * lotsfree is the high-water mark for pageout, and needfree is the
1200789Sahrens 	 * number of needed free pages.  We add extra pages here to make sure
1201789Sahrens 	 * the scanner doesn't start up while we're freeing memory.
1202789Sahrens 	 */
1203789Sahrens 	if (freemem < lotsfree + needfree + extra)
1204789Sahrens 		return (1);
1205789Sahrens 
1206789Sahrens 	/*
1207789Sahrens 	 * check to make sure that swapfs has enough space so that anon
1208789Sahrens 	 * reservations can still succeeed. anon_resvmem() checks that the
1209789Sahrens 	 * availrmem is greater than swapfs_minfree, and the number of reserved
1210789Sahrens 	 * swap pages.  We also add a bit of extra here just to prevent
1211789Sahrens 	 * circumstances from getting really dire.
1212789Sahrens 	 */
1213789Sahrens 	if (availrmem < swapfs_minfree + swapfs_reserve + extra)
1214789Sahrens 		return (1);
1215789Sahrens 
12161936Smaybee #if defined(__i386)
1217789Sahrens 	/*
1218789Sahrens 	 * If we're on an i386 platform, it's possible that we'll exhaust the
1219789Sahrens 	 * kernel heap space before we ever run out of available physical
1220789Sahrens 	 * memory.  Most checks of the size of the heap_area compare against
1221789Sahrens 	 * tune.t_minarmem, which is the minimum available real memory that we
1222789Sahrens 	 * can have in the system.  However, this is generally fixed at 25 pages
1223789Sahrens 	 * which is so low that it's useless.  In this comparison, we seek to
1224789Sahrens 	 * calculate the total heap-size, and reclaim if more than 3/4ths of the
1225789Sahrens 	 * heap is allocated.  (Or, in the caclulation, if less than 1/4th is
1226789Sahrens 	 * free)
1227789Sahrens 	 */
1228789Sahrens 	if (btop(vmem_size(heap_arena, VMEM_FREE)) <
1229789Sahrens 	    (btop(vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC)) >> 2))
1230789Sahrens 		return (1);
1231789Sahrens #endif
1232789Sahrens 
1233789Sahrens #else
1234789Sahrens 	if (spa_get_random(100) == 0)
1235789Sahrens 		return (1);
1236789Sahrens #endif
1237789Sahrens 	return (0);
1238789Sahrens }
1239789Sahrens 
1240789Sahrens static void
1241789Sahrens arc_kmem_reap_now(arc_reclaim_strategy_t strat)
1242789Sahrens {
1243789Sahrens 	size_t			i;
1244789Sahrens 	kmem_cache_t		*prev_cache = NULL;
1245789Sahrens 	extern kmem_cache_t	*zio_buf_cache[];
1246789Sahrens 
12471484Sek110237 #ifdef _KERNEL
12481484Sek110237 	/*
12491484Sek110237 	 * First purge some DNLC entries, in case the DNLC is using
12501484Sek110237 	 * up too much memory.
12511484Sek110237 	 */
12521505Sek110237 	dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent);
12531936Smaybee 
12541936Smaybee #if defined(__i386)
12551936Smaybee 	/*
12561936Smaybee 	 * Reclaim unused memory from all kmem caches.
12571936Smaybee 	 */
12581936Smaybee 	kmem_reap();
12591936Smaybee #endif
12601484Sek110237 #endif
12611484Sek110237 
1262789Sahrens 	/*
12631544Seschrock 	 * An agressive reclamation will shrink the cache size as well as
12641544Seschrock 	 * reap free buffers from the arc kmem caches.
1265789Sahrens 	 */
1266789Sahrens 	if (strat == ARC_RECLAIM_AGGR)
12671544Seschrock 		arc_kmem_reclaim();
1268789Sahrens 
1269789Sahrens 	for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) {
1270789Sahrens 		if (zio_buf_cache[i] != prev_cache) {
1271789Sahrens 			prev_cache = zio_buf_cache[i];
1272789Sahrens 			kmem_cache_reap_now(zio_buf_cache[i]);
1273789Sahrens 		}
1274789Sahrens 	}
12751544Seschrock 	kmem_cache_reap_now(buf_cache);
12761544Seschrock 	kmem_cache_reap_now(hdr_cache);
1277789Sahrens }
1278789Sahrens 
1279789Sahrens static void
1280789Sahrens arc_reclaim_thread(void)
1281789Sahrens {
1282789Sahrens 	clock_t			growtime = 0;
1283789Sahrens 	arc_reclaim_strategy_t	last_reclaim = ARC_RECLAIM_CONS;
1284789Sahrens 	callb_cpr_t		cpr;
1285789Sahrens 
1286789Sahrens 	CALLB_CPR_INIT(&cpr, &arc_reclaim_thr_lock, callb_generic_cpr, FTAG);
1287789Sahrens 
1288789Sahrens 	mutex_enter(&arc_reclaim_thr_lock);
1289789Sahrens 	while (arc_thread_exit == 0) {
1290789Sahrens 		if (arc_reclaim_needed()) {
1291789Sahrens 
1292789Sahrens 			if (arc.no_grow) {
1293789Sahrens 				if (last_reclaim == ARC_RECLAIM_CONS) {
1294789Sahrens 					last_reclaim = ARC_RECLAIM_AGGR;
1295789Sahrens 				} else {
1296789Sahrens 					last_reclaim = ARC_RECLAIM_CONS;
1297789Sahrens 				}
1298789Sahrens 			} else {
1299789Sahrens 				arc.no_grow = TRUE;
1300789Sahrens 				last_reclaim = ARC_RECLAIM_AGGR;
1301789Sahrens 				membar_producer();
1302789Sahrens 			}
1303789Sahrens 
1304789Sahrens 			/* reset the growth delay for every reclaim */
1305789Sahrens 			growtime = lbolt + (arc_grow_retry * hz);
1306*2856Snd150628 			ASSERT(growtime > 0);
1307789Sahrens 
1308789Sahrens 			arc_kmem_reap_now(last_reclaim);
1309789Sahrens 
1310789Sahrens 		} else if ((growtime > 0) && ((growtime - lbolt) <= 0)) {
1311789Sahrens 			arc.no_grow = FALSE;
1312789Sahrens 		}
1313789Sahrens 
13141544Seschrock 		if (arc_eviction_list != NULL)
13151544Seschrock 			arc_do_user_evicts();
13161544Seschrock 
1317789Sahrens 		/* block until needed, or one second, whichever is shorter */
1318789Sahrens 		CALLB_CPR_SAFE_BEGIN(&cpr);
1319789Sahrens 		(void) cv_timedwait(&arc_reclaim_thr_cv,
1320789Sahrens 		    &arc_reclaim_thr_lock, (lbolt + hz));
1321789Sahrens 		CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_thr_lock);
1322789Sahrens 	}
1323789Sahrens 
1324789Sahrens 	arc_thread_exit = 0;
1325789Sahrens 	cv_broadcast(&arc_reclaim_thr_cv);
1326789Sahrens 	CALLB_CPR_EXIT(&cpr);		/* drops arc_reclaim_thr_lock */
1327789Sahrens 	thread_exit();
1328789Sahrens }
1329789Sahrens 
13301544Seschrock /*
13311544Seschrock  * Adapt arc info given the number of bytes we are trying to add and
13321544Seschrock  * the state that we are comming from.  This function is only called
13331544Seschrock  * when we are adding new content to the cache.
13341544Seschrock  */
1335789Sahrens static void
13361544Seschrock arc_adapt(int bytes, arc_state_t *state)
1337789Sahrens {
13381544Seschrock 	int mult;
13391544Seschrock 
13401544Seschrock 	ASSERT(bytes > 0);
1341789Sahrens 	/*
13421544Seschrock 	 * Adapt the target size of the MRU list:
13431544Seschrock 	 *	- if we just hit in the MRU ghost list, then increase
13441544Seschrock 	 *	  the target size of the MRU list.
13451544Seschrock 	 *	- if we just hit in the MFU ghost list, then increase
13461544Seschrock 	 *	  the target size of the MFU list by decreasing the
13471544Seschrock 	 *	  target size of the MRU list.
1348789Sahrens 	 */
13491544Seschrock 	if (state == arc.mru_ghost) {
13501544Seschrock 		mult = ((arc.mru_ghost->size >= arc.mfu_ghost->size) ?
13511544Seschrock 		    1 : (arc.mfu_ghost->size/arc.mru_ghost->size));
13521544Seschrock 
13531544Seschrock 		arc.p = MIN(arc.c, arc.p + bytes * mult);
13541544Seschrock 	} else if (state == arc.mfu_ghost) {
13551544Seschrock 		mult = ((arc.mfu_ghost->size >= arc.mru_ghost->size) ?
13561544Seschrock 		    1 : (arc.mru_ghost->size/arc.mfu_ghost->size));
13571544Seschrock 
13581544Seschrock 		arc.p = MAX(0, (int64_t)arc.p - bytes * mult);
13591544Seschrock 	}
13601544Seschrock 	ASSERT((int64_t)arc.p >= 0);
1361789Sahrens 
1362789Sahrens 	if (arc_reclaim_needed()) {
1363789Sahrens 		cv_signal(&arc_reclaim_thr_cv);
1364789Sahrens 		return;
1365789Sahrens 	}
1366789Sahrens 
1367789Sahrens 	if (arc.no_grow)
1368789Sahrens 		return;
1369789Sahrens 
13701544Seschrock 	if (arc.c >= arc.c_max)
13711544Seschrock 		return;
13721544Seschrock 
1373789Sahrens 	/*
13741544Seschrock 	 * If we're within (2 * maxblocksize) bytes of the target
13751544Seschrock 	 * cache size, increment the target cache size
1376789Sahrens 	 */
13771544Seschrock 	if (arc.size > arc.c - (2ULL << SPA_MAXBLOCKSHIFT)) {
13781544Seschrock 		atomic_add_64(&arc.c, (int64_t)bytes);
1379789Sahrens 		if (arc.c > arc.c_max)
1380789Sahrens 			arc.c = arc.c_max;
13811544Seschrock 		else if (state == arc.anon)
13821544Seschrock 			atomic_add_64(&arc.p, (int64_t)bytes);
13831544Seschrock 		if (arc.p > arc.c)
13841544Seschrock 			arc.p = arc.c;
1385789Sahrens 	}
13861544Seschrock 	ASSERT((int64_t)arc.p >= 0);
1387789Sahrens }
1388789Sahrens 
1389789Sahrens /*
13901544Seschrock  * Check if the cache has reached its limits and eviction is required
13911544Seschrock  * prior to insert.
1392789Sahrens  */
1393789Sahrens static int
1394789Sahrens arc_evict_needed()
1395789Sahrens {
1396789Sahrens 	if (arc_reclaim_needed())
1397789Sahrens 		return (1);
1398789Sahrens 
13991544Seschrock 	return (arc.size > arc.c);
1400789Sahrens }
1401789Sahrens 
1402789Sahrens /*
14032688Smaybee  * The buffer, supplied as the first argument, needs a data block.
14042688Smaybee  * So, if we are at cache max, determine which cache should be victimized.
14052688Smaybee  * We have the following cases:
1406789Sahrens  *
14071544Seschrock  * 1. Insert for MRU, p > sizeof(arc.anon + arc.mru) ->
1408789Sahrens  * In this situation if we're out of space, but the resident size of the MFU is
1409789Sahrens  * under the limit, victimize the MFU cache to satisfy this insertion request.
1410789Sahrens  *
14111544Seschrock  * 2. Insert for MRU, p <= sizeof(arc.anon + arc.mru) ->
1412789Sahrens  * Here, we've used up all of the available space for the MRU, so we need to
1413789Sahrens  * evict from our own cache instead.  Evict from the set of resident MRU
1414789Sahrens  * entries.
1415789Sahrens  *
14161544Seschrock  * 3. Insert for MFU (c - p) > sizeof(arc.mfu) ->
1417789Sahrens  * c minus p represents the MFU space in the cache, since p is the size of the
1418789Sahrens  * cache that is dedicated to the MRU.  In this situation there's still space on
1419789Sahrens  * the MFU side, so the MRU side needs to be victimized.
1420789Sahrens  *
14211544Seschrock  * 4. Insert for MFU (c - p) < sizeof(arc.mfu) ->
1422789Sahrens  * MFU's resident set is consuming more space than it has been allotted.  In
1423789Sahrens  * this situation, we must victimize our own cache, the MFU, for this insertion.
1424789Sahrens  */
1425789Sahrens static void
14262688Smaybee arc_get_data_buf(arc_buf_t *buf)
1427789Sahrens {
14282688Smaybee 	arc_state_t	*state = buf->b_hdr->b_state;
14292688Smaybee 	uint64_t	size = buf->b_hdr->b_size;
14302688Smaybee 
14312688Smaybee 	arc_adapt(size, state);
1432789Sahrens 
14332688Smaybee 	/*
14342688Smaybee 	 * We have not yet reached cache maximum size,
14352688Smaybee 	 * just allocate a new buffer.
14362688Smaybee 	 */
14372688Smaybee 	if (!arc_evict_needed()) {
14382688Smaybee 		buf->b_data = zio_buf_alloc(size);
14392688Smaybee 		atomic_add_64(&arc.size, size);
14402688Smaybee 		goto out;
14412688Smaybee 	}
14422688Smaybee 
14432688Smaybee 	/*
14442688Smaybee 	 * If we are prefetching from the mfu ghost list, this buffer
14452688Smaybee 	 * will end up on the mru list; so steal space from there.
14462688Smaybee 	 */
14472688Smaybee 	if (state == arc.mfu_ghost)
14482688Smaybee 		state = buf->b_hdr->b_flags & ARC_PREFETCH ? arc.mru : arc.mfu;
14492688Smaybee 	else if (state == arc.mru_ghost)
14502688Smaybee 		state = arc.mru;
1451789Sahrens 
14522688Smaybee 	if (state == arc.mru || state == arc.anon) {
14532688Smaybee 		uint64_t mru_used = arc.anon->size + arc.mru->size;
14542688Smaybee 		state = (arc.p > mru_used) ? arc.mfu : arc.mru;
1455789Sahrens 	} else {
14562688Smaybee 		/* MFU cases */
14572688Smaybee 		uint64_t mfu_space = arc.c - arc.p;
14582688Smaybee 		state =  (mfu_space > arc.mfu->size) ? arc.mru : arc.mfu;
14592688Smaybee 	}
14602688Smaybee 	if ((buf->b_data = arc_evict(state, size, TRUE)) == NULL) {
14612688Smaybee 		(void) arc_evict(state, size, FALSE);
14622688Smaybee 		buf->b_data = zio_buf_alloc(size);
14632688Smaybee 		atomic_add_64(&arc.size, size);
14642688Smaybee 		atomic_add_64(&arc.recycle_miss, 1);
14652688Smaybee 		if (arc.size > arc.c)
14662688Smaybee 			arc_adjust();
14672688Smaybee 	}
14682688Smaybee 	ASSERT(buf->b_data != NULL);
14692688Smaybee out:
14702688Smaybee 	/*
14712688Smaybee 	 * Update the state size.  Note that ghost states have a
14722688Smaybee 	 * "ghost size" and so don't need to be updated.
14732688Smaybee 	 */
14742688Smaybee 	if (!GHOST_STATE(buf->b_hdr->b_state)) {
14752688Smaybee 		arc_buf_hdr_t *hdr = buf->b_hdr;
14762688Smaybee 
14772688Smaybee 		atomic_add_64(&hdr->b_state->size, size);
14782688Smaybee 		if (list_link_active(&hdr->b_arc_node)) {
14792688Smaybee 			ASSERT(refcount_is_zero(&hdr->b_refcnt));
14802688Smaybee 			atomic_add_64(&hdr->b_state->lsize, size);
1481789Sahrens 		}
1482789Sahrens 	}
1483789Sahrens }
1484789Sahrens 
1485789Sahrens /*
1486789Sahrens  * This routine is called whenever a buffer is accessed.
14871544Seschrock  * NOTE: the hash lock is dropped in this function.
1488789Sahrens  */
1489789Sahrens static void
14902688Smaybee arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock)
1491789Sahrens {
1492789Sahrens 	ASSERT(MUTEX_HELD(hash_lock));
1493789Sahrens 
1494789Sahrens 	if (buf->b_state == arc.anon) {
1495789Sahrens 		/*
1496789Sahrens 		 * This buffer is not in the cache, and does not
1497789Sahrens 		 * appear in our "ghost" list.  Add the new buffer
1498789Sahrens 		 * to the MRU state.
1499789Sahrens 		 */
1500789Sahrens 
1501789Sahrens 		ASSERT(buf->b_arc_access == 0);
1502789Sahrens 		buf->b_arc_access = lbolt;
15031544Seschrock 		DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf);
15041544Seschrock 		arc_change_state(arc.mru, buf, hash_lock);
1505789Sahrens 
15061544Seschrock 	} else if (buf->b_state == arc.mru) {
1507789Sahrens 		/*
15082391Smaybee 		 * If this buffer is here because of a prefetch, then either:
15092391Smaybee 		 * - clear the flag if this is a "referencing" read
15102391Smaybee 		 *   (any subsequent access will bump this into the MFU state).
15112391Smaybee 		 * or
15122391Smaybee 		 * - move the buffer to the head of the list if this is
15132391Smaybee 		 *   another prefetch (to make it less likely to be evicted).
1514789Sahrens 		 */
1515789Sahrens 		if ((buf->b_flags & ARC_PREFETCH) != 0) {
15162391Smaybee 			if (refcount_count(&buf->b_refcnt) == 0) {
15172391Smaybee 				ASSERT(list_link_active(&buf->b_arc_node));
15182391Smaybee 				mutex_enter(&arc.mru->mtx);
15192391Smaybee 				list_remove(&arc.mru->list, buf);
15202391Smaybee 				list_insert_head(&arc.mru->list, buf);
15212391Smaybee 				mutex_exit(&arc.mru->mtx);
15222391Smaybee 			} else {
15232391Smaybee 				buf->b_flags &= ~ARC_PREFETCH;
15242391Smaybee 				atomic_add_64(&arc.mru->hits, 1);
15252391Smaybee 			}
15262391Smaybee 			buf->b_arc_access = lbolt;
1527789Sahrens 			return;
1528789Sahrens 		}
1529789Sahrens 
1530789Sahrens 		/*
1531789Sahrens 		 * This buffer has been "accessed" only once so far,
1532789Sahrens 		 * but it is still in the cache. Move it to the MFU
1533789Sahrens 		 * state.
1534789Sahrens 		 */
1535789Sahrens 		if (lbolt > buf->b_arc_access + ARC_MINTIME) {
1536789Sahrens 			/*
1537789Sahrens 			 * More than 125ms have passed since we
1538789Sahrens 			 * instantiated this buffer.  Move it to the
1539789Sahrens 			 * most frequently used state.
1540789Sahrens 			 */
1541789Sahrens 			buf->b_arc_access = lbolt;
15421544Seschrock 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
15431544Seschrock 			arc_change_state(arc.mfu, buf, hash_lock);
1544789Sahrens 		}
15451544Seschrock 		atomic_add_64(&arc.mru->hits, 1);
15461544Seschrock 	} else if (buf->b_state == arc.mru_ghost) {
1547789Sahrens 		arc_state_t	*new_state;
1548789Sahrens 		/*
1549789Sahrens 		 * This buffer has been "accessed" recently, but
1550789Sahrens 		 * was evicted from the cache.  Move it to the
1551789Sahrens 		 * MFU state.
1552789Sahrens 		 */
1553789Sahrens 
1554789Sahrens 		if (buf->b_flags & ARC_PREFETCH) {
15551544Seschrock 			new_state = arc.mru;
15562391Smaybee 			if (refcount_count(&buf->b_refcnt) > 0)
15572391Smaybee 				buf->b_flags &= ~ARC_PREFETCH;
15581544Seschrock 			DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf);
1559789Sahrens 		} else {
15601544Seschrock 			new_state = arc.mfu;
15611544Seschrock 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
1562789Sahrens 		}
1563789Sahrens 
1564789Sahrens 		buf->b_arc_access = lbolt;
1565789Sahrens 		arc_change_state(new_state, buf, hash_lock);
1566789Sahrens 
15671544Seschrock 		atomic_add_64(&arc.mru_ghost->hits, 1);
15681544Seschrock 	} else if (buf->b_state == arc.mfu) {
1569789Sahrens 		/*
1570789Sahrens 		 * This buffer has been accessed more than once and is
1571789Sahrens 		 * still in the cache.  Keep it in the MFU state.
1572789Sahrens 		 *
15732391Smaybee 		 * NOTE: an add_reference() that occurred when we did
15742391Smaybee 		 * the arc_read() will have kicked this off the list.
15752391Smaybee 		 * If it was a prefetch, we will explicitly move it to
15762391Smaybee 		 * the head of the list now.
1577789Sahrens 		 */
15782391Smaybee 		if ((buf->b_flags & ARC_PREFETCH) != 0) {
15792391Smaybee 			ASSERT(refcount_count(&buf->b_refcnt) == 0);
15802391Smaybee 			ASSERT(list_link_active(&buf->b_arc_node));
15812391Smaybee 			mutex_enter(&arc.mfu->mtx);
15822391Smaybee 			list_remove(&arc.mfu->list, buf);
15832391Smaybee 			list_insert_head(&arc.mfu->list, buf);
15842391Smaybee 			mutex_exit(&arc.mfu->mtx);
15852391Smaybee 		}
15861544Seschrock 		atomic_add_64(&arc.mfu->hits, 1);
15872391Smaybee 		buf->b_arc_access = lbolt;
15881544Seschrock 	} else if (buf->b_state == arc.mfu_ghost) {
15892391Smaybee 		arc_state_t	*new_state = arc.mfu;
1590789Sahrens 		/*
1591789Sahrens 		 * This buffer has been accessed more than once but has
1592789Sahrens 		 * been evicted from the cache.  Move it back to the
1593789Sahrens 		 * MFU state.
1594789Sahrens 		 */
1595789Sahrens 
15962391Smaybee 		if (buf->b_flags & ARC_PREFETCH) {
15972391Smaybee 			/*
15982391Smaybee 			 * This is a prefetch access...
15992391Smaybee 			 * move this block back to the MRU state.
16002391Smaybee 			 */
16012391Smaybee 			ASSERT3U(refcount_count(&buf->b_refcnt), ==, 0);
16022391Smaybee 			new_state = arc.mru;
16032391Smaybee 		}
16042391Smaybee 
1605789Sahrens 		buf->b_arc_access = lbolt;
16061544Seschrock 		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf);
16072391Smaybee 		arc_change_state(new_state, buf, hash_lock);
1608789Sahrens 
16091544Seschrock 		atomic_add_64(&arc.mfu_ghost->hits, 1);
1610789Sahrens 	} else {
1611789Sahrens 		ASSERT(!"invalid arc state");
1612789Sahrens 	}
1613789Sahrens }
1614789Sahrens 
1615789Sahrens /* a generic arc_done_func_t which you can use */
1616789Sahrens /* ARGSUSED */
1617789Sahrens void
1618789Sahrens arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg)
1619789Sahrens {
1620789Sahrens 	bcopy(buf->b_data, arg, buf->b_hdr->b_size);
16211544Seschrock 	VERIFY(arc_buf_remove_ref(buf, arg) == 1);
1622789Sahrens }
1623789Sahrens 
1624789Sahrens /* a generic arc_done_func_t which you can use */
1625789Sahrens void
1626789Sahrens arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg)
1627789Sahrens {
1628789Sahrens 	arc_buf_t **bufp = arg;
1629789Sahrens 	if (zio && zio->io_error) {
16301544Seschrock 		VERIFY(arc_buf_remove_ref(buf, arg) == 1);
1631789Sahrens 		*bufp = NULL;
1632789Sahrens 	} else {
1633789Sahrens 		*bufp = buf;
1634789Sahrens 	}
1635789Sahrens }
1636789Sahrens 
1637789Sahrens static void
1638789Sahrens arc_read_done(zio_t *zio)
1639789Sahrens {
16401589Smaybee 	arc_buf_hdr_t	*hdr, *found;
1641789Sahrens 	arc_buf_t	*buf;
1642789Sahrens 	arc_buf_t	*abuf;	/* buffer we're assigning to callback */
1643789Sahrens 	kmutex_t	*hash_lock;
1644789Sahrens 	arc_callback_t	*callback_list, *acb;
1645789Sahrens 	int		freeable = FALSE;
1646789Sahrens 
1647789Sahrens 	buf = zio->io_private;
1648789Sahrens 	hdr = buf->b_hdr;
1649789Sahrens 
16501589Smaybee 	/*
16511589Smaybee 	 * The hdr was inserted into hash-table and removed from lists
16521589Smaybee 	 * prior to starting I/O.  We should find this header, since
16531589Smaybee 	 * it's in the hash table, and it should be legit since it's
16541589Smaybee 	 * not possible to evict it during the I/O.  The only possible
16551589Smaybee 	 * reason for it not to be found is if we were freed during the
16561589Smaybee 	 * read.
16571589Smaybee 	 */
16581589Smaybee 	found = buf_hash_find(zio->io_spa, &hdr->b_dva, hdr->b_birth,
1659789Sahrens 		    &hash_lock);
1660789Sahrens 
16611589Smaybee 	ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && hash_lock == NULL) ||
16621589Smaybee 	    (found == hdr && DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))));
1663789Sahrens 
1664789Sahrens 	/* byteswap if necessary */
1665789Sahrens 	callback_list = hdr->b_acb;
1666789Sahrens 	ASSERT(callback_list != NULL);
1667789Sahrens 	if (BP_SHOULD_BYTESWAP(zio->io_bp) && callback_list->acb_byteswap)
1668789Sahrens 		callback_list->acb_byteswap(buf->b_data, hdr->b_size);
1669789Sahrens 
1670789Sahrens 	/* create copies of the data buffer for the callers */
1671789Sahrens 	abuf = buf;
1672789Sahrens 	for (acb = callback_list; acb; acb = acb->acb_next) {
1673789Sahrens 		if (acb->acb_done) {
16742688Smaybee 			if (abuf == NULL)
16752688Smaybee 				abuf = arc_buf_clone(buf);
1676789Sahrens 			acb->acb_buf = abuf;
1677789Sahrens 			abuf = NULL;
1678789Sahrens 		}
1679789Sahrens 	}
1680789Sahrens 	hdr->b_acb = NULL;
1681789Sahrens 	hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
16821544Seschrock 	ASSERT(!HDR_BUF_AVAILABLE(hdr));
16831544Seschrock 	if (abuf == buf)
16841544Seschrock 		hdr->b_flags |= ARC_BUF_AVAILABLE;
1685789Sahrens 
1686789Sahrens 	ASSERT(refcount_is_zero(&hdr->b_refcnt) || callback_list != NULL);
1687789Sahrens 
1688789Sahrens 	if (zio->io_error != 0) {
1689789Sahrens 		hdr->b_flags |= ARC_IO_ERROR;
1690789Sahrens 		if (hdr->b_state != arc.anon)
1691789Sahrens 			arc_change_state(arc.anon, hdr, hash_lock);
16921544Seschrock 		if (HDR_IN_HASH_TABLE(hdr))
16931544Seschrock 			buf_hash_remove(hdr);
1694789Sahrens 		freeable = refcount_is_zero(&hdr->b_refcnt);
16952391Smaybee 		/* convert checksum errors into IO errors */
16961544Seschrock 		if (zio->io_error == ECKSUM)
16971544Seschrock 			zio->io_error = EIO;
1698789Sahrens 	}
1699789Sahrens 
17001544Seschrock 	/*
17012391Smaybee 	 * Broadcast before we drop the hash_lock to avoid the possibility
17022391Smaybee 	 * that the hdr (and hence the cv) might be freed before we get to
17032391Smaybee 	 * the cv_broadcast().
17041544Seschrock 	 */
17051544Seschrock 	cv_broadcast(&hdr->b_cv);
17061544Seschrock 
17071589Smaybee 	if (hash_lock) {
1708789Sahrens 		/*
1709789Sahrens 		 * Only call arc_access on anonymous buffers.  This is because
1710789Sahrens 		 * if we've issued an I/O for an evicted buffer, we've already
1711789Sahrens 		 * called arc_access (to prevent any simultaneous readers from
1712789Sahrens 		 * getting confused).
1713789Sahrens 		 */
1714789Sahrens 		if (zio->io_error == 0 && hdr->b_state == arc.anon)
17152688Smaybee 			arc_access(hdr, hash_lock);
17162688Smaybee 		mutex_exit(hash_lock);
1717789Sahrens 	} else {
1718789Sahrens 		/*
1719789Sahrens 		 * This block was freed while we waited for the read to
1720789Sahrens 		 * complete.  It has been removed from the hash table and
1721789Sahrens 		 * moved to the anonymous state (so that it won't show up
1722789Sahrens 		 * in the cache).
1723789Sahrens 		 */
1724789Sahrens 		ASSERT3P(hdr->b_state, ==, arc.anon);
1725789Sahrens 		freeable = refcount_is_zero(&hdr->b_refcnt);
1726789Sahrens 	}
1727789Sahrens 
1728789Sahrens 	/* execute each callback and free its structure */
1729789Sahrens 	while ((acb = callback_list) != NULL) {
1730789Sahrens 		if (acb->acb_done)
1731789Sahrens 			acb->acb_done(zio, acb->acb_buf, acb->acb_private);
1732789Sahrens 
1733789Sahrens 		if (acb->acb_zio_dummy != NULL) {
1734789Sahrens 			acb->acb_zio_dummy->io_error = zio->io_error;
1735789Sahrens 			zio_nowait(acb->acb_zio_dummy);
1736789Sahrens 		}
1737789Sahrens 
1738789Sahrens 		callback_list = acb->acb_next;
1739789Sahrens 		kmem_free(acb, sizeof (arc_callback_t));
1740789Sahrens 	}
1741789Sahrens 
1742789Sahrens 	if (freeable)
17431544Seschrock 		arc_hdr_destroy(hdr);
1744789Sahrens }
1745789Sahrens 
1746789Sahrens /*
1747789Sahrens  * "Read" the block block at the specified DVA (in bp) via the
1748789Sahrens  * cache.  If the block is found in the cache, invoke the provided
1749789Sahrens  * callback immediately and return.  Note that the `zio' parameter
1750789Sahrens  * in the callback will be NULL in this case, since no IO was
1751789Sahrens  * required.  If the block is not in the cache pass the read request
1752789Sahrens  * on to the spa with a substitute callback function, so that the
1753789Sahrens  * requested block will be added to the cache.
1754789Sahrens  *
1755789Sahrens  * If a read request arrives for a block that has a read in-progress,
1756789Sahrens  * either wait for the in-progress read to complete (and return the
1757789Sahrens  * results); or, if this is a read with a "done" func, add a record
1758789Sahrens  * to the read to invoke the "done" func when the read completes,
1759789Sahrens  * and return; or just return.
1760789Sahrens  *
1761789Sahrens  * arc_read_done() will invoke all the requested "done" functions
1762789Sahrens  * for readers of this block.
1763789Sahrens  */
1764789Sahrens int
1765789Sahrens arc_read(zio_t *pio, spa_t *spa, blkptr_t *bp, arc_byteswap_func_t *swap,
1766789Sahrens     arc_done_func_t *done, void *private, int priority, int flags,
17672391Smaybee     uint32_t *arc_flags, zbookmark_t *zb)
1768789Sahrens {
1769789Sahrens 	arc_buf_hdr_t *hdr;
1770789Sahrens 	arc_buf_t *buf;
1771789Sahrens 	kmutex_t *hash_lock;
1772789Sahrens 	zio_t	*rzio;
1773789Sahrens 
1774789Sahrens top:
1775789Sahrens 	hdr = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_lock);
17761544Seschrock 	if (hdr && hdr->b_datacnt > 0) {
1777789Sahrens 
17782391Smaybee 		*arc_flags |= ARC_CACHED;
17792391Smaybee 
1780789Sahrens 		if (HDR_IO_IN_PROGRESS(hdr)) {
17812391Smaybee 
17822391Smaybee 			if (*arc_flags & ARC_WAIT) {
17832391Smaybee 				cv_wait(&hdr->b_cv, hash_lock);
17842391Smaybee 				mutex_exit(hash_lock);
17852391Smaybee 				goto top;
17862391Smaybee 			}
17872391Smaybee 			ASSERT(*arc_flags & ARC_NOWAIT);
17882391Smaybee 
17892391Smaybee 			if (done) {
1790789Sahrens 				arc_callback_t	*acb = NULL;
1791789Sahrens 
1792789Sahrens 				acb = kmem_zalloc(sizeof (arc_callback_t),
1793789Sahrens 				    KM_SLEEP);
1794789Sahrens 				acb->acb_done = done;
1795789Sahrens 				acb->acb_private = private;
1796789Sahrens 				acb->acb_byteswap = swap;
1797789Sahrens 				if (pio != NULL)
1798789Sahrens 					acb->acb_zio_dummy = zio_null(pio,
1799789Sahrens 					    spa, NULL, NULL, flags);
1800789Sahrens 
1801789Sahrens 				ASSERT(acb->acb_done != NULL);
1802789Sahrens 				acb->acb_next = hdr->b_acb;
1803789Sahrens 				hdr->b_acb = acb;
1804789Sahrens 				add_reference(hdr, hash_lock, private);
1805789Sahrens 				mutex_exit(hash_lock);
1806789Sahrens 				return (0);
1807789Sahrens 			}
1808789Sahrens 			mutex_exit(hash_lock);
1809789Sahrens 			return (0);
1810789Sahrens 		}
1811789Sahrens 
18121544Seschrock 		ASSERT(hdr->b_state == arc.mru || hdr->b_state == arc.mfu);
1813789Sahrens 
18141544Seschrock 		if (done) {
18152688Smaybee 			add_reference(hdr, hash_lock, private);
18161544Seschrock 			/*
18171544Seschrock 			 * If this block is already in use, create a new
18181544Seschrock 			 * copy of the data so that we will be guaranteed
18191544Seschrock 			 * that arc_release() will always succeed.
18201544Seschrock 			 */
18211544Seschrock 			buf = hdr->b_buf;
18221544Seschrock 			ASSERT(buf);
18231544Seschrock 			ASSERT(buf->b_data);
18242688Smaybee 			if (HDR_BUF_AVAILABLE(hdr)) {
18251544Seschrock 				ASSERT(buf->b_efunc == NULL);
18261544Seschrock 				hdr->b_flags &= ~ARC_BUF_AVAILABLE;
18272688Smaybee 			} else {
18282688Smaybee 				buf = arc_buf_clone(buf);
18291544Seschrock 			}
18302391Smaybee 		} else if (*arc_flags & ARC_PREFETCH &&
18312391Smaybee 		    refcount_count(&hdr->b_refcnt) == 0) {
18322391Smaybee 			hdr->b_flags |= ARC_PREFETCH;
1833789Sahrens 		}
1834789Sahrens 		DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
18352688Smaybee 		arc_access(hdr, hash_lock);
18362688Smaybee 		mutex_exit(hash_lock);
1837789Sahrens 		atomic_add_64(&arc.hits, 1);
1838789Sahrens 		if (done)
1839789Sahrens 			done(NULL, buf, private);
1840789Sahrens 	} else {
1841789Sahrens 		uint64_t size = BP_GET_LSIZE(bp);
1842789Sahrens 		arc_callback_t	*acb;
1843789Sahrens 
1844789Sahrens 		if (hdr == NULL) {
1845789Sahrens 			/* this block is not in the cache */
1846789Sahrens 			arc_buf_hdr_t	*exists;
1847789Sahrens 
1848789Sahrens 			buf = arc_buf_alloc(spa, size, private);
1849789Sahrens 			hdr = buf->b_hdr;
1850789Sahrens 			hdr->b_dva = *BP_IDENTITY(bp);
1851789Sahrens 			hdr->b_birth = bp->blk_birth;
1852789Sahrens 			hdr->b_cksum0 = bp->blk_cksum.zc_word[0];
1853789Sahrens 			exists = buf_hash_insert(hdr, &hash_lock);
1854789Sahrens 			if (exists) {
1855789Sahrens 				/* somebody beat us to the hash insert */
1856789Sahrens 				mutex_exit(hash_lock);
1857789Sahrens 				bzero(&hdr->b_dva, sizeof (dva_t));
1858789Sahrens 				hdr->b_birth = 0;
1859789Sahrens 				hdr->b_cksum0 = 0;
18601544Seschrock 				(void) arc_buf_remove_ref(buf, private);
1861789Sahrens 				goto top; /* restart the IO request */
1862789Sahrens 			}
18632391Smaybee 			/* if this is a prefetch, we don't have a reference */
18642391Smaybee 			if (*arc_flags & ARC_PREFETCH) {
18652391Smaybee 				(void) remove_reference(hdr, hash_lock,
18662391Smaybee 				    private);
18672391Smaybee 				hdr->b_flags |= ARC_PREFETCH;
18682391Smaybee 			}
18692391Smaybee 			if (BP_GET_LEVEL(bp) > 0)
18702391Smaybee 				hdr->b_flags |= ARC_INDIRECT;
1871789Sahrens 		} else {
1872789Sahrens 			/* this block is in the ghost cache */
18731544Seschrock 			ASSERT(GHOST_STATE(hdr->b_state));
18741544Seschrock 			ASSERT(!HDR_IO_IN_PROGRESS(hdr));
18752391Smaybee 			ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 0);
18762391Smaybee 			ASSERT(hdr->b_buf == NULL);
1877789Sahrens 
18782391Smaybee 			/* if this is a prefetch, we don't have a reference */
18792391Smaybee 			if (*arc_flags & ARC_PREFETCH)
18802391Smaybee 				hdr->b_flags |= ARC_PREFETCH;
18812391Smaybee 			else
18822391Smaybee 				add_reference(hdr, hash_lock, private);
1883789Sahrens 			buf = kmem_cache_alloc(buf_cache, KM_SLEEP);
18841544Seschrock 			buf->b_hdr = hdr;
18852688Smaybee 			buf->b_data = NULL;
18861544Seschrock 			buf->b_efunc = NULL;
18871544Seschrock 			buf->b_private = NULL;
18881544Seschrock 			buf->b_next = NULL;
18891544Seschrock 			hdr->b_buf = buf;
18902688Smaybee 			arc_get_data_buf(buf);
18911544Seschrock 			ASSERT(hdr->b_datacnt == 0);
18921544Seschrock 			hdr->b_datacnt = 1;
18932391Smaybee 
1894789Sahrens 		}
1895789Sahrens 
1896789Sahrens 		acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
1897789Sahrens 		acb->acb_done = done;
1898789Sahrens 		acb->acb_private = private;
1899789Sahrens 		acb->acb_byteswap = swap;
1900789Sahrens 
1901789Sahrens 		ASSERT(hdr->b_acb == NULL);
1902789Sahrens 		hdr->b_acb = acb;
1903789Sahrens 		hdr->b_flags |= ARC_IO_IN_PROGRESS;
1904789Sahrens 
1905789Sahrens 		/*
1906789Sahrens 		 * If the buffer has been evicted, migrate it to a present state
1907789Sahrens 		 * before issuing the I/O.  Once we drop the hash-table lock,
1908789Sahrens 		 * the header will be marked as I/O in progress and have an
1909789Sahrens 		 * attached buffer.  At this point, anybody who finds this
1910789Sahrens 		 * buffer ought to notice that it's legit but has a pending I/O.
1911789Sahrens 		 */
1912789Sahrens 
19131544Seschrock 		if (GHOST_STATE(hdr->b_state))
19142688Smaybee 			arc_access(hdr, hash_lock);
19152688Smaybee 		mutex_exit(hash_lock);
1916789Sahrens 
1917789Sahrens 		ASSERT3U(hdr->b_size, ==, size);
19181596Sahrens 		DTRACE_PROBE3(arc__miss, blkptr_t *, bp, uint64_t, size,
19191596Sahrens 		    zbookmark_t *, zb);
1920789Sahrens 		atomic_add_64(&arc.misses, 1);
19211544Seschrock 
1922789Sahrens 		rzio = zio_read(pio, spa, bp, buf->b_data, size,
19231544Seschrock 		    arc_read_done, buf, priority, flags, zb);
1924789Sahrens 
19252391Smaybee 		if (*arc_flags & ARC_WAIT)
1926789Sahrens 			return (zio_wait(rzio));
1927789Sahrens 
19282391Smaybee 		ASSERT(*arc_flags & ARC_NOWAIT);
1929789Sahrens 		zio_nowait(rzio);
1930789Sahrens 	}
1931789Sahrens 	return (0);
1932789Sahrens }
1933789Sahrens 
1934789Sahrens /*
1935789Sahrens  * arc_read() variant to support pool traversal.  If the block is already
1936789Sahrens  * in the ARC, make a copy of it; otherwise, the caller will do the I/O.
1937789Sahrens  * The idea is that we don't want pool traversal filling up memory, but
1938789Sahrens  * if the ARC already has the data anyway, we shouldn't pay for the I/O.
1939789Sahrens  */
1940789Sahrens int
1941789Sahrens arc_tryread(spa_t *spa, blkptr_t *bp, void *data)
1942789Sahrens {
1943789Sahrens 	arc_buf_hdr_t *hdr;
1944789Sahrens 	kmutex_t *hash_mtx;
1945789Sahrens 	int rc = 0;
1946789Sahrens 
1947789Sahrens 	hdr = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_mtx);
1948789Sahrens 
19491544Seschrock 	if (hdr && hdr->b_datacnt > 0 && !HDR_IO_IN_PROGRESS(hdr)) {
19501544Seschrock 		arc_buf_t *buf = hdr->b_buf;
19511544Seschrock 
19521544Seschrock 		ASSERT(buf);
19531544Seschrock 		while (buf->b_data == NULL) {
19541544Seschrock 			buf = buf->b_next;
19551544Seschrock 			ASSERT(buf);
19561544Seschrock 		}
19571544Seschrock 		bcopy(buf->b_data, data, hdr->b_size);
19581544Seschrock 	} else {
1959789Sahrens 		rc = ENOENT;
19601544Seschrock 	}
1961789Sahrens 
1962789Sahrens 	if (hash_mtx)
1963789Sahrens 		mutex_exit(hash_mtx);
1964789Sahrens 
1965789Sahrens 	return (rc);
1966789Sahrens }
1967789Sahrens 
19681544Seschrock void
19691544Seschrock arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private)
19701544Seschrock {
19711544Seschrock 	ASSERT(buf->b_hdr != NULL);
19721544Seschrock 	ASSERT(buf->b_hdr->b_state != arc.anon);
19731544Seschrock 	ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt) || func == NULL);
19741544Seschrock 	buf->b_efunc = func;
19751544Seschrock 	buf->b_private = private;
19761544Seschrock }
19771544Seschrock 
19781544Seschrock /*
19791544Seschrock  * This is used by the DMU to let the ARC know that a buffer is
19801544Seschrock  * being evicted, so the ARC should clean up.  If this arc buf
19811544Seschrock  * is not yet in the evicted state, it will be put there.
19821544Seschrock  */
19831544Seschrock int
19841544Seschrock arc_buf_evict(arc_buf_t *buf)
19851544Seschrock {
19862724Smaybee 	arc_buf_hdr_t *hdr = buf->b_hdr;
19871544Seschrock 	kmutex_t *hash_lock;
19881544Seschrock 	arc_buf_t **bufp;
19891544Seschrock 
19901544Seschrock 	if (hdr == NULL) {
19911544Seschrock 		/*
19921544Seschrock 		 * We are in arc_do_user_evicts().
19931544Seschrock 		 */
19941544Seschrock 		ASSERT(buf->b_data == NULL);
19951544Seschrock 		return (0);
19961544Seschrock 	}
19971544Seschrock 
19981544Seschrock 	hash_lock = HDR_LOCK(hdr);
19991544Seschrock 	mutex_enter(hash_lock);
20001544Seschrock 
20012724Smaybee 	if (buf->b_data == NULL) {
20022724Smaybee 		/*
20032724Smaybee 		 * We are on the eviction list.
20042724Smaybee 		 */
20052724Smaybee 		mutex_exit(hash_lock);
20062724Smaybee 		mutex_enter(&arc_eviction_mtx);
20072724Smaybee 		if (buf->b_hdr == NULL) {
20082724Smaybee 			/*
20092724Smaybee 			 * We are already in arc_do_user_evicts().
20102724Smaybee 			 */
20112724Smaybee 			mutex_exit(&arc_eviction_mtx);
20122724Smaybee 			return (0);
20132724Smaybee 		} else {
20142724Smaybee 			arc_buf_t copy = *buf; /* structure assignment */
20152724Smaybee 			/*
20162724Smaybee 			 * Process this buffer now
20172724Smaybee 			 * but let arc_do_user_evicts() do the reaping.
20182724Smaybee 			 */
20192724Smaybee 			buf->b_efunc = NULL;
20202724Smaybee 			mutex_exit(&arc_eviction_mtx);
20212724Smaybee 			VERIFY(copy.b_efunc(&copy) == 0);
20222724Smaybee 			return (1);
20232724Smaybee 		}
20242724Smaybee 	}
20252724Smaybee 
20262724Smaybee 	ASSERT(buf->b_hdr == hdr);
20272724Smaybee 	ASSERT3U(refcount_count(&hdr->b_refcnt), <, hdr->b_datacnt);
20281544Seschrock 	ASSERT(hdr->b_state == arc.mru || hdr->b_state == arc.mfu);
20291544Seschrock 
20301544Seschrock 	/*
20311544Seschrock 	 * Pull this buffer off of the hdr
20321544Seschrock 	 */
20331544Seschrock 	bufp = &hdr->b_buf;
20341544Seschrock 	while (*bufp != buf)
20351544Seschrock 		bufp = &(*bufp)->b_next;
20361544Seschrock 	*bufp = buf->b_next;
20371544Seschrock 
20381544Seschrock 	ASSERT(buf->b_data != NULL);
20391544Seschrock 	buf->b_hdr = hdr;
20402688Smaybee 	arc_buf_destroy(buf, FALSE, FALSE);
20411544Seschrock 
20421544Seschrock 	if (hdr->b_datacnt == 0) {
20431544Seschrock 		arc_state_t *old_state = hdr->b_state;
20441544Seschrock 		arc_state_t *evicted_state;
20451544Seschrock 
20461544Seschrock 		ASSERT(refcount_is_zero(&hdr->b_refcnt));
20471544Seschrock 
20481544Seschrock 		evicted_state =
20491544Seschrock 		    (old_state == arc.mru) ? arc.mru_ghost : arc.mfu_ghost;
20501544Seschrock 
20511544Seschrock 		mutex_enter(&old_state->mtx);
20521544Seschrock 		mutex_enter(&evicted_state->mtx);
20531544Seschrock 
20541544Seschrock 		arc_change_state(evicted_state, hdr, hash_lock);
20551544Seschrock 		ASSERT(HDR_IN_HASH_TABLE(hdr));
20561544Seschrock 		hdr->b_flags = ARC_IN_HASH_TABLE;
20571544Seschrock 
20581544Seschrock 		mutex_exit(&evicted_state->mtx);
20591544Seschrock 		mutex_exit(&old_state->mtx);
20601544Seschrock 	}
20611544Seschrock 	mutex_exit(hash_lock);
20621819Smaybee 
20631544Seschrock 	VERIFY(buf->b_efunc(buf) == 0);
20641544Seschrock 	buf->b_efunc = NULL;
20651544Seschrock 	buf->b_private = NULL;
20661544Seschrock 	buf->b_hdr = NULL;
20671544Seschrock 	kmem_cache_free(buf_cache, buf);
20681544Seschrock 	return (1);
20691544Seschrock }
20701544Seschrock 
2071789Sahrens /*
2072789Sahrens  * Release this buffer from the cache.  This must be done
2073789Sahrens  * after a read and prior to modifying the buffer contents.
2074789Sahrens  * If the buffer has more than one reference, we must make
2075789Sahrens  * make a new hdr for the buffer.
2076789Sahrens  */
2077789Sahrens void
2078789Sahrens arc_release(arc_buf_t *buf, void *tag)
2079789Sahrens {
2080789Sahrens 	arc_buf_hdr_t *hdr = buf->b_hdr;
2081789Sahrens 	kmutex_t *hash_lock = HDR_LOCK(hdr);
2082789Sahrens 
2083789Sahrens 	/* this buffer is not on any list */
2084789Sahrens 	ASSERT(refcount_count(&hdr->b_refcnt) > 0);
2085789Sahrens 
2086789Sahrens 	if (hdr->b_state == arc.anon) {
2087789Sahrens 		/* this buffer is already released */
2088789Sahrens 		ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 1);
2089789Sahrens 		ASSERT(BUF_EMPTY(hdr));
20901544Seschrock 		ASSERT(buf->b_efunc == NULL);
2091789Sahrens 		return;
2092789Sahrens 	}
2093789Sahrens 
2094789Sahrens 	mutex_enter(hash_lock);
2095789Sahrens 
20961544Seschrock 	/*
20971544Seschrock 	 * Do we have more than one buf?
20981544Seschrock 	 */
20991544Seschrock 	if (hdr->b_buf != buf || buf->b_next != NULL) {
2100789Sahrens 		arc_buf_hdr_t *nhdr;
2101789Sahrens 		arc_buf_t **bufp;
2102789Sahrens 		uint64_t blksz = hdr->b_size;
2103789Sahrens 		spa_t *spa = hdr->b_spa;
2104789Sahrens 
21051544Seschrock 		ASSERT(hdr->b_datacnt > 1);
2106789Sahrens 		/*
2107789Sahrens 		 * Pull the data off of this buf and attach it to
2108789Sahrens 		 * a new anonymous buf.
2109789Sahrens 		 */
21101544Seschrock 		(void) remove_reference(hdr, hash_lock, tag);
2111789Sahrens 		bufp = &hdr->b_buf;
21121544Seschrock 		while (*bufp != buf)
2113789Sahrens 			bufp = &(*bufp)->b_next;
2114789Sahrens 		*bufp = (*bufp)->b_next;
21151544Seschrock 
2116789Sahrens 		ASSERT3U(hdr->b_state->size, >=, hdr->b_size);
2117789Sahrens 		atomic_add_64(&hdr->b_state->size, -hdr->b_size);
21181544Seschrock 		if (refcount_is_zero(&hdr->b_refcnt)) {
21191544Seschrock 			ASSERT3U(hdr->b_state->lsize, >=, hdr->b_size);
21201544Seschrock 			atomic_add_64(&hdr->b_state->lsize, -hdr->b_size);
21211544Seschrock 		}
21221544Seschrock 		hdr->b_datacnt -= 1;
21231544Seschrock 
2124789Sahrens 		mutex_exit(hash_lock);
2125789Sahrens 
2126789Sahrens 		nhdr = kmem_cache_alloc(hdr_cache, KM_SLEEP);
2127789Sahrens 		nhdr->b_size = blksz;
2128789Sahrens 		nhdr->b_spa = spa;
2129789Sahrens 		nhdr->b_buf = buf;
2130789Sahrens 		nhdr->b_state = arc.anon;
2131789Sahrens 		nhdr->b_arc_access = 0;
2132789Sahrens 		nhdr->b_flags = 0;
21331544Seschrock 		nhdr->b_datacnt = 1;
2134789Sahrens 		buf->b_hdr = nhdr;
2135789Sahrens 		buf->b_next = NULL;
2136789Sahrens 		(void) refcount_add(&nhdr->b_refcnt, tag);
2137789Sahrens 		atomic_add_64(&arc.anon->size, blksz);
2138789Sahrens 
2139789Sahrens 		hdr = nhdr;
2140789Sahrens 	} else {
21411544Seschrock 		ASSERT(refcount_count(&hdr->b_refcnt) == 1);
2142789Sahrens 		ASSERT(!list_link_active(&hdr->b_arc_node));
2143789Sahrens 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
2144789Sahrens 		arc_change_state(arc.anon, hdr, hash_lock);
2145789Sahrens 		hdr->b_arc_access = 0;
2146789Sahrens 		mutex_exit(hash_lock);
2147789Sahrens 		bzero(&hdr->b_dva, sizeof (dva_t));
2148789Sahrens 		hdr->b_birth = 0;
2149789Sahrens 		hdr->b_cksum0 = 0;
2150789Sahrens 	}
21511544Seschrock 	buf->b_efunc = NULL;
21521544Seschrock 	buf->b_private = NULL;
2153789Sahrens }
2154789Sahrens 
2155789Sahrens int
2156789Sahrens arc_released(arc_buf_t *buf)
2157789Sahrens {
21581544Seschrock 	return (buf->b_data != NULL && buf->b_hdr->b_state == arc.anon);
21591544Seschrock }
21601544Seschrock 
21611544Seschrock int
21621544Seschrock arc_has_callback(arc_buf_t *buf)
21631544Seschrock {
21641544Seschrock 	return (buf->b_efunc != NULL);
2165789Sahrens }
2166789Sahrens 
21671544Seschrock #ifdef ZFS_DEBUG
21681544Seschrock int
21691544Seschrock arc_referenced(arc_buf_t *buf)
21701544Seschrock {
21711544Seschrock 	return (refcount_count(&buf->b_hdr->b_refcnt));
21721544Seschrock }
21731544Seschrock #endif
21741544Seschrock 
2175789Sahrens static void
2176789Sahrens arc_write_done(zio_t *zio)
2177789Sahrens {
2178789Sahrens 	arc_buf_t *buf;
2179789Sahrens 	arc_buf_hdr_t *hdr;
2180789Sahrens 	arc_callback_t *acb;
2181789Sahrens 
2182789Sahrens 	buf = zio->io_private;
2183789Sahrens 	hdr = buf->b_hdr;
2184789Sahrens 	acb = hdr->b_acb;
2185789Sahrens 	hdr->b_acb = NULL;
21861544Seschrock 	ASSERT(acb != NULL);
2187789Sahrens 
2188789Sahrens 	/* this buffer is on no lists and is not in the hash table */
2189789Sahrens 	ASSERT3P(hdr->b_state, ==, arc.anon);
2190789Sahrens 
2191789Sahrens 	hdr->b_dva = *BP_IDENTITY(zio->io_bp);
2192789Sahrens 	hdr->b_birth = zio->io_bp->blk_birth;
2193789Sahrens 	hdr->b_cksum0 = zio->io_bp->blk_cksum.zc_word[0];
21941544Seschrock 	/*
21951544Seschrock 	 * If the block to be written was all-zero, we may have
21961544Seschrock 	 * compressed it away.  In this case no write was performed
21971544Seschrock 	 * so there will be no dva/birth-date/checksum.  The buffer
21981544Seschrock 	 * must therefor remain anonymous (and uncached).
21991544Seschrock 	 */
2200789Sahrens 	if (!BUF_EMPTY(hdr)) {
2201789Sahrens 		arc_buf_hdr_t *exists;
2202789Sahrens 		kmutex_t *hash_lock;
2203789Sahrens 
2204789Sahrens 		exists = buf_hash_insert(hdr, &hash_lock);
2205789Sahrens 		if (exists) {
2206789Sahrens 			/*
2207789Sahrens 			 * This can only happen if we overwrite for
2208789Sahrens 			 * sync-to-convergence, because we remove
2209789Sahrens 			 * buffers from the hash table when we arc_free().
2210789Sahrens 			 */
2211789Sahrens 			ASSERT(DVA_EQUAL(BP_IDENTITY(&zio->io_bp_orig),
2212789Sahrens 			    BP_IDENTITY(zio->io_bp)));
2213789Sahrens 			ASSERT3U(zio->io_bp_orig.blk_birth, ==,
2214789Sahrens 			    zio->io_bp->blk_birth);
2215789Sahrens 
2216789Sahrens 			ASSERT(refcount_is_zero(&exists->b_refcnt));
2217789Sahrens 			arc_change_state(arc.anon, exists, hash_lock);
2218789Sahrens 			mutex_exit(hash_lock);
22191544Seschrock 			arc_hdr_destroy(exists);
2220789Sahrens 			exists = buf_hash_insert(hdr, &hash_lock);
2221789Sahrens 			ASSERT3P(exists, ==, NULL);
2222789Sahrens 		}
22231544Seschrock 		hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
22242688Smaybee 		arc_access(hdr, hash_lock);
22252688Smaybee 		mutex_exit(hash_lock);
22261544Seschrock 	} else if (acb->acb_done == NULL) {
22271544Seschrock 		int destroy_hdr;
22281544Seschrock 		/*
22291544Seschrock 		 * This is an anonymous buffer with no user callback,
22301544Seschrock 		 * destroy it if there are no active references.
22311544Seschrock 		 */
22321544Seschrock 		mutex_enter(&arc_eviction_mtx);
22331544Seschrock 		destroy_hdr = refcount_is_zero(&hdr->b_refcnt);
22341544Seschrock 		hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
22351544Seschrock 		mutex_exit(&arc_eviction_mtx);
22361544Seschrock 		if (destroy_hdr)
22371544Seschrock 			arc_hdr_destroy(hdr);
22381544Seschrock 	} else {
22391544Seschrock 		hdr->b_flags &= ~ARC_IO_IN_PROGRESS;
2240789Sahrens 	}
22411544Seschrock 
22421544Seschrock 	if (acb->acb_done) {
2243789Sahrens 		ASSERT(!refcount_is_zero(&hdr->b_refcnt));
2244789Sahrens 		acb->acb_done(zio, buf, acb->acb_private);
2245789Sahrens 	}
2246789Sahrens 
22471544Seschrock 	kmem_free(acb, sizeof (arc_callback_t));
2248789Sahrens }
2249789Sahrens 
2250789Sahrens int
22511775Sbillm arc_write(zio_t *pio, spa_t *spa, int checksum, int compress, int ncopies,
2252789Sahrens     uint64_t txg, blkptr_t *bp, arc_buf_t *buf,
2253789Sahrens     arc_done_func_t *done, void *private, int priority, int flags,
22541544Seschrock     uint32_t arc_flags, zbookmark_t *zb)
2255789Sahrens {
2256789Sahrens 	arc_buf_hdr_t *hdr = buf->b_hdr;
2257789Sahrens 	arc_callback_t	*acb;
2258789Sahrens 	zio_t	*rzio;
2259789Sahrens 
2260789Sahrens 	/* this is a private buffer - no locking required */
2261789Sahrens 	ASSERT3P(hdr->b_state, ==, arc.anon);
2262789Sahrens 	ASSERT(BUF_EMPTY(hdr));
2263789Sahrens 	ASSERT(!HDR_IO_ERROR(hdr));
22642237Smaybee 	ASSERT((hdr->b_flags & ARC_IO_IN_PROGRESS) == 0);
22652237Smaybee 	ASSERT(hdr->b_acb == 0);
2266789Sahrens 	acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
2267789Sahrens 	acb->acb_done = done;
2268789Sahrens 	acb->acb_private = private;
2269789Sahrens 	acb->acb_byteswap = (arc_byteswap_func_t *)-1;
2270789Sahrens 	hdr->b_acb = acb;
22711544Seschrock 	hdr->b_flags |= ARC_IO_IN_PROGRESS;
22721775Sbillm 	rzio = zio_write(pio, spa, checksum, compress, ncopies, txg, bp,
22731544Seschrock 	    buf->b_data, hdr->b_size, arc_write_done, buf, priority, flags, zb);
2274789Sahrens 
2275789Sahrens 	if (arc_flags & ARC_WAIT)
2276789Sahrens 		return (zio_wait(rzio));
2277789Sahrens 
2278789Sahrens 	ASSERT(arc_flags & ARC_NOWAIT);
2279789Sahrens 	zio_nowait(rzio);
2280789Sahrens 
2281789Sahrens 	return (0);
2282789Sahrens }
2283789Sahrens 
2284789Sahrens int
2285789Sahrens arc_free(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp,
2286789Sahrens     zio_done_func_t *done, void *private, uint32_t arc_flags)
2287789Sahrens {
2288789Sahrens 	arc_buf_hdr_t *ab;
2289789Sahrens 	kmutex_t *hash_lock;
2290789Sahrens 	zio_t	*zio;
2291789Sahrens 
2292789Sahrens 	/*
2293789Sahrens 	 * If this buffer is in the cache, release it, so it
2294789Sahrens 	 * can be re-used.
2295789Sahrens 	 */
2296789Sahrens 	ab = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_lock);
2297789Sahrens 	if (ab != NULL) {
2298789Sahrens 		/*
2299789Sahrens 		 * The checksum of blocks to free is not always
2300789Sahrens 		 * preserved (eg. on the deadlist).  However, if it is
2301789Sahrens 		 * nonzero, it should match what we have in the cache.
2302789Sahrens 		 */
2303789Sahrens 		ASSERT(bp->blk_cksum.zc_word[0] == 0 ||
2304789Sahrens 		    ab->b_cksum0 == bp->blk_cksum.zc_word[0]);
23051990Smaybee 		if (ab->b_state != arc.anon)
23061990Smaybee 			arc_change_state(arc.anon, ab, hash_lock);
23072391Smaybee 		if (HDR_IO_IN_PROGRESS(ab)) {
23082391Smaybee 			/*
23092391Smaybee 			 * This should only happen when we prefetch.
23102391Smaybee 			 */
23112391Smaybee 			ASSERT(ab->b_flags & ARC_PREFETCH);
23122391Smaybee 			ASSERT3U(ab->b_datacnt, ==, 1);
23132391Smaybee 			ab->b_flags |= ARC_FREED_IN_READ;
23142391Smaybee 			if (HDR_IN_HASH_TABLE(ab))
23152391Smaybee 				buf_hash_remove(ab);
23162391Smaybee 			ab->b_arc_access = 0;
23172391Smaybee 			bzero(&ab->b_dva, sizeof (dva_t));
23182391Smaybee 			ab->b_birth = 0;
23192391Smaybee 			ab->b_cksum0 = 0;
23202391Smaybee 			ab->b_buf->b_efunc = NULL;
23212391Smaybee 			ab->b_buf->b_private = NULL;
23222391Smaybee 			mutex_exit(hash_lock);
23232391Smaybee 		} else if (refcount_is_zero(&ab->b_refcnt)) {
2324789Sahrens 			mutex_exit(hash_lock);
23251544Seschrock 			arc_hdr_destroy(ab);
2326789Sahrens 			atomic_add_64(&arc.deleted, 1);
2327789Sahrens 		} else {
23281589Smaybee 			/*
23292391Smaybee 			 * We still have an active reference on this
23302391Smaybee 			 * buffer.  This can happen, e.g., from
23312391Smaybee 			 * dbuf_unoverride().
23321589Smaybee 			 */
23332391Smaybee 			ASSERT(!HDR_IN_HASH_TABLE(ab));
2334789Sahrens 			ab->b_arc_access = 0;
2335789Sahrens 			bzero(&ab->b_dva, sizeof (dva_t));
2336789Sahrens 			ab->b_birth = 0;
2337789Sahrens 			ab->b_cksum0 = 0;
23381544Seschrock 			ab->b_buf->b_efunc = NULL;
23391544Seschrock 			ab->b_buf->b_private = NULL;
2340789Sahrens 			mutex_exit(hash_lock);
2341789Sahrens 		}
2342789Sahrens 	}
2343789Sahrens 
2344789Sahrens 	zio = zio_free(pio, spa, txg, bp, done, private);
2345789Sahrens 
2346789Sahrens 	if (arc_flags & ARC_WAIT)
2347789Sahrens 		return (zio_wait(zio));
2348789Sahrens 
2349789Sahrens 	ASSERT(arc_flags & ARC_NOWAIT);
2350789Sahrens 	zio_nowait(zio);
2351789Sahrens 
2352789Sahrens 	return (0);
2353789Sahrens }
2354789Sahrens 
2355789Sahrens void
2356789Sahrens arc_tempreserve_clear(uint64_t tempreserve)
2357789Sahrens {
2358789Sahrens 	atomic_add_64(&arc_tempreserve, -tempreserve);
2359789Sahrens 	ASSERT((int64_t)arc_tempreserve >= 0);
2360789Sahrens }
2361789Sahrens 
2362789Sahrens int
2363789Sahrens arc_tempreserve_space(uint64_t tempreserve)
2364789Sahrens {
2365789Sahrens #ifdef ZFS_DEBUG
2366789Sahrens 	/*
2367789Sahrens 	 * Once in a while, fail for no reason.  Everything should cope.
2368789Sahrens 	 */
2369789Sahrens 	if (spa_get_random(10000) == 0) {
2370789Sahrens 		dprintf("forcing random failure\n");
2371789Sahrens 		return (ERESTART);
2372789Sahrens 	}
2373789Sahrens #endif
2374982Smaybee 	if (tempreserve > arc.c/4 && !arc.no_grow)
2375982Smaybee 		arc.c = MIN(arc.c_max, tempreserve * 4);
2376982Smaybee 	if (tempreserve > arc.c)
2377982Smaybee 		return (ENOMEM);
2378982Smaybee 
2379789Sahrens 	/*
2380982Smaybee 	 * Throttle writes when the amount of dirty data in the cache
2381982Smaybee 	 * gets too large.  We try to keep the cache less than half full
2382982Smaybee 	 * of dirty blocks so that our sync times don't grow too large.
2383982Smaybee 	 * Note: if two requests come in concurrently, we might let them
2384982Smaybee 	 * both succeed, when one of them should fail.  Not a huge deal.
2385982Smaybee 	 *
2386982Smaybee 	 * XXX The limit should be adjusted dynamically to keep the time
2387982Smaybee 	 * to sync a dataset fixed (around 1-5 seconds?).
2388789Sahrens 	 */
2389789Sahrens 
2390982Smaybee 	if (tempreserve + arc_tempreserve + arc.anon->size > arc.c / 2 &&
2391982Smaybee 	    arc_tempreserve + arc.anon->size > arc.c / 4) {
2392789Sahrens 		dprintf("failing, arc_tempreserve=%lluK anon=%lluK "
2393789Sahrens 		    "tempreserve=%lluK arc.c=%lluK\n",
2394789Sahrens 		    arc_tempreserve>>10, arc.anon->lsize>>10,
2395789Sahrens 		    tempreserve>>10, arc.c>>10);
2396789Sahrens 		return (ERESTART);
2397789Sahrens 	}
2398789Sahrens 	atomic_add_64(&arc_tempreserve, tempreserve);
2399789Sahrens 	return (0);
2400789Sahrens }
2401789Sahrens 
2402789Sahrens void
2403789Sahrens arc_init(void)
2404789Sahrens {
2405789Sahrens 	mutex_init(&arc_reclaim_lock, NULL, MUTEX_DEFAULT, NULL);
2406789Sahrens 	mutex_init(&arc_reclaim_thr_lock, NULL, MUTEX_DEFAULT, NULL);
2407789Sahrens 	cv_init(&arc_reclaim_thr_cv, NULL, CV_DEFAULT, NULL);
2408789Sahrens 
24092391Smaybee 	/* Convert seconds to clock ticks */
24102638Sperrin 	arc_min_prefetch_lifespan = 1 * hz;
24112391Smaybee 
2412789Sahrens 	/* Start out with 1/8 of all memory */
2413789Sahrens 	arc.c = physmem * PAGESIZE / 8;
2414789Sahrens 
2415789Sahrens #ifdef _KERNEL
2416789Sahrens 	/*
2417789Sahrens 	 * On architectures where the physical memory can be larger
2418789Sahrens 	 * than the addressable space (intel in 32-bit mode), we may
2419789Sahrens 	 * need to limit the cache to 1/8 of VM size.
2420789Sahrens 	 */
2421789Sahrens 	arc.c = MIN(arc.c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8);
2422789Sahrens #endif
2423789Sahrens 
2424982Smaybee 	/* set min cache to 1/32 of all memory, or 64MB, whichever is more */
2425789Sahrens 	arc.c_min = MAX(arc.c / 4, 64<<20);
2426982Smaybee 	/* set max to 3/4 of all memory, or all but 1GB, whichever is more */
2427789Sahrens 	if (arc.c * 8 >= 1<<30)
2428789Sahrens 		arc.c_max = (arc.c * 8) - (1<<30);
2429789Sahrens 	else
2430789Sahrens 		arc.c_max = arc.c_min;
2431789Sahrens 	arc.c_max = MAX(arc.c * 6, arc.c_max);
2432789Sahrens 	arc.c = arc.c_max;
2433789Sahrens 	arc.p = (arc.c >> 1);
2434789Sahrens 
2435789Sahrens 	/* if kmem_flags are set, lets try to use less memory */
2436789Sahrens 	if (kmem_debugging())
2437789Sahrens 		arc.c = arc.c / 2;
2438789Sahrens 	if (arc.c < arc.c_min)
2439789Sahrens 		arc.c = arc.c_min;
2440789Sahrens 
2441789Sahrens 	arc.anon = &ARC_anon;
24421544Seschrock 	arc.mru = &ARC_mru;
24431544Seschrock 	arc.mru_ghost = &ARC_mru_ghost;
24441544Seschrock 	arc.mfu = &ARC_mfu;
24451544Seschrock 	arc.mfu_ghost = &ARC_mfu_ghost;
24461544Seschrock 	arc.size = 0;
2447789Sahrens 
24482688Smaybee 	arc.hits = 0;
24492688Smaybee 	arc.recycle_miss = 0;
24502688Smaybee 	arc.evict_skip = 0;
24512688Smaybee 	arc.mutex_miss = 0;
24522688Smaybee 
2453*2856Snd150628 	mutex_init(&arc.anon->mtx, NULL, MUTEX_DEFAULT, NULL);
2454*2856Snd150628 	mutex_init(&arc.mru->mtx, NULL, MUTEX_DEFAULT, NULL);
2455*2856Snd150628 	mutex_init(&arc.mru_ghost->mtx, NULL, MUTEX_DEFAULT, NULL);
2456*2856Snd150628 	mutex_init(&arc.mfu->mtx, NULL, MUTEX_DEFAULT, NULL);
2457*2856Snd150628 	mutex_init(&arc.mfu_ghost->mtx, NULL, MUTEX_DEFAULT, NULL);
2458*2856Snd150628 
24591544Seschrock 	list_create(&arc.mru->list, sizeof (arc_buf_hdr_t),
2460789Sahrens 	    offsetof(arc_buf_hdr_t, b_arc_node));
24611544Seschrock 	list_create(&arc.mru_ghost->list, sizeof (arc_buf_hdr_t),
2462789Sahrens 	    offsetof(arc_buf_hdr_t, b_arc_node));
24631544Seschrock 	list_create(&arc.mfu->list, sizeof (arc_buf_hdr_t),
2464789Sahrens 	    offsetof(arc_buf_hdr_t, b_arc_node));
24651544Seschrock 	list_create(&arc.mfu_ghost->list, sizeof (arc_buf_hdr_t),
2466789Sahrens 	    offsetof(arc_buf_hdr_t, b_arc_node));
2467789Sahrens 
2468789Sahrens 	buf_init();
2469789Sahrens 
2470789Sahrens 	arc_thread_exit = 0;
24711544Seschrock 	arc_eviction_list = NULL;
24721544Seschrock 	mutex_init(&arc_eviction_mtx, NULL, MUTEX_DEFAULT, NULL);
2473789Sahrens 
2474789Sahrens 	(void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0,
2475789Sahrens 	    TS_RUN, minclsyspri);
2476789Sahrens }
2477789Sahrens 
2478789Sahrens void
2479789Sahrens arc_fini(void)
2480789Sahrens {
2481789Sahrens 	mutex_enter(&arc_reclaim_thr_lock);
2482789Sahrens 	arc_thread_exit = 1;
2483789Sahrens 	while (arc_thread_exit != 0)
2484789Sahrens 		cv_wait(&arc_reclaim_thr_cv, &arc_reclaim_thr_lock);
2485789Sahrens 	mutex_exit(&arc_reclaim_thr_lock);
2486789Sahrens 
2487789Sahrens 	arc_flush();
2488789Sahrens 
2489789Sahrens 	arc_dead = TRUE;
2490789Sahrens 
24911544Seschrock 	mutex_destroy(&arc_eviction_mtx);
2492789Sahrens 	mutex_destroy(&arc_reclaim_lock);
2493789Sahrens 	mutex_destroy(&arc_reclaim_thr_lock);
2494789Sahrens 	cv_destroy(&arc_reclaim_thr_cv);
2495789Sahrens 
24961544Seschrock 	list_destroy(&arc.mru->list);
24971544Seschrock 	list_destroy(&arc.mru_ghost->list);
24981544Seschrock 	list_destroy(&arc.mfu->list);
24991544Seschrock 	list_destroy(&arc.mfu_ghost->list);
2500789Sahrens 
2501*2856Snd150628 	mutex_destroy(&arc.anon->mtx);
2502*2856Snd150628 	mutex_destroy(&arc.mru->mtx);
2503*2856Snd150628 	mutex_destroy(&arc.mru_ghost->mtx);
2504*2856Snd150628 	mutex_destroy(&arc.mfu->mtx);
2505*2856Snd150628 	mutex_destroy(&arc.mfu_ghost->mtx);
2506*2856Snd150628 
2507789Sahrens 	buf_fini();
2508789Sahrens }
2509