1789Sahrens /* 2789Sahrens * CDDL HEADER START 3789Sahrens * 4789Sahrens * The contents of this file are subject to the terms of the 51484Sek110237 * Common Development and Distribution License (the "License"). 61484Sek110237 * You may not use this file except in compliance with the License. 7789Sahrens * 8789Sahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9789Sahrens * or http://www.opensolaris.org/os/licensing. 10789Sahrens * See the License for the specific language governing permissions 11789Sahrens * and limitations under the License. 12789Sahrens * 13789Sahrens * When distributing Covered Code, include this CDDL HEADER in each 14789Sahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15789Sahrens * If applicable, add the following below this CDDL HEADER, with the 16789Sahrens * fields enclosed by brackets "[]" replaced with your own identifying 17789Sahrens * information: Portions Copyright [yyyy] [name of copyright owner] 18789Sahrens * 19789Sahrens * CDDL HEADER END 20789Sahrens */ 21789Sahrens /* 221484Sek110237 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 23789Sahrens * Use is subject to license terms. 24789Sahrens */ 25789Sahrens 26789Sahrens #pragma ident "%Z%%M% %I% %E% SMI" 27789Sahrens 28789Sahrens /* 29789Sahrens * DVA-based Adjustable Relpacement Cache 30789Sahrens * 311544Seschrock * While much of the theory of operation used here is 321544Seschrock * based on the self-tuning, low overhead replacement cache 33789Sahrens * presented by Megiddo and Modha at FAST 2003, there are some 34789Sahrens * significant differences: 35789Sahrens * 36789Sahrens * 1. The Megiddo and Modha model assumes any page is evictable. 37789Sahrens * Pages in its cache cannot be "locked" into memory. This makes 38789Sahrens * the eviction algorithm simple: evict the last page in the list. 39789Sahrens * This also make the performance characteristics easy to reason 40789Sahrens * about. Our cache is not so simple. At any given moment, some 41789Sahrens * subset of the blocks in the cache are un-evictable because we 42789Sahrens * have handed out a reference to them. Blocks are only evictable 43789Sahrens * when there are no external references active. This makes 44789Sahrens * eviction far more problematic: we choose to evict the evictable 45789Sahrens * blocks that are the "lowest" in the list. 46789Sahrens * 47789Sahrens * There are times when it is not possible to evict the requested 48789Sahrens * space. In these circumstances we are unable to adjust the cache 49789Sahrens * size. To prevent the cache growing unbounded at these times we 50789Sahrens * implement a "cache throttle" that slowes the flow of new data 51789Sahrens * into the cache until we can make space avaiable. 52789Sahrens * 53789Sahrens * 2. The Megiddo and Modha model assumes a fixed cache size. 54789Sahrens * Pages are evicted when the cache is full and there is a cache 55789Sahrens * miss. Our model has a variable sized cache. It grows with 56789Sahrens * high use, but also tries to react to memory preasure from the 57789Sahrens * operating system: decreasing its size when system memory is 58789Sahrens * tight. 59789Sahrens * 60789Sahrens * 3. The Megiddo and Modha model assumes a fixed page size. All 61789Sahrens * elements of the cache are therefor exactly the same size. So 62789Sahrens * when adjusting the cache size following a cache miss, its simply 63789Sahrens * a matter of choosing a single page to evict. In our model, we 64789Sahrens * have variable sized cache blocks (rangeing from 512 bytes to 65789Sahrens * 128K bytes). We therefor choose a set of blocks to evict to make 66789Sahrens * space for a cache miss that approximates as closely as possible 67789Sahrens * the space used by the new block. 68789Sahrens * 69789Sahrens * See also: "ARC: A Self-Tuning, Low Overhead Replacement Cache" 70789Sahrens * by N. Megiddo & D. Modha, FAST 2003 71789Sahrens */ 72789Sahrens 73789Sahrens /* 74789Sahrens * The locking model: 75789Sahrens * 76789Sahrens * A new reference to a cache buffer can be obtained in two 77789Sahrens * ways: 1) via a hash table lookup using the DVA as a key, 78789Sahrens * or 2) via one of the ARC lists. The arc_read() inerface 79789Sahrens * uses method 1, while the internal arc algorithms for 80789Sahrens * adjusting the cache use method 2. We therefor provide two 81789Sahrens * types of locks: 1) the hash table lock array, and 2) the 82789Sahrens * arc list locks. 83789Sahrens * 84789Sahrens * Buffers do not have their own mutexs, rather they rely on the 85789Sahrens * hash table mutexs for the bulk of their protection (i.e. most 86789Sahrens * fields in the arc_buf_hdr_t are protected by these mutexs). 87789Sahrens * 88789Sahrens * buf_hash_find() returns the appropriate mutex (held) when it 89789Sahrens * locates the requested buffer in the hash table. It returns 90789Sahrens * NULL for the mutex if the buffer was not in the table. 91789Sahrens * 92789Sahrens * buf_hash_remove() expects the appropriate hash mutex to be 93789Sahrens * already held before it is invoked. 94789Sahrens * 95789Sahrens * Each arc state also has a mutex which is used to protect the 96789Sahrens * buffer list associated with the state. When attempting to 97789Sahrens * obtain a hash table lock while holding an arc list lock you 98789Sahrens * must use: mutex_tryenter() to avoid deadlock. Also note that 992688Smaybee * the active state mutex must be held before the ghost state mutex. 100789Sahrens * 1011544Seschrock * Arc buffers may have an associated eviction callback function. 1021544Seschrock * This function will be invoked prior to removing the buffer (e.g. 1031544Seschrock * in arc_do_user_evicts()). Note however that the data associated 1041544Seschrock * with the buffer may be evicted prior to the callback. The callback 1051544Seschrock * must be made with *no locks held* (to prevent deadlock). Additionally, 1061544Seschrock * the users of callbacks must ensure that their private data is 1071544Seschrock * protected from simultaneous callbacks from arc_buf_evict() 1081544Seschrock * and arc_do_user_evicts(). 1091544Seschrock * 110789Sahrens * Note that the majority of the performance stats are manipulated 111789Sahrens * with atomic operations. 112789Sahrens */ 113789Sahrens 114789Sahrens #include <sys/spa.h> 115789Sahrens #include <sys/zio.h> 116789Sahrens #include <sys/zfs_context.h> 117789Sahrens #include <sys/arc.h> 118789Sahrens #include <sys/refcount.h> 119789Sahrens #ifdef _KERNEL 120789Sahrens #include <sys/vmsystm.h> 121789Sahrens #include <vm/anon.h> 122789Sahrens #include <sys/fs/swapnode.h> 1231484Sek110237 #include <sys/dnlc.h> 124789Sahrens #endif 125789Sahrens #include <sys/callb.h> 126789Sahrens 127789Sahrens static kmutex_t arc_reclaim_thr_lock; 128789Sahrens static kcondvar_t arc_reclaim_thr_cv; /* used to signal reclaim thr */ 129789Sahrens static uint8_t arc_thread_exit; 130789Sahrens 1311484Sek110237 #define ARC_REDUCE_DNLC_PERCENT 3 1321484Sek110237 uint_t arc_reduce_dnlc_percent = ARC_REDUCE_DNLC_PERCENT; 1331484Sek110237 134789Sahrens typedef enum arc_reclaim_strategy { 135789Sahrens ARC_RECLAIM_AGGR, /* Aggressive reclaim strategy */ 136789Sahrens ARC_RECLAIM_CONS /* Conservative reclaim strategy */ 137789Sahrens } arc_reclaim_strategy_t; 138789Sahrens 139789Sahrens /* number of seconds before growing cache again */ 140789Sahrens static int arc_grow_retry = 60; 141789Sahrens 1422391Smaybee /* 1432638Sperrin * minimum lifespan of a prefetch block in clock ticks 1442638Sperrin * (initialized in arc_init()) 1452391Smaybee */ 1462638Sperrin static int arc_min_prefetch_lifespan; 1472391Smaybee 148789Sahrens static kmutex_t arc_reclaim_lock; 149789Sahrens static int arc_dead; 150789Sahrens 151789Sahrens /* 152789Sahrens * Note that buffers can be on one of 5 states: 153789Sahrens * ARC_anon - anonymous (discussed below) 1541544Seschrock * ARC_mru - recently used, currently cached 1551544Seschrock * ARC_mru_ghost - recentely used, no longer in cache 1561544Seschrock * ARC_mfu - frequently used, currently cached 1571544Seschrock * ARC_mfu_ghost - frequently used, no longer in cache 158789Sahrens * When there are no active references to the buffer, they 159789Sahrens * are linked onto one of the lists in arc. These are the 160789Sahrens * only buffers that can be evicted or deleted. 161789Sahrens * 162789Sahrens * Anonymous buffers are buffers that are not associated with 163789Sahrens * a DVA. These are buffers that hold dirty block copies 164789Sahrens * before they are written to stable storage. By definition, 1651544Seschrock * they are "ref'd" and are considered part of arc_mru 166789Sahrens * that cannot be freed. Generally, they will aquire a DVA 1671544Seschrock * as they are written and migrate onto the arc_mru list. 168789Sahrens */ 169789Sahrens 170789Sahrens typedef struct arc_state { 171789Sahrens list_t list; /* linked list of evictable buffer in state */ 172789Sahrens uint64_t lsize; /* total size of buffers in the linked list */ 173789Sahrens uint64_t size; /* total size of all buffers in this state */ 174789Sahrens uint64_t hits; 175789Sahrens kmutex_t mtx; 176789Sahrens } arc_state_t; 177789Sahrens 178789Sahrens /* The 5 states: */ 179789Sahrens static arc_state_t ARC_anon; 1801544Seschrock static arc_state_t ARC_mru; 1811544Seschrock static arc_state_t ARC_mru_ghost; 1821544Seschrock static arc_state_t ARC_mfu; 1831544Seschrock static arc_state_t ARC_mfu_ghost; 184789Sahrens 185789Sahrens static struct arc { 186789Sahrens arc_state_t *anon; 1871544Seschrock arc_state_t *mru; 1881544Seschrock arc_state_t *mru_ghost; 1891544Seschrock arc_state_t *mfu; 1901544Seschrock arc_state_t *mfu_ghost; 191789Sahrens uint64_t size; /* Actual total arc size */ 1921544Seschrock uint64_t p; /* Target size (in bytes) of mru */ 193789Sahrens uint64_t c; /* Target size of cache (in bytes) */ 194789Sahrens uint64_t c_min; /* Minimum target cache size */ 195789Sahrens uint64_t c_max; /* Maximum target cache size */ 196789Sahrens 197789Sahrens /* performance stats */ 198789Sahrens uint64_t hits; 199789Sahrens uint64_t misses; 200789Sahrens uint64_t deleted; 2012688Smaybee uint64_t recycle_miss; 2022688Smaybee uint64_t mutex_miss; 2032688Smaybee uint64_t evict_skip; 204789Sahrens uint64_t hash_elements; 205789Sahrens uint64_t hash_elements_max; 206789Sahrens uint64_t hash_collisions; 207789Sahrens uint64_t hash_chains; 208789Sahrens uint32_t hash_chain_max; 209789Sahrens 210789Sahrens int no_grow; /* Don't try to grow cache size */ 211789Sahrens } arc; 212789Sahrens 213789Sahrens static uint64_t arc_tempreserve; 214789Sahrens 215789Sahrens typedef struct arc_callback arc_callback_t; 216789Sahrens 217789Sahrens struct arc_callback { 218789Sahrens arc_done_func_t *acb_done; 219789Sahrens void *acb_private; 220789Sahrens arc_byteswap_func_t *acb_byteswap; 221789Sahrens arc_buf_t *acb_buf; 222789Sahrens zio_t *acb_zio_dummy; 223789Sahrens arc_callback_t *acb_next; 224789Sahrens }; 225789Sahrens 226789Sahrens struct arc_buf_hdr { 227789Sahrens /* immutable */ 228789Sahrens uint64_t b_size; 229789Sahrens spa_t *b_spa; 230789Sahrens 231789Sahrens /* protected by hash lock */ 232789Sahrens dva_t b_dva; 233789Sahrens uint64_t b_birth; 234789Sahrens uint64_t b_cksum0; 235789Sahrens 236789Sahrens arc_buf_hdr_t *b_hash_next; 237789Sahrens arc_buf_t *b_buf; 238789Sahrens uint32_t b_flags; 2391544Seschrock uint32_t b_datacnt; 240789Sahrens 241789Sahrens kcondvar_t b_cv; 242789Sahrens arc_callback_t *b_acb; 243789Sahrens 244789Sahrens /* protected by arc state mutex */ 245789Sahrens arc_state_t *b_state; 246789Sahrens list_node_t b_arc_node; 247789Sahrens 248789Sahrens /* updated atomically */ 249789Sahrens clock_t b_arc_access; 250789Sahrens 251789Sahrens /* self protecting */ 252789Sahrens refcount_t b_refcnt; 253789Sahrens }; 254789Sahrens 2551544Seschrock static arc_buf_t *arc_eviction_list; 2561544Seschrock static kmutex_t arc_eviction_mtx; 2572688Smaybee static void arc_get_data_buf(arc_buf_t *buf); 2582688Smaybee static void arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock); 2591544Seschrock 2601544Seschrock #define GHOST_STATE(state) \ 2611544Seschrock ((state) == arc.mru_ghost || (state) == arc.mfu_ghost) 2621544Seschrock 263789Sahrens /* 264789Sahrens * Private ARC flags. These flags are private ARC only flags that will show up 265789Sahrens * in b_flags in the arc_hdr_buf_t. Some flags are publicly declared, and can 266789Sahrens * be passed in as arc_flags in things like arc_read. However, these flags 267789Sahrens * should never be passed and should only be set by ARC code. When adding new 268789Sahrens * public flags, make sure not to smash the private ones. 269789Sahrens */ 270789Sahrens 2711544Seschrock #define ARC_IN_HASH_TABLE (1 << 9) /* this buffer is hashed */ 272789Sahrens #define ARC_IO_IN_PROGRESS (1 << 10) /* I/O in progress for buf */ 273789Sahrens #define ARC_IO_ERROR (1 << 11) /* I/O failed for buf */ 274789Sahrens #define ARC_FREED_IN_READ (1 << 12) /* buf freed while in read */ 2751544Seschrock #define ARC_BUF_AVAILABLE (1 << 13) /* block not in active use */ 2762391Smaybee #define ARC_INDIRECT (1 << 14) /* this is an indirect block */ 277789Sahrens 2781544Seschrock #define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_IN_HASH_TABLE) 279789Sahrens #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_IO_IN_PROGRESS) 280789Sahrens #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_IO_ERROR) 281789Sahrens #define HDR_FREED_IN_READ(hdr) ((hdr)->b_flags & ARC_FREED_IN_READ) 2821544Seschrock #define HDR_BUF_AVAILABLE(hdr) ((hdr)->b_flags & ARC_BUF_AVAILABLE) 283789Sahrens 284789Sahrens /* 285789Sahrens * Hash table routines 286789Sahrens */ 287789Sahrens 288789Sahrens #define HT_LOCK_PAD 64 289789Sahrens 290789Sahrens struct ht_lock { 291789Sahrens kmutex_t ht_lock; 292789Sahrens #ifdef _KERNEL 293789Sahrens unsigned char pad[(HT_LOCK_PAD - sizeof (kmutex_t))]; 294789Sahrens #endif 295789Sahrens }; 296789Sahrens 297789Sahrens #define BUF_LOCKS 256 298789Sahrens typedef struct buf_hash_table { 299789Sahrens uint64_t ht_mask; 300789Sahrens arc_buf_hdr_t **ht_table; 301789Sahrens struct ht_lock ht_locks[BUF_LOCKS]; 302789Sahrens } buf_hash_table_t; 303789Sahrens 304789Sahrens static buf_hash_table_t buf_hash_table; 305789Sahrens 306789Sahrens #define BUF_HASH_INDEX(spa, dva, birth) \ 307789Sahrens (buf_hash(spa, dva, birth) & buf_hash_table.ht_mask) 308789Sahrens #define BUF_HASH_LOCK_NTRY(idx) (buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)]) 309789Sahrens #define BUF_HASH_LOCK(idx) (&(BUF_HASH_LOCK_NTRY(idx).ht_lock)) 310789Sahrens #define HDR_LOCK(buf) \ 311789Sahrens (BUF_HASH_LOCK(BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth))) 312789Sahrens 313789Sahrens uint64_t zfs_crc64_table[256]; 314789Sahrens 315789Sahrens static uint64_t 316789Sahrens buf_hash(spa_t *spa, dva_t *dva, uint64_t birth) 317789Sahrens { 318789Sahrens uintptr_t spav = (uintptr_t)spa; 319789Sahrens uint8_t *vdva = (uint8_t *)dva; 320789Sahrens uint64_t crc = -1ULL; 321789Sahrens int i; 322789Sahrens 323789Sahrens ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 324789Sahrens 325789Sahrens for (i = 0; i < sizeof (dva_t); i++) 326789Sahrens crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ vdva[i]) & 0xFF]; 327789Sahrens 328789Sahrens crc ^= (spav>>8) ^ birth; 329789Sahrens 330789Sahrens return (crc); 331789Sahrens } 332789Sahrens 333789Sahrens #define BUF_EMPTY(buf) \ 334789Sahrens ((buf)->b_dva.dva_word[0] == 0 && \ 335789Sahrens (buf)->b_dva.dva_word[1] == 0 && \ 336789Sahrens (buf)->b_birth == 0) 337789Sahrens 338789Sahrens #define BUF_EQUAL(spa, dva, birth, buf) \ 339789Sahrens ((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \ 340789Sahrens ((buf)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \ 341789Sahrens ((buf)->b_birth == birth) && ((buf)->b_spa == spa) 342789Sahrens 343789Sahrens static arc_buf_hdr_t * 344789Sahrens buf_hash_find(spa_t *spa, dva_t *dva, uint64_t birth, kmutex_t **lockp) 345789Sahrens { 346789Sahrens uint64_t idx = BUF_HASH_INDEX(spa, dva, birth); 347789Sahrens kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 348789Sahrens arc_buf_hdr_t *buf; 349789Sahrens 350789Sahrens mutex_enter(hash_lock); 351789Sahrens for (buf = buf_hash_table.ht_table[idx]; buf != NULL; 352789Sahrens buf = buf->b_hash_next) { 353789Sahrens if (BUF_EQUAL(spa, dva, birth, buf)) { 354789Sahrens *lockp = hash_lock; 355789Sahrens return (buf); 356789Sahrens } 357789Sahrens } 358789Sahrens mutex_exit(hash_lock); 359789Sahrens *lockp = NULL; 360789Sahrens return (NULL); 361789Sahrens } 362789Sahrens 363789Sahrens /* 364789Sahrens * Insert an entry into the hash table. If there is already an element 365789Sahrens * equal to elem in the hash table, then the already existing element 366789Sahrens * will be returned and the new element will not be inserted. 367789Sahrens * Otherwise returns NULL. 368789Sahrens */ 369789Sahrens static arc_buf_hdr_t * 370789Sahrens buf_hash_insert(arc_buf_hdr_t *buf, kmutex_t **lockp) 371789Sahrens { 372789Sahrens uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth); 373789Sahrens kmutex_t *hash_lock = BUF_HASH_LOCK(idx); 374789Sahrens arc_buf_hdr_t *fbuf; 375789Sahrens uint32_t max, i; 376789Sahrens 3771544Seschrock ASSERT(!HDR_IN_HASH_TABLE(buf)); 378789Sahrens *lockp = hash_lock; 379789Sahrens mutex_enter(hash_lock); 380789Sahrens for (fbuf = buf_hash_table.ht_table[idx], i = 0; fbuf != NULL; 381789Sahrens fbuf = fbuf->b_hash_next, i++) { 382789Sahrens if (BUF_EQUAL(buf->b_spa, &buf->b_dva, buf->b_birth, fbuf)) 383789Sahrens return (fbuf); 384789Sahrens } 385789Sahrens 386789Sahrens buf->b_hash_next = buf_hash_table.ht_table[idx]; 387789Sahrens buf_hash_table.ht_table[idx] = buf; 3881544Seschrock buf->b_flags |= ARC_IN_HASH_TABLE; 389789Sahrens 390789Sahrens /* collect some hash table performance data */ 391789Sahrens if (i > 0) { 392789Sahrens atomic_add_64(&arc.hash_collisions, 1); 393789Sahrens if (i == 1) 394789Sahrens atomic_add_64(&arc.hash_chains, 1); 395789Sahrens } 396789Sahrens while (i > (max = arc.hash_chain_max) && 397789Sahrens max != atomic_cas_32(&arc.hash_chain_max, max, i)) { 398789Sahrens continue; 399789Sahrens } 400789Sahrens atomic_add_64(&arc.hash_elements, 1); 401789Sahrens if (arc.hash_elements > arc.hash_elements_max) 402789Sahrens atomic_add_64(&arc.hash_elements_max, 1); 403789Sahrens 404789Sahrens return (NULL); 405789Sahrens } 406789Sahrens 407789Sahrens static void 408789Sahrens buf_hash_remove(arc_buf_hdr_t *buf) 409789Sahrens { 410789Sahrens arc_buf_hdr_t *fbuf, **bufp; 411789Sahrens uint64_t idx = BUF_HASH_INDEX(buf->b_spa, &buf->b_dva, buf->b_birth); 412789Sahrens 413789Sahrens ASSERT(MUTEX_HELD(BUF_HASH_LOCK(idx))); 4141544Seschrock ASSERT(HDR_IN_HASH_TABLE(buf)); 415789Sahrens 416789Sahrens bufp = &buf_hash_table.ht_table[idx]; 417789Sahrens while ((fbuf = *bufp) != buf) { 418789Sahrens ASSERT(fbuf != NULL); 419789Sahrens bufp = &fbuf->b_hash_next; 420789Sahrens } 421789Sahrens *bufp = buf->b_hash_next; 422789Sahrens buf->b_hash_next = NULL; 4231544Seschrock buf->b_flags &= ~ARC_IN_HASH_TABLE; 424789Sahrens 425789Sahrens /* collect some hash table performance data */ 426789Sahrens atomic_add_64(&arc.hash_elements, -1); 427789Sahrens if (buf_hash_table.ht_table[idx] && 428789Sahrens buf_hash_table.ht_table[idx]->b_hash_next == NULL) 429789Sahrens atomic_add_64(&arc.hash_chains, -1); 430789Sahrens } 431789Sahrens 432789Sahrens /* 433789Sahrens * Global data structures and functions for the buf kmem cache. 434789Sahrens */ 435789Sahrens static kmem_cache_t *hdr_cache; 436789Sahrens static kmem_cache_t *buf_cache; 437789Sahrens 438789Sahrens static void 439789Sahrens buf_fini(void) 440789Sahrens { 441789Sahrens int i; 442789Sahrens 443789Sahrens kmem_free(buf_hash_table.ht_table, 444789Sahrens (buf_hash_table.ht_mask + 1) * sizeof (void *)); 445789Sahrens for (i = 0; i < BUF_LOCKS; i++) 446789Sahrens mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock); 447789Sahrens kmem_cache_destroy(hdr_cache); 448789Sahrens kmem_cache_destroy(buf_cache); 449789Sahrens } 450789Sahrens 451789Sahrens /* 452789Sahrens * Constructor callback - called when the cache is empty 453789Sahrens * and a new buf is requested. 454789Sahrens */ 455789Sahrens /* ARGSUSED */ 456789Sahrens static int 457789Sahrens hdr_cons(void *vbuf, void *unused, int kmflag) 458789Sahrens { 459789Sahrens arc_buf_hdr_t *buf = vbuf; 460789Sahrens 461789Sahrens bzero(buf, sizeof (arc_buf_hdr_t)); 462789Sahrens refcount_create(&buf->b_refcnt); 463789Sahrens cv_init(&buf->b_cv, NULL, CV_DEFAULT, NULL); 464789Sahrens return (0); 465789Sahrens } 466789Sahrens 467789Sahrens /* 468789Sahrens * Destructor callback - called when a cached buf is 469789Sahrens * no longer required. 470789Sahrens */ 471789Sahrens /* ARGSUSED */ 472789Sahrens static void 473789Sahrens hdr_dest(void *vbuf, void *unused) 474789Sahrens { 475789Sahrens arc_buf_hdr_t *buf = vbuf; 476789Sahrens 477789Sahrens refcount_destroy(&buf->b_refcnt); 478789Sahrens cv_destroy(&buf->b_cv); 479789Sahrens } 480789Sahrens 4811544Seschrock static int arc_reclaim_needed(void); 482789Sahrens void arc_kmem_reclaim(void); 483789Sahrens 484789Sahrens /* 485789Sahrens * Reclaim callback -- invoked when memory is low. 486789Sahrens */ 487789Sahrens /* ARGSUSED */ 488789Sahrens static void 489789Sahrens hdr_recl(void *unused) 490789Sahrens { 491789Sahrens dprintf("hdr_recl called\n"); 4921544Seschrock if (arc_reclaim_needed()) 4931544Seschrock arc_kmem_reclaim(); 494789Sahrens } 495789Sahrens 496789Sahrens static void 497789Sahrens buf_init(void) 498789Sahrens { 499789Sahrens uint64_t *ct; 5001544Seschrock uint64_t hsize = 1ULL << 12; 501789Sahrens int i, j; 502789Sahrens 503789Sahrens /* 504789Sahrens * The hash table is big enough to fill all of physical memory 5051544Seschrock * with an average 64K block size. The table will take up 5061544Seschrock * totalmem*sizeof(void*)/64K (eg. 128KB/GB with 8-byte pointers). 507789Sahrens */ 5081544Seschrock while (hsize * 65536 < physmem * PAGESIZE) 509789Sahrens hsize <<= 1; 5101544Seschrock retry: 511789Sahrens buf_hash_table.ht_mask = hsize - 1; 5121544Seschrock buf_hash_table.ht_table = 5131544Seschrock kmem_zalloc(hsize * sizeof (void*), KM_NOSLEEP); 5141544Seschrock if (buf_hash_table.ht_table == NULL) { 5151544Seschrock ASSERT(hsize > (1ULL << 8)); 5161544Seschrock hsize >>= 1; 5171544Seschrock goto retry; 5181544Seschrock } 519789Sahrens 520789Sahrens hdr_cache = kmem_cache_create("arc_buf_hdr_t", sizeof (arc_buf_hdr_t), 521789Sahrens 0, hdr_cons, hdr_dest, hdr_recl, NULL, NULL, 0); 522789Sahrens buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t), 523789Sahrens 0, NULL, NULL, NULL, NULL, NULL, 0); 524789Sahrens 525789Sahrens for (i = 0; i < 256; i++) 526789Sahrens for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--) 527789Sahrens *ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY); 528789Sahrens 529789Sahrens for (i = 0; i < BUF_LOCKS; i++) { 530789Sahrens mutex_init(&buf_hash_table.ht_locks[i].ht_lock, 531789Sahrens NULL, MUTEX_DEFAULT, NULL); 532789Sahrens } 533789Sahrens } 534789Sahrens 535789Sahrens #define ARC_MINTIME (hz>>4) /* 62 ms */ 536789Sahrens 537789Sahrens static void 538789Sahrens add_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag) 539789Sahrens { 540789Sahrens ASSERT(MUTEX_HELD(hash_lock)); 541789Sahrens 542789Sahrens if ((refcount_add(&ab->b_refcnt, tag) == 1) && 543789Sahrens (ab->b_state != arc.anon)) { 5441544Seschrock int delta = ab->b_size * ab->b_datacnt; 545789Sahrens 546789Sahrens ASSERT(!MUTEX_HELD(&ab->b_state->mtx)); 547789Sahrens mutex_enter(&ab->b_state->mtx); 548789Sahrens ASSERT(list_link_active(&ab->b_arc_node)); 549789Sahrens list_remove(&ab->b_state->list, ab); 5501544Seschrock if (GHOST_STATE(ab->b_state)) { 5511544Seschrock ASSERT3U(ab->b_datacnt, ==, 0); 5521544Seschrock ASSERT3P(ab->b_buf, ==, NULL); 5531544Seschrock delta = ab->b_size; 5541544Seschrock } 5551544Seschrock ASSERT(delta > 0); 5561544Seschrock ASSERT3U(ab->b_state->lsize, >=, delta); 5571544Seschrock atomic_add_64(&ab->b_state->lsize, -delta); 558789Sahrens mutex_exit(&ab->b_state->mtx); 5592391Smaybee /* remove the prefetch flag is we get a reference */ 5602391Smaybee if (ab->b_flags & ARC_PREFETCH) 5612391Smaybee ab->b_flags &= ~ARC_PREFETCH; 562789Sahrens } 563789Sahrens } 564789Sahrens 565789Sahrens static int 566789Sahrens remove_reference(arc_buf_hdr_t *ab, kmutex_t *hash_lock, void *tag) 567789Sahrens { 568789Sahrens int cnt; 569789Sahrens 5701544Seschrock ASSERT(ab->b_state == arc.anon || MUTEX_HELD(hash_lock)); 5711544Seschrock ASSERT(!GHOST_STATE(ab->b_state)); 572789Sahrens 573789Sahrens if (((cnt = refcount_remove(&ab->b_refcnt, tag)) == 0) && 574789Sahrens (ab->b_state != arc.anon)) { 575789Sahrens 576789Sahrens ASSERT(!MUTEX_HELD(&ab->b_state->mtx)); 577789Sahrens mutex_enter(&ab->b_state->mtx); 578789Sahrens ASSERT(!list_link_active(&ab->b_arc_node)); 579789Sahrens list_insert_head(&ab->b_state->list, ab); 5801544Seschrock ASSERT(ab->b_datacnt > 0); 5811544Seschrock atomic_add_64(&ab->b_state->lsize, ab->b_size * ab->b_datacnt); 5821544Seschrock ASSERT3U(ab->b_state->size, >=, ab->b_state->lsize); 583789Sahrens mutex_exit(&ab->b_state->mtx); 584789Sahrens } 585789Sahrens return (cnt); 586789Sahrens } 587789Sahrens 588789Sahrens /* 589789Sahrens * Move the supplied buffer to the indicated state. The mutex 590789Sahrens * for the buffer must be held by the caller. 591789Sahrens */ 592789Sahrens static void 5931544Seschrock arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *ab, kmutex_t *hash_lock) 594789Sahrens { 5951544Seschrock arc_state_t *old_state = ab->b_state; 5961544Seschrock int refcnt = refcount_count(&ab->b_refcnt); 5971544Seschrock int from_delta, to_delta; 598789Sahrens 599789Sahrens ASSERT(MUTEX_HELD(hash_lock)); 6001544Seschrock ASSERT(new_state != old_state); 6011544Seschrock ASSERT(refcnt == 0 || ab->b_datacnt > 0); 6021544Seschrock ASSERT(ab->b_datacnt == 0 || !GHOST_STATE(new_state)); 6031544Seschrock 6041544Seschrock from_delta = to_delta = ab->b_datacnt * ab->b_size; 605789Sahrens 606789Sahrens /* 607789Sahrens * If this buffer is evictable, transfer it from the 608789Sahrens * old state list to the new state list. 609789Sahrens */ 6101544Seschrock if (refcnt == 0) { 6111544Seschrock if (old_state != arc.anon) { 6121544Seschrock int use_mutex = !MUTEX_HELD(&old_state->mtx); 6131544Seschrock 6141544Seschrock if (use_mutex) 6151544Seschrock mutex_enter(&old_state->mtx); 6161544Seschrock 6171544Seschrock ASSERT(list_link_active(&ab->b_arc_node)); 6181544Seschrock list_remove(&old_state->list, ab); 619789Sahrens 6202391Smaybee /* 6212391Smaybee * If prefetching out of the ghost cache, 6222391Smaybee * we will have a non-null datacnt. 6232391Smaybee */ 6242391Smaybee if (GHOST_STATE(old_state) && ab->b_datacnt == 0) { 6252391Smaybee /* ghost elements have a ghost size */ 6261544Seschrock ASSERT(ab->b_buf == NULL); 6271544Seschrock from_delta = ab->b_size; 628789Sahrens } 6291544Seschrock ASSERT3U(old_state->lsize, >=, from_delta); 6301544Seschrock atomic_add_64(&old_state->lsize, -from_delta); 6311544Seschrock 6321544Seschrock if (use_mutex) 6331544Seschrock mutex_exit(&old_state->mtx); 634789Sahrens } 635789Sahrens if (new_state != arc.anon) { 6361544Seschrock int use_mutex = !MUTEX_HELD(&new_state->mtx); 637789Sahrens 6381544Seschrock if (use_mutex) 639789Sahrens mutex_enter(&new_state->mtx); 6401544Seschrock 641789Sahrens list_insert_head(&new_state->list, ab); 6421544Seschrock 6431544Seschrock /* ghost elements have a ghost size */ 6441544Seschrock if (GHOST_STATE(new_state)) { 6451544Seschrock ASSERT(ab->b_datacnt == 0); 6461544Seschrock ASSERT(ab->b_buf == NULL); 6471544Seschrock to_delta = ab->b_size; 6481544Seschrock } 6491544Seschrock atomic_add_64(&new_state->lsize, to_delta); 6501544Seschrock ASSERT3U(new_state->size + to_delta, >=, 6511544Seschrock new_state->lsize); 6521544Seschrock 6531544Seschrock if (use_mutex) 654789Sahrens mutex_exit(&new_state->mtx); 655789Sahrens } 656789Sahrens } 657789Sahrens 658789Sahrens ASSERT(!BUF_EMPTY(ab)); 6591544Seschrock if (new_state == arc.anon && old_state != arc.anon) { 660789Sahrens buf_hash_remove(ab); 661789Sahrens } 662789Sahrens 6631544Seschrock /* adjust state sizes */ 6641544Seschrock if (to_delta) 6651544Seschrock atomic_add_64(&new_state->size, to_delta); 6661544Seschrock if (from_delta) { 6671544Seschrock ASSERT3U(old_state->size, >=, from_delta); 6681544Seschrock atomic_add_64(&old_state->size, -from_delta); 669789Sahrens } 670789Sahrens ab->b_state = new_state; 671789Sahrens } 672789Sahrens 673789Sahrens arc_buf_t * 674789Sahrens arc_buf_alloc(spa_t *spa, int size, void *tag) 675789Sahrens { 676789Sahrens arc_buf_hdr_t *hdr; 677789Sahrens arc_buf_t *buf; 678789Sahrens 679789Sahrens ASSERT3U(size, >, 0); 680789Sahrens hdr = kmem_cache_alloc(hdr_cache, KM_SLEEP); 681789Sahrens ASSERT(BUF_EMPTY(hdr)); 682789Sahrens hdr->b_size = size; 683789Sahrens hdr->b_spa = spa; 684789Sahrens hdr->b_state = arc.anon; 685789Sahrens hdr->b_arc_access = 0; 686789Sahrens buf = kmem_cache_alloc(buf_cache, KM_SLEEP); 687789Sahrens buf->b_hdr = hdr; 6882688Smaybee buf->b_data = NULL; 6891544Seschrock buf->b_efunc = NULL; 6901544Seschrock buf->b_private = NULL; 691789Sahrens buf->b_next = NULL; 692789Sahrens hdr->b_buf = buf; 6932688Smaybee arc_get_data_buf(buf); 6941544Seschrock hdr->b_datacnt = 1; 695789Sahrens hdr->b_flags = 0; 696789Sahrens ASSERT(refcount_is_zero(&hdr->b_refcnt)); 697789Sahrens (void) refcount_add(&hdr->b_refcnt, tag); 698789Sahrens 699789Sahrens return (buf); 700789Sahrens } 701789Sahrens 7022688Smaybee static arc_buf_t * 7032688Smaybee arc_buf_clone(arc_buf_t *from) 7041544Seschrock { 7052688Smaybee arc_buf_t *buf; 7062688Smaybee arc_buf_hdr_t *hdr = from->b_hdr; 7072688Smaybee uint64_t size = hdr->b_size; 7081544Seschrock 7092688Smaybee buf = kmem_cache_alloc(buf_cache, KM_SLEEP); 7102688Smaybee buf->b_hdr = hdr; 7112688Smaybee buf->b_data = NULL; 7122688Smaybee buf->b_efunc = NULL; 7132688Smaybee buf->b_private = NULL; 7142688Smaybee buf->b_next = hdr->b_buf; 7152688Smaybee hdr->b_buf = buf; 7162688Smaybee arc_get_data_buf(buf); 7172688Smaybee bcopy(from->b_data, buf->b_data, size); 7182688Smaybee hdr->b_datacnt += 1; 7192688Smaybee return (buf); 7201544Seschrock } 7211544Seschrock 7221544Seschrock void 7231544Seschrock arc_buf_add_ref(arc_buf_t *buf, void* tag) 7241544Seschrock { 7252724Smaybee arc_buf_hdr_t *hdr = buf->b_hdr; 7261544Seschrock kmutex_t *hash_lock; 7271544Seschrock 7282724Smaybee /* 7292724Smaybee * Check to see if this buffer is currently being evicted via 7302724Smaybee * arc_do_user_evicts(). We can do this without holding any 7312724Smaybee * locks because if we happen to obtain the header before its 7322724Smaybee * cleared, we will find b_data is NULL later. 7332724Smaybee */ 7342724Smaybee if (hdr == NULL) 7352724Smaybee return; 7362724Smaybee 7372724Smaybee hash_lock = HDR_LOCK(hdr); 7382724Smaybee mutex_enter(hash_lock); 7391544Seschrock if (buf->b_data == NULL) { 7401544Seschrock /* 7411544Seschrock * This buffer is evicted. 7421544Seschrock */ 7432724Smaybee mutex_exit(hash_lock); 7441544Seschrock return; 7451544Seschrock } 7461544Seschrock 7472724Smaybee ASSERT(buf->b_hdr == hdr); 7482724Smaybee ASSERT(hdr->b_state == arc.mru || hdr->b_state == arc.mfu); 7491544Seschrock add_reference(hdr, hash_lock, tag); 7502688Smaybee arc_access(hdr, hash_lock); 7512688Smaybee mutex_exit(hash_lock); 7521544Seschrock atomic_add_64(&arc.hits, 1); 7531544Seschrock } 7541544Seschrock 755789Sahrens static void 7562688Smaybee arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t all) 7571544Seschrock { 7581544Seschrock arc_buf_t **bufp; 7591544Seschrock 7601544Seschrock /* free up data associated with the buf */ 7611544Seschrock if (buf->b_data) { 7621544Seschrock arc_state_t *state = buf->b_hdr->b_state; 7631544Seschrock uint64_t size = buf->b_hdr->b_size; 7641544Seschrock 7652688Smaybee if (!recycle) { 7662688Smaybee zio_buf_free(buf->b_data, size); 7672688Smaybee atomic_add_64(&arc.size, -size); 7682688Smaybee } 7691544Seschrock if (list_link_active(&buf->b_hdr->b_arc_node)) { 7701544Seschrock ASSERT(refcount_is_zero(&buf->b_hdr->b_refcnt)); 7711544Seschrock ASSERT(state != arc.anon); 7721544Seschrock ASSERT3U(state->lsize, >=, size); 7731544Seschrock atomic_add_64(&state->lsize, -size); 7741544Seschrock } 7751544Seschrock ASSERT3U(state->size, >=, size); 7761544Seschrock atomic_add_64(&state->size, -size); 7771544Seschrock buf->b_data = NULL; 7781544Seschrock ASSERT(buf->b_hdr->b_datacnt > 0); 7791544Seschrock buf->b_hdr->b_datacnt -= 1; 7801544Seschrock } 7811544Seschrock 7821544Seschrock /* only remove the buf if requested */ 7831544Seschrock if (!all) 7841544Seschrock return; 7851544Seschrock 7861544Seschrock /* remove the buf from the hdr list */ 7871544Seschrock for (bufp = &buf->b_hdr->b_buf; *bufp != buf; bufp = &(*bufp)->b_next) 7881544Seschrock continue; 7891544Seschrock *bufp = buf->b_next; 7901544Seschrock 7911544Seschrock ASSERT(buf->b_efunc == NULL); 7921544Seschrock 7931544Seschrock /* clean up the buf */ 7941544Seschrock buf->b_hdr = NULL; 7951544Seschrock kmem_cache_free(buf_cache, buf); 7961544Seschrock } 7971544Seschrock 7981544Seschrock static void 7991544Seschrock arc_hdr_destroy(arc_buf_hdr_t *hdr) 800789Sahrens { 801789Sahrens ASSERT(refcount_is_zero(&hdr->b_refcnt)); 802789Sahrens ASSERT3P(hdr->b_state, ==, arc.anon); 8031544Seschrock ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 804789Sahrens 805789Sahrens if (!BUF_EMPTY(hdr)) { 8061544Seschrock ASSERT(!HDR_IN_HASH_TABLE(hdr)); 807789Sahrens bzero(&hdr->b_dva, sizeof (dva_t)); 808789Sahrens hdr->b_birth = 0; 809789Sahrens hdr->b_cksum0 = 0; 810789Sahrens } 8111544Seschrock while (hdr->b_buf) { 812789Sahrens arc_buf_t *buf = hdr->b_buf; 813789Sahrens 8141544Seschrock if (buf->b_efunc) { 8151544Seschrock mutex_enter(&arc_eviction_mtx); 8161544Seschrock ASSERT(buf->b_hdr != NULL); 8172688Smaybee arc_buf_destroy(hdr->b_buf, FALSE, FALSE); 8181544Seschrock hdr->b_buf = buf->b_next; 8191544Seschrock buf->b_next = arc_eviction_list; 8201544Seschrock arc_eviction_list = buf; 8211544Seschrock mutex_exit(&arc_eviction_mtx); 8221544Seschrock } else { 8232688Smaybee arc_buf_destroy(hdr->b_buf, FALSE, TRUE); 8241544Seschrock } 825789Sahrens } 8261544Seschrock 827789Sahrens ASSERT(!list_link_active(&hdr->b_arc_node)); 828789Sahrens ASSERT3P(hdr->b_hash_next, ==, NULL); 829789Sahrens ASSERT3P(hdr->b_acb, ==, NULL); 830789Sahrens kmem_cache_free(hdr_cache, hdr); 831789Sahrens } 832789Sahrens 833789Sahrens void 834789Sahrens arc_buf_free(arc_buf_t *buf, void *tag) 835789Sahrens { 836789Sahrens arc_buf_hdr_t *hdr = buf->b_hdr; 8371544Seschrock int hashed = hdr->b_state != arc.anon; 8381544Seschrock 8391544Seschrock ASSERT(buf->b_efunc == NULL); 8401544Seschrock ASSERT(buf->b_data != NULL); 8411544Seschrock 8421544Seschrock if (hashed) { 8431544Seschrock kmutex_t *hash_lock = HDR_LOCK(hdr); 8441544Seschrock 8451544Seschrock mutex_enter(hash_lock); 8461544Seschrock (void) remove_reference(hdr, hash_lock, tag); 8471544Seschrock if (hdr->b_datacnt > 1) 8482688Smaybee arc_buf_destroy(buf, FALSE, TRUE); 8491544Seschrock else 8501544Seschrock hdr->b_flags |= ARC_BUF_AVAILABLE; 8511544Seschrock mutex_exit(hash_lock); 8521544Seschrock } else if (HDR_IO_IN_PROGRESS(hdr)) { 8531544Seschrock int destroy_hdr; 8541544Seschrock /* 8551544Seschrock * We are in the middle of an async write. Don't destroy 8561544Seschrock * this buffer unless the write completes before we finish 8571544Seschrock * decrementing the reference count. 8581544Seschrock */ 8591544Seschrock mutex_enter(&arc_eviction_mtx); 8601544Seschrock (void) remove_reference(hdr, NULL, tag); 8611544Seschrock ASSERT(refcount_is_zero(&hdr->b_refcnt)); 8621544Seschrock destroy_hdr = !HDR_IO_IN_PROGRESS(hdr); 8631544Seschrock mutex_exit(&arc_eviction_mtx); 8641544Seschrock if (destroy_hdr) 8651544Seschrock arc_hdr_destroy(hdr); 8661544Seschrock } else { 8671544Seschrock if (remove_reference(hdr, NULL, tag) > 0) { 8681544Seschrock ASSERT(HDR_IO_ERROR(hdr)); 8692688Smaybee arc_buf_destroy(buf, FALSE, TRUE); 8701544Seschrock } else { 8711544Seschrock arc_hdr_destroy(hdr); 8721544Seschrock } 8731544Seschrock } 8741544Seschrock } 8751544Seschrock 8761544Seschrock int 8771544Seschrock arc_buf_remove_ref(arc_buf_t *buf, void* tag) 8781544Seschrock { 8791544Seschrock arc_buf_hdr_t *hdr = buf->b_hdr; 880789Sahrens kmutex_t *hash_lock = HDR_LOCK(hdr); 8811544Seschrock int no_callback = (buf->b_efunc == NULL); 8821544Seschrock 8831544Seschrock if (hdr->b_state == arc.anon) { 8841544Seschrock arc_buf_free(buf, tag); 8851544Seschrock return (no_callback); 8861544Seschrock } 887789Sahrens 888789Sahrens mutex_enter(hash_lock); 8891544Seschrock ASSERT(hdr->b_state != arc.anon); 8901544Seschrock ASSERT(buf->b_data != NULL); 891789Sahrens 8921544Seschrock (void) remove_reference(hdr, hash_lock, tag); 8931544Seschrock if (hdr->b_datacnt > 1) { 8941544Seschrock if (no_callback) 8952688Smaybee arc_buf_destroy(buf, FALSE, TRUE); 8961544Seschrock } else if (no_callback) { 8971544Seschrock ASSERT(hdr->b_buf == buf && buf->b_next == NULL); 8981544Seschrock hdr->b_flags |= ARC_BUF_AVAILABLE; 899789Sahrens } 9001544Seschrock ASSERT(no_callback || hdr->b_datacnt > 1 || 9011544Seschrock refcount_is_zero(&hdr->b_refcnt)); 902789Sahrens mutex_exit(hash_lock); 9031544Seschrock return (no_callback); 904789Sahrens } 905789Sahrens 906789Sahrens int 907789Sahrens arc_buf_size(arc_buf_t *buf) 908789Sahrens { 909789Sahrens return (buf->b_hdr->b_size); 910789Sahrens } 911789Sahrens 912789Sahrens /* 913789Sahrens * Evict buffers from list until we've removed the specified number of 914789Sahrens * bytes. Move the removed buffers to the appropriate evict state. 9152688Smaybee * If the recycle flag is set, then attempt to "recycle" a buffer: 9162688Smaybee * - look for a buffer to evict that is `bytes' long. 9172688Smaybee * - return the data block from this buffer rather than freeing it. 9182688Smaybee * This flag is used by callers that are trying to make space for a 9192688Smaybee * new buffer in a full arc cache. 920789Sahrens */ 9212688Smaybee static void * 9222688Smaybee arc_evict(arc_state_t *state, int64_t bytes, boolean_t recycle) 923789Sahrens { 924789Sahrens arc_state_t *evicted_state; 9252688Smaybee uint64_t bytes_evicted = 0, skipped = 0, missed = 0; 926789Sahrens arc_buf_hdr_t *ab, *ab_prev; 927789Sahrens kmutex_t *hash_lock; 9282688Smaybee boolean_t have_lock; 9292688Smaybee void *steal = NULL; 930789Sahrens 9311544Seschrock ASSERT(state == arc.mru || state == arc.mfu); 932789Sahrens 9331544Seschrock evicted_state = (state == arc.mru) ? arc.mru_ghost : arc.mfu_ghost; 934789Sahrens 935789Sahrens mutex_enter(&state->mtx); 936789Sahrens mutex_enter(&evicted_state->mtx); 937789Sahrens 938789Sahrens for (ab = list_tail(&state->list); ab; ab = ab_prev) { 939789Sahrens ab_prev = list_prev(&state->list, ab); 9402391Smaybee /* prefetch buffers have a minimum lifespan */ 9412688Smaybee if (HDR_IO_IN_PROGRESS(ab) || 9422688Smaybee (ab->b_flags & (ARC_PREFETCH|ARC_INDIRECT) && 9432688Smaybee lbolt - ab->b_arc_access < arc_min_prefetch_lifespan)) { 9442391Smaybee skipped++; 9452391Smaybee continue; 9462391Smaybee } 9472688Smaybee if (recycle && (ab->b_size != bytes || ab->b_datacnt > 1)) 9482688Smaybee continue; 949789Sahrens hash_lock = HDR_LOCK(ab); 9502688Smaybee have_lock = MUTEX_HELD(hash_lock); 9512688Smaybee if (have_lock || mutex_tryenter(hash_lock)) { 952789Sahrens ASSERT3U(refcount_count(&ab->b_refcnt), ==, 0); 9531544Seschrock ASSERT(ab->b_datacnt > 0); 9541544Seschrock while (ab->b_buf) { 9551544Seschrock arc_buf_t *buf = ab->b_buf; 9562688Smaybee if (buf->b_data) { 9571544Seschrock bytes_evicted += ab->b_size; 9582688Smaybee if (recycle) 9592688Smaybee steal = buf->b_data; 9602688Smaybee } 9611544Seschrock if (buf->b_efunc) { 9621544Seschrock mutex_enter(&arc_eviction_mtx); 9632688Smaybee arc_buf_destroy(buf, recycle, FALSE); 9641544Seschrock ab->b_buf = buf->b_next; 9651544Seschrock buf->b_next = arc_eviction_list; 9661544Seschrock arc_eviction_list = buf; 9671544Seschrock mutex_exit(&arc_eviction_mtx); 9681544Seschrock } else { 9692688Smaybee arc_buf_destroy(buf, recycle, TRUE); 9701544Seschrock } 9711544Seschrock } 9721544Seschrock ASSERT(ab->b_datacnt == 0); 973789Sahrens arc_change_state(evicted_state, ab, hash_lock); 9741544Seschrock ASSERT(HDR_IN_HASH_TABLE(ab)); 9751544Seschrock ab->b_flags = ARC_IN_HASH_TABLE; 976789Sahrens DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, ab); 9772688Smaybee if (!have_lock) 9782688Smaybee mutex_exit(hash_lock); 9791544Seschrock if (bytes >= 0 && bytes_evicted >= bytes) 980789Sahrens break; 981789Sahrens } else { 9822688Smaybee missed += 1; 983789Sahrens } 984789Sahrens } 985789Sahrens mutex_exit(&evicted_state->mtx); 986789Sahrens mutex_exit(&state->mtx); 987789Sahrens 988789Sahrens if (bytes_evicted < bytes) 989789Sahrens dprintf("only evicted %lld bytes from %x", 990789Sahrens (longlong_t)bytes_evicted, state); 991789Sahrens 9922688Smaybee if (skipped) 9932688Smaybee atomic_add_64(&arc.evict_skip, skipped); 9942688Smaybee if (missed) 9952688Smaybee atomic_add_64(&arc.mutex_miss, missed); 9962688Smaybee return (steal); 997789Sahrens } 998789Sahrens 999789Sahrens /* 1000789Sahrens * Remove buffers from list until we've removed the specified number of 1001789Sahrens * bytes. Destroy the buffers that are removed. 1002789Sahrens */ 1003789Sahrens static void 10041544Seschrock arc_evict_ghost(arc_state_t *state, int64_t bytes) 1005789Sahrens { 1006789Sahrens arc_buf_hdr_t *ab, *ab_prev; 1007789Sahrens kmutex_t *hash_lock; 10081544Seschrock uint64_t bytes_deleted = 0; 10091544Seschrock uint_t bufs_skipped = 0; 1010789Sahrens 10111544Seschrock ASSERT(GHOST_STATE(state)); 1012789Sahrens top: 1013789Sahrens mutex_enter(&state->mtx); 1014789Sahrens for (ab = list_tail(&state->list); ab; ab = ab_prev) { 1015789Sahrens ab_prev = list_prev(&state->list, ab); 1016789Sahrens hash_lock = HDR_LOCK(ab); 1017789Sahrens if (mutex_tryenter(hash_lock)) { 10182391Smaybee ASSERT(!HDR_IO_IN_PROGRESS(ab)); 10191544Seschrock ASSERT(ab->b_buf == NULL); 1020789Sahrens arc_change_state(arc.anon, ab, hash_lock); 1021789Sahrens mutex_exit(hash_lock); 1022789Sahrens atomic_add_64(&arc.deleted, 1); 10231544Seschrock bytes_deleted += ab->b_size; 10241544Seschrock arc_hdr_destroy(ab); 1025789Sahrens DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, ab); 1026789Sahrens if (bytes >= 0 && bytes_deleted >= bytes) 1027789Sahrens break; 1028789Sahrens } else { 1029789Sahrens if (bytes < 0) { 1030789Sahrens mutex_exit(&state->mtx); 1031789Sahrens mutex_enter(hash_lock); 1032789Sahrens mutex_exit(hash_lock); 1033789Sahrens goto top; 1034789Sahrens } 1035789Sahrens bufs_skipped += 1; 1036789Sahrens } 1037789Sahrens } 1038789Sahrens mutex_exit(&state->mtx); 1039789Sahrens 1040789Sahrens if (bufs_skipped) { 10412688Smaybee atomic_add_64(&arc.mutex_miss, bufs_skipped); 1042789Sahrens ASSERT(bytes >= 0); 1043789Sahrens } 1044789Sahrens 1045789Sahrens if (bytes_deleted < bytes) 1046789Sahrens dprintf("only deleted %lld bytes from %p", 1047789Sahrens (longlong_t)bytes_deleted, state); 1048789Sahrens } 1049789Sahrens 1050789Sahrens static void 1051789Sahrens arc_adjust(void) 1052789Sahrens { 1053789Sahrens int64_t top_sz, mru_over, arc_over; 1054789Sahrens 10551544Seschrock top_sz = arc.anon->size + arc.mru->size; 1056789Sahrens 10571544Seschrock if (top_sz > arc.p && arc.mru->lsize > 0) { 10581544Seschrock int64_t toevict = MIN(arc.mru->lsize, top_sz-arc.p); 10592688Smaybee (void) arc_evict(arc.mru, toevict, FALSE); 10601544Seschrock top_sz = arc.anon->size + arc.mru->size; 1061789Sahrens } 1062789Sahrens 10631544Seschrock mru_over = top_sz + arc.mru_ghost->size - arc.c; 1064789Sahrens 1065789Sahrens if (mru_over > 0) { 10661544Seschrock if (arc.mru_ghost->lsize > 0) { 10671544Seschrock int64_t todelete = MIN(arc.mru_ghost->lsize, mru_over); 10681544Seschrock arc_evict_ghost(arc.mru_ghost, todelete); 1069789Sahrens } 1070789Sahrens } 1071789Sahrens 1072789Sahrens if ((arc_over = arc.size - arc.c) > 0) { 10731544Seschrock int64_t tbl_over; 1074789Sahrens 10751544Seschrock if (arc.mfu->lsize > 0) { 10761544Seschrock int64_t toevict = MIN(arc.mfu->lsize, arc_over); 10772688Smaybee (void) arc_evict(arc.mfu, toevict, FALSE); 1078789Sahrens } 1079789Sahrens 10801544Seschrock tbl_over = arc.size + arc.mru_ghost->lsize + 10811544Seschrock arc.mfu_ghost->lsize - arc.c*2; 1082789Sahrens 10831544Seschrock if (tbl_over > 0 && arc.mfu_ghost->lsize > 0) { 10841544Seschrock int64_t todelete = MIN(arc.mfu_ghost->lsize, tbl_over); 10851544Seschrock arc_evict_ghost(arc.mfu_ghost, todelete); 1086789Sahrens } 1087789Sahrens } 1088789Sahrens } 1089789Sahrens 10901544Seschrock static void 10911544Seschrock arc_do_user_evicts(void) 10921544Seschrock { 10931544Seschrock mutex_enter(&arc_eviction_mtx); 10941544Seschrock while (arc_eviction_list != NULL) { 10951544Seschrock arc_buf_t *buf = arc_eviction_list; 10961544Seschrock arc_eviction_list = buf->b_next; 10971544Seschrock buf->b_hdr = NULL; 10981544Seschrock mutex_exit(&arc_eviction_mtx); 10991544Seschrock 11001819Smaybee if (buf->b_efunc != NULL) 11011819Smaybee VERIFY(buf->b_efunc(buf) == 0); 11021544Seschrock 11031544Seschrock buf->b_efunc = NULL; 11041544Seschrock buf->b_private = NULL; 11051544Seschrock kmem_cache_free(buf_cache, buf); 11061544Seschrock mutex_enter(&arc_eviction_mtx); 11071544Seschrock } 11081544Seschrock mutex_exit(&arc_eviction_mtx); 11091544Seschrock } 11101544Seschrock 1111789Sahrens /* 1112789Sahrens * Flush all *evictable* data from the cache. 1113789Sahrens * NOTE: this will not touch "active" (i.e. referenced) data. 1114789Sahrens */ 1115789Sahrens void 1116789Sahrens arc_flush(void) 1117789Sahrens { 11182688Smaybee while (list_head(&arc.mru->list)) 11192688Smaybee (void) arc_evict(arc.mru, -1, FALSE); 11202688Smaybee while (list_head(&arc.mfu->list)) 11212688Smaybee (void) arc_evict(arc.mfu, -1, FALSE); 1122789Sahrens 11231544Seschrock arc_evict_ghost(arc.mru_ghost, -1); 11241544Seschrock arc_evict_ghost(arc.mfu_ghost, -1); 11251544Seschrock 11261544Seschrock mutex_enter(&arc_reclaim_thr_lock); 11271544Seschrock arc_do_user_evicts(); 11281544Seschrock mutex_exit(&arc_reclaim_thr_lock); 11291544Seschrock ASSERT(arc_eviction_list == NULL); 1130789Sahrens } 1131789Sahrens 11322391Smaybee int arc_kmem_reclaim_shift = 5; /* log2(fraction of arc to reclaim) */ 11332391Smaybee 1134789Sahrens void 1135789Sahrens arc_kmem_reclaim(void) 1136789Sahrens { 11372048Sstans uint64_t to_free; 11382048Sstans 1139789Sahrens /* 1140789Sahrens * We need arc_reclaim_lock because we don't want multiple 1141789Sahrens * threads trying to reclaim concurrently. 1142789Sahrens */ 1143789Sahrens 1144789Sahrens /* 1145789Sahrens * umem calls the reclaim func when we destroy the buf cache, 1146789Sahrens * which is after we do arc_fini(). So we set a flag to prevent 1147789Sahrens * accessing the destroyed mutexes and lists. 1148789Sahrens */ 1149789Sahrens if (arc_dead) 1150789Sahrens return; 1151789Sahrens 11521544Seschrock if (arc.c <= arc.c_min) 11531544Seschrock return; 11541544Seschrock 1155789Sahrens mutex_enter(&arc_reclaim_lock); 1156789Sahrens 11572048Sstans #ifdef _KERNEL 11582391Smaybee to_free = MAX(arc.c >> arc_kmem_reclaim_shift, ptob(needfree)); 11592048Sstans #else 11602391Smaybee to_free = arc.c >> arc_kmem_reclaim_shift; 11612048Sstans #endif 11622048Sstans if (arc.c > to_free) 11632048Sstans atomic_add_64(&arc.c, -to_free); 11642048Sstans else 11652048Sstans arc.c = arc.c_min; 11662048Sstans 11672391Smaybee atomic_add_64(&arc.p, -(arc.p >> arc_kmem_reclaim_shift)); 11681544Seschrock if (arc.c > arc.size) 11691544Seschrock arc.c = arc.size; 1170789Sahrens if (arc.c < arc.c_min) 1171789Sahrens arc.c = arc.c_min; 11721544Seschrock if (arc.p > arc.c) 11731544Seschrock arc.p = (arc.c >> 1); 11741544Seschrock ASSERT((int64_t)arc.p >= 0); 1175789Sahrens 1176789Sahrens arc_adjust(); 1177789Sahrens 1178789Sahrens mutex_exit(&arc_reclaim_lock); 1179789Sahrens } 1180789Sahrens 1181789Sahrens static int 1182789Sahrens arc_reclaim_needed(void) 1183789Sahrens { 1184789Sahrens uint64_t extra; 1185789Sahrens 1186789Sahrens #ifdef _KERNEL 11872048Sstans 11882048Sstans if (needfree) 11892048Sstans return (1); 11902048Sstans 1191789Sahrens /* 1192789Sahrens * take 'desfree' extra pages, so we reclaim sooner, rather than later 1193789Sahrens */ 1194789Sahrens extra = desfree; 1195789Sahrens 1196789Sahrens /* 1197789Sahrens * check that we're out of range of the pageout scanner. It starts to 1198789Sahrens * schedule paging if freemem is less than lotsfree and needfree. 1199789Sahrens * lotsfree is the high-water mark for pageout, and needfree is the 1200789Sahrens * number of needed free pages. We add extra pages here to make sure 1201789Sahrens * the scanner doesn't start up while we're freeing memory. 1202789Sahrens */ 1203789Sahrens if (freemem < lotsfree + needfree + extra) 1204789Sahrens return (1); 1205789Sahrens 1206789Sahrens /* 1207789Sahrens * check to make sure that swapfs has enough space so that anon 1208789Sahrens * reservations can still succeeed. anon_resvmem() checks that the 1209789Sahrens * availrmem is greater than swapfs_minfree, and the number of reserved 1210789Sahrens * swap pages. We also add a bit of extra here just to prevent 1211789Sahrens * circumstances from getting really dire. 1212789Sahrens */ 1213789Sahrens if (availrmem < swapfs_minfree + swapfs_reserve + extra) 1214789Sahrens return (1); 1215789Sahrens 12161936Smaybee #if defined(__i386) 1217789Sahrens /* 1218789Sahrens * If we're on an i386 platform, it's possible that we'll exhaust the 1219789Sahrens * kernel heap space before we ever run out of available physical 1220789Sahrens * memory. Most checks of the size of the heap_area compare against 1221789Sahrens * tune.t_minarmem, which is the minimum available real memory that we 1222789Sahrens * can have in the system. However, this is generally fixed at 25 pages 1223789Sahrens * which is so low that it's useless. In this comparison, we seek to 1224789Sahrens * calculate the total heap-size, and reclaim if more than 3/4ths of the 1225789Sahrens * heap is allocated. (Or, in the caclulation, if less than 1/4th is 1226789Sahrens * free) 1227789Sahrens */ 1228789Sahrens if (btop(vmem_size(heap_arena, VMEM_FREE)) < 1229789Sahrens (btop(vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC)) >> 2)) 1230789Sahrens return (1); 1231789Sahrens #endif 1232789Sahrens 1233789Sahrens #else 1234789Sahrens if (spa_get_random(100) == 0) 1235789Sahrens return (1); 1236789Sahrens #endif 1237789Sahrens return (0); 1238789Sahrens } 1239789Sahrens 1240789Sahrens static void 1241789Sahrens arc_kmem_reap_now(arc_reclaim_strategy_t strat) 1242789Sahrens { 1243789Sahrens size_t i; 1244789Sahrens kmem_cache_t *prev_cache = NULL; 1245789Sahrens extern kmem_cache_t *zio_buf_cache[]; 1246789Sahrens 12471484Sek110237 #ifdef _KERNEL 12481484Sek110237 /* 12491484Sek110237 * First purge some DNLC entries, in case the DNLC is using 12501484Sek110237 * up too much memory. 12511484Sek110237 */ 12521505Sek110237 dnlc_reduce_cache((void *)(uintptr_t)arc_reduce_dnlc_percent); 12531936Smaybee 12541936Smaybee #if defined(__i386) 12551936Smaybee /* 12561936Smaybee * Reclaim unused memory from all kmem caches. 12571936Smaybee */ 12581936Smaybee kmem_reap(); 12591936Smaybee #endif 12601484Sek110237 #endif 12611484Sek110237 1262789Sahrens /* 12631544Seschrock * An agressive reclamation will shrink the cache size as well as 12641544Seschrock * reap free buffers from the arc kmem caches. 1265789Sahrens */ 1266789Sahrens if (strat == ARC_RECLAIM_AGGR) 12671544Seschrock arc_kmem_reclaim(); 1268789Sahrens 1269789Sahrens for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) { 1270789Sahrens if (zio_buf_cache[i] != prev_cache) { 1271789Sahrens prev_cache = zio_buf_cache[i]; 1272789Sahrens kmem_cache_reap_now(zio_buf_cache[i]); 1273789Sahrens } 1274789Sahrens } 12751544Seschrock kmem_cache_reap_now(buf_cache); 12761544Seschrock kmem_cache_reap_now(hdr_cache); 1277789Sahrens } 1278789Sahrens 1279789Sahrens static void 1280789Sahrens arc_reclaim_thread(void) 1281789Sahrens { 1282789Sahrens clock_t growtime = 0; 1283789Sahrens arc_reclaim_strategy_t last_reclaim = ARC_RECLAIM_CONS; 1284789Sahrens callb_cpr_t cpr; 1285789Sahrens 1286789Sahrens CALLB_CPR_INIT(&cpr, &arc_reclaim_thr_lock, callb_generic_cpr, FTAG); 1287789Sahrens 1288789Sahrens mutex_enter(&arc_reclaim_thr_lock); 1289789Sahrens while (arc_thread_exit == 0) { 1290789Sahrens if (arc_reclaim_needed()) { 1291789Sahrens 1292789Sahrens if (arc.no_grow) { 1293789Sahrens if (last_reclaim == ARC_RECLAIM_CONS) { 1294789Sahrens last_reclaim = ARC_RECLAIM_AGGR; 1295789Sahrens } else { 1296789Sahrens last_reclaim = ARC_RECLAIM_CONS; 1297789Sahrens } 1298789Sahrens } else { 1299789Sahrens arc.no_grow = TRUE; 1300789Sahrens last_reclaim = ARC_RECLAIM_AGGR; 1301789Sahrens membar_producer(); 1302789Sahrens } 1303789Sahrens 1304789Sahrens /* reset the growth delay for every reclaim */ 1305789Sahrens growtime = lbolt + (arc_grow_retry * hz); 1306*2856Snd150628 ASSERT(growtime > 0); 1307789Sahrens 1308789Sahrens arc_kmem_reap_now(last_reclaim); 1309789Sahrens 1310789Sahrens } else if ((growtime > 0) && ((growtime - lbolt) <= 0)) { 1311789Sahrens arc.no_grow = FALSE; 1312789Sahrens } 1313789Sahrens 13141544Seschrock if (arc_eviction_list != NULL) 13151544Seschrock arc_do_user_evicts(); 13161544Seschrock 1317789Sahrens /* block until needed, or one second, whichever is shorter */ 1318789Sahrens CALLB_CPR_SAFE_BEGIN(&cpr); 1319789Sahrens (void) cv_timedwait(&arc_reclaim_thr_cv, 1320789Sahrens &arc_reclaim_thr_lock, (lbolt + hz)); 1321789Sahrens CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_thr_lock); 1322789Sahrens } 1323789Sahrens 1324789Sahrens arc_thread_exit = 0; 1325789Sahrens cv_broadcast(&arc_reclaim_thr_cv); 1326789Sahrens CALLB_CPR_EXIT(&cpr); /* drops arc_reclaim_thr_lock */ 1327789Sahrens thread_exit(); 1328789Sahrens } 1329789Sahrens 13301544Seschrock /* 13311544Seschrock * Adapt arc info given the number of bytes we are trying to add and 13321544Seschrock * the state that we are comming from. This function is only called 13331544Seschrock * when we are adding new content to the cache. 13341544Seschrock */ 1335789Sahrens static void 13361544Seschrock arc_adapt(int bytes, arc_state_t *state) 1337789Sahrens { 13381544Seschrock int mult; 13391544Seschrock 13401544Seschrock ASSERT(bytes > 0); 1341789Sahrens /* 13421544Seschrock * Adapt the target size of the MRU list: 13431544Seschrock * - if we just hit in the MRU ghost list, then increase 13441544Seschrock * the target size of the MRU list. 13451544Seschrock * - if we just hit in the MFU ghost list, then increase 13461544Seschrock * the target size of the MFU list by decreasing the 13471544Seschrock * target size of the MRU list. 1348789Sahrens */ 13491544Seschrock if (state == arc.mru_ghost) { 13501544Seschrock mult = ((arc.mru_ghost->size >= arc.mfu_ghost->size) ? 13511544Seschrock 1 : (arc.mfu_ghost->size/arc.mru_ghost->size)); 13521544Seschrock 13531544Seschrock arc.p = MIN(arc.c, arc.p + bytes * mult); 13541544Seschrock } else if (state == arc.mfu_ghost) { 13551544Seschrock mult = ((arc.mfu_ghost->size >= arc.mru_ghost->size) ? 13561544Seschrock 1 : (arc.mru_ghost->size/arc.mfu_ghost->size)); 13571544Seschrock 13581544Seschrock arc.p = MAX(0, (int64_t)arc.p - bytes * mult); 13591544Seschrock } 13601544Seschrock ASSERT((int64_t)arc.p >= 0); 1361789Sahrens 1362789Sahrens if (arc_reclaim_needed()) { 1363789Sahrens cv_signal(&arc_reclaim_thr_cv); 1364789Sahrens return; 1365789Sahrens } 1366789Sahrens 1367789Sahrens if (arc.no_grow) 1368789Sahrens return; 1369789Sahrens 13701544Seschrock if (arc.c >= arc.c_max) 13711544Seschrock return; 13721544Seschrock 1373789Sahrens /* 13741544Seschrock * If we're within (2 * maxblocksize) bytes of the target 13751544Seschrock * cache size, increment the target cache size 1376789Sahrens */ 13771544Seschrock if (arc.size > arc.c - (2ULL << SPA_MAXBLOCKSHIFT)) { 13781544Seschrock atomic_add_64(&arc.c, (int64_t)bytes); 1379789Sahrens if (arc.c > arc.c_max) 1380789Sahrens arc.c = arc.c_max; 13811544Seschrock else if (state == arc.anon) 13821544Seschrock atomic_add_64(&arc.p, (int64_t)bytes); 13831544Seschrock if (arc.p > arc.c) 13841544Seschrock arc.p = arc.c; 1385789Sahrens } 13861544Seschrock ASSERT((int64_t)arc.p >= 0); 1387789Sahrens } 1388789Sahrens 1389789Sahrens /* 13901544Seschrock * Check if the cache has reached its limits and eviction is required 13911544Seschrock * prior to insert. 1392789Sahrens */ 1393789Sahrens static int 1394789Sahrens arc_evict_needed() 1395789Sahrens { 1396789Sahrens if (arc_reclaim_needed()) 1397789Sahrens return (1); 1398789Sahrens 13991544Seschrock return (arc.size > arc.c); 1400789Sahrens } 1401789Sahrens 1402789Sahrens /* 14032688Smaybee * The buffer, supplied as the first argument, needs a data block. 14042688Smaybee * So, if we are at cache max, determine which cache should be victimized. 14052688Smaybee * We have the following cases: 1406789Sahrens * 14071544Seschrock * 1. Insert for MRU, p > sizeof(arc.anon + arc.mru) -> 1408789Sahrens * In this situation if we're out of space, but the resident size of the MFU is 1409789Sahrens * under the limit, victimize the MFU cache to satisfy this insertion request. 1410789Sahrens * 14111544Seschrock * 2. Insert for MRU, p <= sizeof(arc.anon + arc.mru) -> 1412789Sahrens * Here, we've used up all of the available space for the MRU, so we need to 1413789Sahrens * evict from our own cache instead. Evict from the set of resident MRU 1414789Sahrens * entries. 1415789Sahrens * 14161544Seschrock * 3. Insert for MFU (c - p) > sizeof(arc.mfu) -> 1417789Sahrens * c minus p represents the MFU space in the cache, since p is the size of the 1418789Sahrens * cache that is dedicated to the MRU. In this situation there's still space on 1419789Sahrens * the MFU side, so the MRU side needs to be victimized. 1420789Sahrens * 14211544Seschrock * 4. Insert for MFU (c - p) < sizeof(arc.mfu) -> 1422789Sahrens * MFU's resident set is consuming more space than it has been allotted. In 1423789Sahrens * this situation, we must victimize our own cache, the MFU, for this insertion. 1424789Sahrens */ 1425789Sahrens static void 14262688Smaybee arc_get_data_buf(arc_buf_t *buf) 1427789Sahrens { 14282688Smaybee arc_state_t *state = buf->b_hdr->b_state; 14292688Smaybee uint64_t size = buf->b_hdr->b_size; 14302688Smaybee 14312688Smaybee arc_adapt(size, state); 1432789Sahrens 14332688Smaybee /* 14342688Smaybee * We have not yet reached cache maximum size, 14352688Smaybee * just allocate a new buffer. 14362688Smaybee */ 14372688Smaybee if (!arc_evict_needed()) { 14382688Smaybee buf->b_data = zio_buf_alloc(size); 14392688Smaybee atomic_add_64(&arc.size, size); 14402688Smaybee goto out; 14412688Smaybee } 14422688Smaybee 14432688Smaybee /* 14442688Smaybee * If we are prefetching from the mfu ghost list, this buffer 14452688Smaybee * will end up on the mru list; so steal space from there. 14462688Smaybee */ 14472688Smaybee if (state == arc.mfu_ghost) 14482688Smaybee state = buf->b_hdr->b_flags & ARC_PREFETCH ? arc.mru : arc.mfu; 14492688Smaybee else if (state == arc.mru_ghost) 14502688Smaybee state = arc.mru; 1451789Sahrens 14522688Smaybee if (state == arc.mru || state == arc.anon) { 14532688Smaybee uint64_t mru_used = arc.anon->size + arc.mru->size; 14542688Smaybee state = (arc.p > mru_used) ? arc.mfu : arc.mru; 1455789Sahrens } else { 14562688Smaybee /* MFU cases */ 14572688Smaybee uint64_t mfu_space = arc.c - arc.p; 14582688Smaybee state = (mfu_space > arc.mfu->size) ? arc.mru : arc.mfu; 14592688Smaybee } 14602688Smaybee if ((buf->b_data = arc_evict(state, size, TRUE)) == NULL) { 14612688Smaybee (void) arc_evict(state, size, FALSE); 14622688Smaybee buf->b_data = zio_buf_alloc(size); 14632688Smaybee atomic_add_64(&arc.size, size); 14642688Smaybee atomic_add_64(&arc.recycle_miss, 1); 14652688Smaybee if (arc.size > arc.c) 14662688Smaybee arc_adjust(); 14672688Smaybee } 14682688Smaybee ASSERT(buf->b_data != NULL); 14692688Smaybee out: 14702688Smaybee /* 14712688Smaybee * Update the state size. Note that ghost states have a 14722688Smaybee * "ghost size" and so don't need to be updated. 14732688Smaybee */ 14742688Smaybee if (!GHOST_STATE(buf->b_hdr->b_state)) { 14752688Smaybee arc_buf_hdr_t *hdr = buf->b_hdr; 14762688Smaybee 14772688Smaybee atomic_add_64(&hdr->b_state->size, size); 14782688Smaybee if (list_link_active(&hdr->b_arc_node)) { 14792688Smaybee ASSERT(refcount_is_zero(&hdr->b_refcnt)); 14802688Smaybee atomic_add_64(&hdr->b_state->lsize, size); 1481789Sahrens } 1482789Sahrens } 1483789Sahrens } 1484789Sahrens 1485789Sahrens /* 1486789Sahrens * This routine is called whenever a buffer is accessed. 14871544Seschrock * NOTE: the hash lock is dropped in this function. 1488789Sahrens */ 1489789Sahrens static void 14902688Smaybee arc_access(arc_buf_hdr_t *buf, kmutex_t *hash_lock) 1491789Sahrens { 1492789Sahrens ASSERT(MUTEX_HELD(hash_lock)); 1493789Sahrens 1494789Sahrens if (buf->b_state == arc.anon) { 1495789Sahrens /* 1496789Sahrens * This buffer is not in the cache, and does not 1497789Sahrens * appear in our "ghost" list. Add the new buffer 1498789Sahrens * to the MRU state. 1499789Sahrens */ 1500789Sahrens 1501789Sahrens ASSERT(buf->b_arc_access == 0); 1502789Sahrens buf->b_arc_access = lbolt; 15031544Seschrock DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf); 15041544Seschrock arc_change_state(arc.mru, buf, hash_lock); 1505789Sahrens 15061544Seschrock } else if (buf->b_state == arc.mru) { 1507789Sahrens /* 15082391Smaybee * If this buffer is here because of a prefetch, then either: 15092391Smaybee * - clear the flag if this is a "referencing" read 15102391Smaybee * (any subsequent access will bump this into the MFU state). 15112391Smaybee * or 15122391Smaybee * - move the buffer to the head of the list if this is 15132391Smaybee * another prefetch (to make it less likely to be evicted). 1514789Sahrens */ 1515789Sahrens if ((buf->b_flags & ARC_PREFETCH) != 0) { 15162391Smaybee if (refcount_count(&buf->b_refcnt) == 0) { 15172391Smaybee ASSERT(list_link_active(&buf->b_arc_node)); 15182391Smaybee mutex_enter(&arc.mru->mtx); 15192391Smaybee list_remove(&arc.mru->list, buf); 15202391Smaybee list_insert_head(&arc.mru->list, buf); 15212391Smaybee mutex_exit(&arc.mru->mtx); 15222391Smaybee } else { 15232391Smaybee buf->b_flags &= ~ARC_PREFETCH; 15242391Smaybee atomic_add_64(&arc.mru->hits, 1); 15252391Smaybee } 15262391Smaybee buf->b_arc_access = lbolt; 1527789Sahrens return; 1528789Sahrens } 1529789Sahrens 1530789Sahrens /* 1531789Sahrens * This buffer has been "accessed" only once so far, 1532789Sahrens * but it is still in the cache. Move it to the MFU 1533789Sahrens * state. 1534789Sahrens */ 1535789Sahrens if (lbolt > buf->b_arc_access + ARC_MINTIME) { 1536789Sahrens /* 1537789Sahrens * More than 125ms have passed since we 1538789Sahrens * instantiated this buffer. Move it to the 1539789Sahrens * most frequently used state. 1540789Sahrens */ 1541789Sahrens buf->b_arc_access = lbolt; 15421544Seschrock DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 15431544Seschrock arc_change_state(arc.mfu, buf, hash_lock); 1544789Sahrens } 15451544Seschrock atomic_add_64(&arc.mru->hits, 1); 15461544Seschrock } else if (buf->b_state == arc.mru_ghost) { 1547789Sahrens arc_state_t *new_state; 1548789Sahrens /* 1549789Sahrens * This buffer has been "accessed" recently, but 1550789Sahrens * was evicted from the cache. Move it to the 1551789Sahrens * MFU state. 1552789Sahrens */ 1553789Sahrens 1554789Sahrens if (buf->b_flags & ARC_PREFETCH) { 15551544Seschrock new_state = arc.mru; 15562391Smaybee if (refcount_count(&buf->b_refcnt) > 0) 15572391Smaybee buf->b_flags &= ~ARC_PREFETCH; 15581544Seschrock DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, buf); 1559789Sahrens } else { 15601544Seschrock new_state = arc.mfu; 15611544Seschrock DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 1562789Sahrens } 1563789Sahrens 1564789Sahrens buf->b_arc_access = lbolt; 1565789Sahrens arc_change_state(new_state, buf, hash_lock); 1566789Sahrens 15671544Seschrock atomic_add_64(&arc.mru_ghost->hits, 1); 15681544Seschrock } else if (buf->b_state == arc.mfu) { 1569789Sahrens /* 1570789Sahrens * This buffer has been accessed more than once and is 1571789Sahrens * still in the cache. Keep it in the MFU state. 1572789Sahrens * 15732391Smaybee * NOTE: an add_reference() that occurred when we did 15742391Smaybee * the arc_read() will have kicked this off the list. 15752391Smaybee * If it was a prefetch, we will explicitly move it to 15762391Smaybee * the head of the list now. 1577789Sahrens */ 15782391Smaybee if ((buf->b_flags & ARC_PREFETCH) != 0) { 15792391Smaybee ASSERT(refcount_count(&buf->b_refcnt) == 0); 15802391Smaybee ASSERT(list_link_active(&buf->b_arc_node)); 15812391Smaybee mutex_enter(&arc.mfu->mtx); 15822391Smaybee list_remove(&arc.mfu->list, buf); 15832391Smaybee list_insert_head(&arc.mfu->list, buf); 15842391Smaybee mutex_exit(&arc.mfu->mtx); 15852391Smaybee } 15861544Seschrock atomic_add_64(&arc.mfu->hits, 1); 15872391Smaybee buf->b_arc_access = lbolt; 15881544Seschrock } else if (buf->b_state == arc.mfu_ghost) { 15892391Smaybee arc_state_t *new_state = arc.mfu; 1590789Sahrens /* 1591789Sahrens * This buffer has been accessed more than once but has 1592789Sahrens * been evicted from the cache. Move it back to the 1593789Sahrens * MFU state. 1594789Sahrens */ 1595789Sahrens 15962391Smaybee if (buf->b_flags & ARC_PREFETCH) { 15972391Smaybee /* 15982391Smaybee * This is a prefetch access... 15992391Smaybee * move this block back to the MRU state. 16002391Smaybee */ 16012391Smaybee ASSERT3U(refcount_count(&buf->b_refcnt), ==, 0); 16022391Smaybee new_state = arc.mru; 16032391Smaybee } 16042391Smaybee 1605789Sahrens buf->b_arc_access = lbolt; 16061544Seschrock DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, buf); 16072391Smaybee arc_change_state(new_state, buf, hash_lock); 1608789Sahrens 16091544Seschrock atomic_add_64(&arc.mfu_ghost->hits, 1); 1610789Sahrens } else { 1611789Sahrens ASSERT(!"invalid arc state"); 1612789Sahrens } 1613789Sahrens } 1614789Sahrens 1615789Sahrens /* a generic arc_done_func_t which you can use */ 1616789Sahrens /* ARGSUSED */ 1617789Sahrens void 1618789Sahrens arc_bcopy_func(zio_t *zio, arc_buf_t *buf, void *arg) 1619789Sahrens { 1620789Sahrens bcopy(buf->b_data, arg, buf->b_hdr->b_size); 16211544Seschrock VERIFY(arc_buf_remove_ref(buf, arg) == 1); 1622789Sahrens } 1623789Sahrens 1624789Sahrens /* a generic arc_done_func_t which you can use */ 1625789Sahrens void 1626789Sahrens arc_getbuf_func(zio_t *zio, arc_buf_t *buf, void *arg) 1627789Sahrens { 1628789Sahrens arc_buf_t **bufp = arg; 1629789Sahrens if (zio && zio->io_error) { 16301544Seschrock VERIFY(arc_buf_remove_ref(buf, arg) == 1); 1631789Sahrens *bufp = NULL; 1632789Sahrens } else { 1633789Sahrens *bufp = buf; 1634789Sahrens } 1635789Sahrens } 1636789Sahrens 1637789Sahrens static void 1638789Sahrens arc_read_done(zio_t *zio) 1639789Sahrens { 16401589Smaybee arc_buf_hdr_t *hdr, *found; 1641789Sahrens arc_buf_t *buf; 1642789Sahrens arc_buf_t *abuf; /* buffer we're assigning to callback */ 1643789Sahrens kmutex_t *hash_lock; 1644789Sahrens arc_callback_t *callback_list, *acb; 1645789Sahrens int freeable = FALSE; 1646789Sahrens 1647789Sahrens buf = zio->io_private; 1648789Sahrens hdr = buf->b_hdr; 1649789Sahrens 16501589Smaybee /* 16511589Smaybee * The hdr was inserted into hash-table and removed from lists 16521589Smaybee * prior to starting I/O. We should find this header, since 16531589Smaybee * it's in the hash table, and it should be legit since it's 16541589Smaybee * not possible to evict it during the I/O. The only possible 16551589Smaybee * reason for it not to be found is if we were freed during the 16561589Smaybee * read. 16571589Smaybee */ 16581589Smaybee found = buf_hash_find(zio->io_spa, &hdr->b_dva, hdr->b_birth, 1659789Sahrens &hash_lock); 1660789Sahrens 16611589Smaybee ASSERT((found == NULL && HDR_FREED_IN_READ(hdr) && hash_lock == NULL) || 16621589Smaybee (found == hdr && DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp)))); 1663789Sahrens 1664789Sahrens /* byteswap if necessary */ 1665789Sahrens callback_list = hdr->b_acb; 1666789Sahrens ASSERT(callback_list != NULL); 1667789Sahrens if (BP_SHOULD_BYTESWAP(zio->io_bp) && callback_list->acb_byteswap) 1668789Sahrens callback_list->acb_byteswap(buf->b_data, hdr->b_size); 1669789Sahrens 1670789Sahrens /* create copies of the data buffer for the callers */ 1671789Sahrens abuf = buf; 1672789Sahrens for (acb = callback_list; acb; acb = acb->acb_next) { 1673789Sahrens if (acb->acb_done) { 16742688Smaybee if (abuf == NULL) 16752688Smaybee abuf = arc_buf_clone(buf); 1676789Sahrens acb->acb_buf = abuf; 1677789Sahrens abuf = NULL; 1678789Sahrens } 1679789Sahrens } 1680789Sahrens hdr->b_acb = NULL; 1681789Sahrens hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 16821544Seschrock ASSERT(!HDR_BUF_AVAILABLE(hdr)); 16831544Seschrock if (abuf == buf) 16841544Seschrock hdr->b_flags |= ARC_BUF_AVAILABLE; 1685789Sahrens 1686789Sahrens ASSERT(refcount_is_zero(&hdr->b_refcnt) || callback_list != NULL); 1687789Sahrens 1688789Sahrens if (zio->io_error != 0) { 1689789Sahrens hdr->b_flags |= ARC_IO_ERROR; 1690789Sahrens if (hdr->b_state != arc.anon) 1691789Sahrens arc_change_state(arc.anon, hdr, hash_lock); 16921544Seschrock if (HDR_IN_HASH_TABLE(hdr)) 16931544Seschrock buf_hash_remove(hdr); 1694789Sahrens freeable = refcount_is_zero(&hdr->b_refcnt); 16952391Smaybee /* convert checksum errors into IO errors */ 16961544Seschrock if (zio->io_error == ECKSUM) 16971544Seschrock zio->io_error = EIO; 1698789Sahrens } 1699789Sahrens 17001544Seschrock /* 17012391Smaybee * Broadcast before we drop the hash_lock to avoid the possibility 17022391Smaybee * that the hdr (and hence the cv) might be freed before we get to 17032391Smaybee * the cv_broadcast(). 17041544Seschrock */ 17051544Seschrock cv_broadcast(&hdr->b_cv); 17061544Seschrock 17071589Smaybee if (hash_lock) { 1708789Sahrens /* 1709789Sahrens * Only call arc_access on anonymous buffers. This is because 1710789Sahrens * if we've issued an I/O for an evicted buffer, we've already 1711789Sahrens * called arc_access (to prevent any simultaneous readers from 1712789Sahrens * getting confused). 1713789Sahrens */ 1714789Sahrens if (zio->io_error == 0 && hdr->b_state == arc.anon) 17152688Smaybee arc_access(hdr, hash_lock); 17162688Smaybee mutex_exit(hash_lock); 1717789Sahrens } else { 1718789Sahrens /* 1719789Sahrens * This block was freed while we waited for the read to 1720789Sahrens * complete. It has been removed from the hash table and 1721789Sahrens * moved to the anonymous state (so that it won't show up 1722789Sahrens * in the cache). 1723789Sahrens */ 1724789Sahrens ASSERT3P(hdr->b_state, ==, arc.anon); 1725789Sahrens freeable = refcount_is_zero(&hdr->b_refcnt); 1726789Sahrens } 1727789Sahrens 1728789Sahrens /* execute each callback and free its structure */ 1729789Sahrens while ((acb = callback_list) != NULL) { 1730789Sahrens if (acb->acb_done) 1731789Sahrens acb->acb_done(zio, acb->acb_buf, acb->acb_private); 1732789Sahrens 1733789Sahrens if (acb->acb_zio_dummy != NULL) { 1734789Sahrens acb->acb_zio_dummy->io_error = zio->io_error; 1735789Sahrens zio_nowait(acb->acb_zio_dummy); 1736789Sahrens } 1737789Sahrens 1738789Sahrens callback_list = acb->acb_next; 1739789Sahrens kmem_free(acb, sizeof (arc_callback_t)); 1740789Sahrens } 1741789Sahrens 1742789Sahrens if (freeable) 17431544Seschrock arc_hdr_destroy(hdr); 1744789Sahrens } 1745789Sahrens 1746789Sahrens /* 1747789Sahrens * "Read" the block block at the specified DVA (in bp) via the 1748789Sahrens * cache. If the block is found in the cache, invoke the provided 1749789Sahrens * callback immediately and return. Note that the `zio' parameter 1750789Sahrens * in the callback will be NULL in this case, since no IO was 1751789Sahrens * required. If the block is not in the cache pass the read request 1752789Sahrens * on to the spa with a substitute callback function, so that the 1753789Sahrens * requested block will be added to the cache. 1754789Sahrens * 1755789Sahrens * If a read request arrives for a block that has a read in-progress, 1756789Sahrens * either wait for the in-progress read to complete (and return the 1757789Sahrens * results); or, if this is a read with a "done" func, add a record 1758789Sahrens * to the read to invoke the "done" func when the read completes, 1759789Sahrens * and return; or just return. 1760789Sahrens * 1761789Sahrens * arc_read_done() will invoke all the requested "done" functions 1762789Sahrens * for readers of this block. 1763789Sahrens */ 1764789Sahrens int 1765789Sahrens arc_read(zio_t *pio, spa_t *spa, blkptr_t *bp, arc_byteswap_func_t *swap, 1766789Sahrens arc_done_func_t *done, void *private, int priority, int flags, 17672391Smaybee uint32_t *arc_flags, zbookmark_t *zb) 1768789Sahrens { 1769789Sahrens arc_buf_hdr_t *hdr; 1770789Sahrens arc_buf_t *buf; 1771789Sahrens kmutex_t *hash_lock; 1772789Sahrens zio_t *rzio; 1773789Sahrens 1774789Sahrens top: 1775789Sahrens hdr = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_lock); 17761544Seschrock if (hdr && hdr->b_datacnt > 0) { 1777789Sahrens 17782391Smaybee *arc_flags |= ARC_CACHED; 17792391Smaybee 1780789Sahrens if (HDR_IO_IN_PROGRESS(hdr)) { 17812391Smaybee 17822391Smaybee if (*arc_flags & ARC_WAIT) { 17832391Smaybee cv_wait(&hdr->b_cv, hash_lock); 17842391Smaybee mutex_exit(hash_lock); 17852391Smaybee goto top; 17862391Smaybee } 17872391Smaybee ASSERT(*arc_flags & ARC_NOWAIT); 17882391Smaybee 17892391Smaybee if (done) { 1790789Sahrens arc_callback_t *acb = NULL; 1791789Sahrens 1792789Sahrens acb = kmem_zalloc(sizeof (arc_callback_t), 1793789Sahrens KM_SLEEP); 1794789Sahrens acb->acb_done = done; 1795789Sahrens acb->acb_private = private; 1796789Sahrens acb->acb_byteswap = swap; 1797789Sahrens if (pio != NULL) 1798789Sahrens acb->acb_zio_dummy = zio_null(pio, 1799789Sahrens spa, NULL, NULL, flags); 1800789Sahrens 1801789Sahrens ASSERT(acb->acb_done != NULL); 1802789Sahrens acb->acb_next = hdr->b_acb; 1803789Sahrens hdr->b_acb = acb; 1804789Sahrens add_reference(hdr, hash_lock, private); 1805789Sahrens mutex_exit(hash_lock); 1806789Sahrens return (0); 1807789Sahrens } 1808789Sahrens mutex_exit(hash_lock); 1809789Sahrens return (0); 1810789Sahrens } 1811789Sahrens 18121544Seschrock ASSERT(hdr->b_state == arc.mru || hdr->b_state == arc.mfu); 1813789Sahrens 18141544Seschrock if (done) { 18152688Smaybee add_reference(hdr, hash_lock, private); 18161544Seschrock /* 18171544Seschrock * If this block is already in use, create a new 18181544Seschrock * copy of the data so that we will be guaranteed 18191544Seschrock * that arc_release() will always succeed. 18201544Seschrock */ 18211544Seschrock buf = hdr->b_buf; 18221544Seschrock ASSERT(buf); 18231544Seschrock ASSERT(buf->b_data); 18242688Smaybee if (HDR_BUF_AVAILABLE(hdr)) { 18251544Seschrock ASSERT(buf->b_efunc == NULL); 18261544Seschrock hdr->b_flags &= ~ARC_BUF_AVAILABLE; 18272688Smaybee } else { 18282688Smaybee buf = arc_buf_clone(buf); 18291544Seschrock } 18302391Smaybee } else if (*arc_flags & ARC_PREFETCH && 18312391Smaybee refcount_count(&hdr->b_refcnt) == 0) { 18322391Smaybee hdr->b_flags |= ARC_PREFETCH; 1833789Sahrens } 1834789Sahrens DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); 18352688Smaybee arc_access(hdr, hash_lock); 18362688Smaybee mutex_exit(hash_lock); 1837789Sahrens atomic_add_64(&arc.hits, 1); 1838789Sahrens if (done) 1839789Sahrens done(NULL, buf, private); 1840789Sahrens } else { 1841789Sahrens uint64_t size = BP_GET_LSIZE(bp); 1842789Sahrens arc_callback_t *acb; 1843789Sahrens 1844789Sahrens if (hdr == NULL) { 1845789Sahrens /* this block is not in the cache */ 1846789Sahrens arc_buf_hdr_t *exists; 1847789Sahrens 1848789Sahrens buf = arc_buf_alloc(spa, size, private); 1849789Sahrens hdr = buf->b_hdr; 1850789Sahrens hdr->b_dva = *BP_IDENTITY(bp); 1851789Sahrens hdr->b_birth = bp->blk_birth; 1852789Sahrens hdr->b_cksum0 = bp->blk_cksum.zc_word[0]; 1853789Sahrens exists = buf_hash_insert(hdr, &hash_lock); 1854789Sahrens if (exists) { 1855789Sahrens /* somebody beat us to the hash insert */ 1856789Sahrens mutex_exit(hash_lock); 1857789Sahrens bzero(&hdr->b_dva, sizeof (dva_t)); 1858789Sahrens hdr->b_birth = 0; 1859789Sahrens hdr->b_cksum0 = 0; 18601544Seschrock (void) arc_buf_remove_ref(buf, private); 1861789Sahrens goto top; /* restart the IO request */ 1862789Sahrens } 18632391Smaybee /* if this is a prefetch, we don't have a reference */ 18642391Smaybee if (*arc_flags & ARC_PREFETCH) { 18652391Smaybee (void) remove_reference(hdr, hash_lock, 18662391Smaybee private); 18672391Smaybee hdr->b_flags |= ARC_PREFETCH; 18682391Smaybee } 18692391Smaybee if (BP_GET_LEVEL(bp) > 0) 18702391Smaybee hdr->b_flags |= ARC_INDIRECT; 1871789Sahrens } else { 1872789Sahrens /* this block is in the ghost cache */ 18731544Seschrock ASSERT(GHOST_STATE(hdr->b_state)); 18741544Seschrock ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 18752391Smaybee ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 0); 18762391Smaybee ASSERT(hdr->b_buf == NULL); 1877789Sahrens 18782391Smaybee /* if this is a prefetch, we don't have a reference */ 18792391Smaybee if (*arc_flags & ARC_PREFETCH) 18802391Smaybee hdr->b_flags |= ARC_PREFETCH; 18812391Smaybee else 18822391Smaybee add_reference(hdr, hash_lock, private); 1883789Sahrens buf = kmem_cache_alloc(buf_cache, KM_SLEEP); 18841544Seschrock buf->b_hdr = hdr; 18852688Smaybee buf->b_data = NULL; 18861544Seschrock buf->b_efunc = NULL; 18871544Seschrock buf->b_private = NULL; 18881544Seschrock buf->b_next = NULL; 18891544Seschrock hdr->b_buf = buf; 18902688Smaybee arc_get_data_buf(buf); 18911544Seschrock ASSERT(hdr->b_datacnt == 0); 18921544Seschrock hdr->b_datacnt = 1; 18932391Smaybee 1894789Sahrens } 1895789Sahrens 1896789Sahrens acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP); 1897789Sahrens acb->acb_done = done; 1898789Sahrens acb->acb_private = private; 1899789Sahrens acb->acb_byteswap = swap; 1900789Sahrens 1901789Sahrens ASSERT(hdr->b_acb == NULL); 1902789Sahrens hdr->b_acb = acb; 1903789Sahrens hdr->b_flags |= ARC_IO_IN_PROGRESS; 1904789Sahrens 1905789Sahrens /* 1906789Sahrens * If the buffer has been evicted, migrate it to a present state 1907789Sahrens * before issuing the I/O. Once we drop the hash-table lock, 1908789Sahrens * the header will be marked as I/O in progress and have an 1909789Sahrens * attached buffer. At this point, anybody who finds this 1910789Sahrens * buffer ought to notice that it's legit but has a pending I/O. 1911789Sahrens */ 1912789Sahrens 19131544Seschrock if (GHOST_STATE(hdr->b_state)) 19142688Smaybee arc_access(hdr, hash_lock); 19152688Smaybee mutex_exit(hash_lock); 1916789Sahrens 1917789Sahrens ASSERT3U(hdr->b_size, ==, size); 19181596Sahrens DTRACE_PROBE3(arc__miss, blkptr_t *, bp, uint64_t, size, 19191596Sahrens zbookmark_t *, zb); 1920789Sahrens atomic_add_64(&arc.misses, 1); 19211544Seschrock 1922789Sahrens rzio = zio_read(pio, spa, bp, buf->b_data, size, 19231544Seschrock arc_read_done, buf, priority, flags, zb); 1924789Sahrens 19252391Smaybee if (*arc_flags & ARC_WAIT) 1926789Sahrens return (zio_wait(rzio)); 1927789Sahrens 19282391Smaybee ASSERT(*arc_flags & ARC_NOWAIT); 1929789Sahrens zio_nowait(rzio); 1930789Sahrens } 1931789Sahrens return (0); 1932789Sahrens } 1933789Sahrens 1934789Sahrens /* 1935789Sahrens * arc_read() variant to support pool traversal. If the block is already 1936789Sahrens * in the ARC, make a copy of it; otherwise, the caller will do the I/O. 1937789Sahrens * The idea is that we don't want pool traversal filling up memory, but 1938789Sahrens * if the ARC already has the data anyway, we shouldn't pay for the I/O. 1939789Sahrens */ 1940789Sahrens int 1941789Sahrens arc_tryread(spa_t *spa, blkptr_t *bp, void *data) 1942789Sahrens { 1943789Sahrens arc_buf_hdr_t *hdr; 1944789Sahrens kmutex_t *hash_mtx; 1945789Sahrens int rc = 0; 1946789Sahrens 1947789Sahrens hdr = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_mtx); 1948789Sahrens 19491544Seschrock if (hdr && hdr->b_datacnt > 0 && !HDR_IO_IN_PROGRESS(hdr)) { 19501544Seschrock arc_buf_t *buf = hdr->b_buf; 19511544Seschrock 19521544Seschrock ASSERT(buf); 19531544Seschrock while (buf->b_data == NULL) { 19541544Seschrock buf = buf->b_next; 19551544Seschrock ASSERT(buf); 19561544Seschrock } 19571544Seschrock bcopy(buf->b_data, data, hdr->b_size); 19581544Seschrock } else { 1959789Sahrens rc = ENOENT; 19601544Seschrock } 1961789Sahrens 1962789Sahrens if (hash_mtx) 1963789Sahrens mutex_exit(hash_mtx); 1964789Sahrens 1965789Sahrens return (rc); 1966789Sahrens } 1967789Sahrens 19681544Seschrock void 19691544Seschrock arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private) 19701544Seschrock { 19711544Seschrock ASSERT(buf->b_hdr != NULL); 19721544Seschrock ASSERT(buf->b_hdr->b_state != arc.anon); 19731544Seschrock ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt) || func == NULL); 19741544Seschrock buf->b_efunc = func; 19751544Seschrock buf->b_private = private; 19761544Seschrock } 19771544Seschrock 19781544Seschrock /* 19791544Seschrock * This is used by the DMU to let the ARC know that a buffer is 19801544Seschrock * being evicted, so the ARC should clean up. If this arc buf 19811544Seschrock * is not yet in the evicted state, it will be put there. 19821544Seschrock */ 19831544Seschrock int 19841544Seschrock arc_buf_evict(arc_buf_t *buf) 19851544Seschrock { 19862724Smaybee arc_buf_hdr_t *hdr = buf->b_hdr; 19871544Seschrock kmutex_t *hash_lock; 19881544Seschrock arc_buf_t **bufp; 19891544Seschrock 19901544Seschrock if (hdr == NULL) { 19911544Seschrock /* 19921544Seschrock * We are in arc_do_user_evicts(). 19931544Seschrock */ 19941544Seschrock ASSERT(buf->b_data == NULL); 19951544Seschrock return (0); 19961544Seschrock } 19971544Seschrock 19981544Seschrock hash_lock = HDR_LOCK(hdr); 19991544Seschrock mutex_enter(hash_lock); 20001544Seschrock 20012724Smaybee if (buf->b_data == NULL) { 20022724Smaybee /* 20032724Smaybee * We are on the eviction list. 20042724Smaybee */ 20052724Smaybee mutex_exit(hash_lock); 20062724Smaybee mutex_enter(&arc_eviction_mtx); 20072724Smaybee if (buf->b_hdr == NULL) { 20082724Smaybee /* 20092724Smaybee * We are already in arc_do_user_evicts(). 20102724Smaybee */ 20112724Smaybee mutex_exit(&arc_eviction_mtx); 20122724Smaybee return (0); 20132724Smaybee } else { 20142724Smaybee arc_buf_t copy = *buf; /* structure assignment */ 20152724Smaybee /* 20162724Smaybee * Process this buffer now 20172724Smaybee * but let arc_do_user_evicts() do the reaping. 20182724Smaybee */ 20192724Smaybee buf->b_efunc = NULL; 20202724Smaybee mutex_exit(&arc_eviction_mtx); 20212724Smaybee VERIFY(copy.b_efunc(©) == 0); 20222724Smaybee return (1); 20232724Smaybee } 20242724Smaybee } 20252724Smaybee 20262724Smaybee ASSERT(buf->b_hdr == hdr); 20272724Smaybee ASSERT3U(refcount_count(&hdr->b_refcnt), <, hdr->b_datacnt); 20281544Seschrock ASSERT(hdr->b_state == arc.mru || hdr->b_state == arc.mfu); 20291544Seschrock 20301544Seschrock /* 20311544Seschrock * Pull this buffer off of the hdr 20321544Seschrock */ 20331544Seschrock bufp = &hdr->b_buf; 20341544Seschrock while (*bufp != buf) 20351544Seschrock bufp = &(*bufp)->b_next; 20361544Seschrock *bufp = buf->b_next; 20371544Seschrock 20381544Seschrock ASSERT(buf->b_data != NULL); 20391544Seschrock buf->b_hdr = hdr; 20402688Smaybee arc_buf_destroy(buf, FALSE, FALSE); 20411544Seschrock 20421544Seschrock if (hdr->b_datacnt == 0) { 20431544Seschrock arc_state_t *old_state = hdr->b_state; 20441544Seschrock arc_state_t *evicted_state; 20451544Seschrock 20461544Seschrock ASSERT(refcount_is_zero(&hdr->b_refcnt)); 20471544Seschrock 20481544Seschrock evicted_state = 20491544Seschrock (old_state == arc.mru) ? arc.mru_ghost : arc.mfu_ghost; 20501544Seschrock 20511544Seschrock mutex_enter(&old_state->mtx); 20521544Seschrock mutex_enter(&evicted_state->mtx); 20531544Seschrock 20541544Seschrock arc_change_state(evicted_state, hdr, hash_lock); 20551544Seschrock ASSERT(HDR_IN_HASH_TABLE(hdr)); 20561544Seschrock hdr->b_flags = ARC_IN_HASH_TABLE; 20571544Seschrock 20581544Seschrock mutex_exit(&evicted_state->mtx); 20591544Seschrock mutex_exit(&old_state->mtx); 20601544Seschrock } 20611544Seschrock mutex_exit(hash_lock); 20621819Smaybee 20631544Seschrock VERIFY(buf->b_efunc(buf) == 0); 20641544Seschrock buf->b_efunc = NULL; 20651544Seschrock buf->b_private = NULL; 20661544Seschrock buf->b_hdr = NULL; 20671544Seschrock kmem_cache_free(buf_cache, buf); 20681544Seschrock return (1); 20691544Seschrock } 20701544Seschrock 2071789Sahrens /* 2072789Sahrens * Release this buffer from the cache. This must be done 2073789Sahrens * after a read and prior to modifying the buffer contents. 2074789Sahrens * If the buffer has more than one reference, we must make 2075789Sahrens * make a new hdr for the buffer. 2076789Sahrens */ 2077789Sahrens void 2078789Sahrens arc_release(arc_buf_t *buf, void *tag) 2079789Sahrens { 2080789Sahrens arc_buf_hdr_t *hdr = buf->b_hdr; 2081789Sahrens kmutex_t *hash_lock = HDR_LOCK(hdr); 2082789Sahrens 2083789Sahrens /* this buffer is not on any list */ 2084789Sahrens ASSERT(refcount_count(&hdr->b_refcnt) > 0); 2085789Sahrens 2086789Sahrens if (hdr->b_state == arc.anon) { 2087789Sahrens /* this buffer is already released */ 2088789Sahrens ASSERT3U(refcount_count(&hdr->b_refcnt), ==, 1); 2089789Sahrens ASSERT(BUF_EMPTY(hdr)); 20901544Seschrock ASSERT(buf->b_efunc == NULL); 2091789Sahrens return; 2092789Sahrens } 2093789Sahrens 2094789Sahrens mutex_enter(hash_lock); 2095789Sahrens 20961544Seschrock /* 20971544Seschrock * Do we have more than one buf? 20981544Seschrock */ 20991544Seschrock if (hdr->b_buf != buf || buf->b_next != NULL) { 2100789Sahrens arc_buf_hdr_t *nhdr; 2101789Sahrens arc_buf_t **bufp; 2102789Sahrens uint64_t blksz = hdr->b_size; 2103789Sahrens spa_t *spa = hdr->b_spa; 2104789Sahrens 21051544Seschrock ASSERT(hdr->b_datacnt > 1); 2106789Sahrens /* 2107789Sahrens * Pull the data off of this buf and attach it to 2108789Sahrens * a new anonymous buf. 2109789Sahrens */ 21101544Seschrock (void) remove_reference(hdr, hash_lock, tag); 2111789Sahrens bufp = &hdr->b_buf; 21121544Seschrock while (*bufp != buf) 2113789Sahrens bufp = &(*bufp)->b_next; 2114789Sahrens *bufp = (*bufp)->b_next; 21151544Seschrock 2116789Sahrens ASSERT3U(hdr->b_state->size, >=, hdr->b_size); 2117789Sahrens atomic_add_64(&hdr->b_state->size, -hdr->b_size); 21181544Seschrock if (refcount_is_zero(&hdr->b_refcnt)) { 21191544Seschrock ASSERT3U(hdr->b_state->lsize, >=, hdr->b_size); 21201544Seschrock atomic_add_64(&hdr->b_state->lsize, -hdr->b_size); 21211544Seschrock } 21221544Seschrock hdr->b_datacnt -= 1; 21231544Seschrock 2124789Sahrens mutex_exit(hash_lock); 2125789Sahrens 2126789Sahrens nhdr = kmem_cache_alloc(hdr_cache, KM_SLEEP); 2127789Sahrens nhdr->b_size = blksz; 2128789Sahrens nhdr->b_spa = spa; 2129789Sahrens nhdr->b_buf = buf; 2130789Sahrens nhdr->b_state = arc.anon; 2131789Sahrens nhdr->b_arc_access = 0; 2132789Sahrens nhdr->b_flags = 0; 21331544Seschrock nhdr->b_datacnt = 1; 2134789Sahrens buf->b_hdr = nhdr; 2135789Sahrens buf->b_next = NULL; 2136789Sahrens (void) refcount_add(&nhdr->b_refcnt, tag); 2137789Sahrens atomic_add_64(&arc.anon->size, blksz); 2138789Sahrens 2139789Sahrens hdr = nhdr; 2140789Sahrens } else { 21411544Seschrock ASSERT(refcount_count(&hdr->b_refcnt) == 1); 2142789Sahrens ASSERT(!list_link_active(&hdr->b_arc_node)); 2143789Sahrens ASSERT(!HDR_IO_IN_PROGRESS(hdr)); 2144789Sahrens arc_change_state(arc.anon, hdr, hash_lock); 2145789Sahrens hdr->b_arc_access = 0; 2146789Sahrens mutex_exit(hash_lock); 2147789Sahrens bzero(&hdr->b_dva, sizeof (dva_t)); 2148789Sahrens hdr->b_birth = 0; 2149789Sahrens hdr->b_cksum0 = 0; 2150789Sahrens } 21511544Seschrock buf->b_efunc = NULL; 21521544Seschrock buf->b_private = NULL; 2153789Sahrens } 2154789Sahrens 2155789Sahrens int 2156789Sahrens arc_released(arc_buf_t *buf) 2157789Sahrens { 21581544Seschrock return (buf->b_data != NULL && buf->b_hdr->b_state == arc.anon); 21591544Seschrock } 21601544Seschrock 21611544Seschrock int 21621544Seschrock arc_has_callback(arc_buf_t *buf) 21631544Seschrock { 21641544Seschrock return (buf->b_efunc != NULL); 2165789Sahrens } 2166789Sahrens 21671544Seschrock #ifdef ZFS_DEBUG 21681544Seschrock int 21691544Seschrock arc_referenced(arc_buf_t *buf) 21701544Seschrock { 21711544Seschrock return (refcount_count(&buf->b_hdr->b_refcnt)); 21721544Seschrock } 21731544Seschrock #endif 21741544Seschrock 2175789Sahrens static void 2176789Sahrens arc_write_done(zio_t *zio) 2177789Sahrens { 2178789Sahrens arc_buf_t *buf; 2179789Sahrens arc_buf_hdr_t *hdr; 2180789Sahrens arc_callback_t *acb; 2181789Sahrens 2182789Sahrens buf = zio->io_private; 2183789Sahrens hdr = buf->b_hdr; 2184789Sahrens acb = hdr->b_acb; 2185789Sahrens hdr->b_acb = NULL; 21861544Seschrock ASSERT(acb != NULL); 2187789Sahrens 2188789Sahrens /* this buffer is on no lists and is not in the hash table */ 2189789Sahrens ASSERT3P(hdr->b_state, ==, arc.anon); 2190789Sahrens 2191789Sahrens hdr->b_dva = *BP_IDENTITY(zio->io_bp); 2192789Sahrens hdr->b_birth = zio->io_bp->blk_birth; 2193789Sahrens hdr->b_cksum0 = zio->io_bp->blk_cksum.zc_word[0]; 21941544Seschrock /* 21951544Seschrock * If the block to be written was all-zero, we may have 21961544Seschrock * compressed it away. In this case no write was performed 21971544Seschrock * so there will be no dva/birth-date/checksum. The buffer 21981544Seschrock * must therefor remain anonymous (and uncached). 21991544Seschrock */ 2200789Sahrens if (!BUF_EMPTY(hdr)) { 2201789Sahrens arc_buf_hdr_t *exists; 2202789Sahrens kmutex_t *hash_lock; 2203789Sahrens 2204789Sahrens exists = buf_hash_insert(hdr, &hash_lock); 2205789Sahrens if (exists) { 2206789Sahrens /* 2207789Sahrens * This can only happen if we overwrite for 2208789Sahrens * sync-to-convergence, because we remove 2209789Sahrens * buffers from the hash table when we arc_free(). 2210789Sahrens */ 2211789Sahrens ASSERT(DVA_EQUAL(BP_IDENTITY(&zio->io_bp_orig), 2212789Sahrens BP_IDENTITY(zio->io_bp))); 2213789Sahrens ASSERT3U(zio->io_bp_orig.blk_birth, ==, 2214789Sahrens zio->io_bp->blk_birth); 2215789Sahrens 2216789Sahrens ASSERT(refcount_is_zero(&exists->b_refcnt)); 2217789Sahrens arc_change_state(arc.anon, exists, hash_lock); 2218789Sahrens mutex_exit(hash_lock); 22191544Seschrock arc_hdr_destroy(exists); 2220789Sahrens exists = buf_hash_insert(hdr, &hash_lock); 2221789Sahrens ASSERT3P(exists, ==, NULL); 2222789Sahrens } 22231544Seschrock hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 22242688Smaybee arc_access(hdr, hash_lock); 22252688Smaybee mutex_exit(hash_lock); 22261544Seschrock } else if (acb->acb_done == NULL) { 22271544Seschrock int destroy_hdr; 22281544Seschrock /* 22291544Seschrock * This is an anonymous buffer with no user callback, 22301544Seschrock * destroy it if there are no active references. 22311544Seschrock */ 22321544Seschrock mutex_enter(&arc_eviction_mtx); 22331544Seschrock destroy_hdr = refcount_is_zero(&hdr->b_refcnt); 22341544Seschrock hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 22351544Seschrock mutex_exit(&arc_eviction_mtx); 22361544Seschrock if (destroy_hdr) 22371544Seschrock arc_hdr_destroy(hdr); 22381544Seschrock } else { 22391544Seschrock hdr->b_flags &= ~ARC_IO_IN_PROGRESS; 2240789Sahrens } 22411544Seschrock 22421544Seschrock if (acb->acb_done) { 2243789Sahrens ASSERT(!refcount_is_zero(&hdr->b_refcnt)); 2244789Sahrens acb->acb_done(zio, buf, acb->acb_private); 2245789Sahrens } 2246789Sahrens 22471544Seschrock kmem_free(acb, sizeof (arc_callback_t)); 2248789Sahrens } 2249789Sahrens 2250789Sahrens int 22511775Sbillm arc_write(zio_t *pio, spa_t *spa, int checksum, int compress, int ncopies, 2252789Sahrens uint64_t txg, blkptr_t *bp, arc_buf_t *buf, 2253789Sahrens arc_done_func_t *done, void *private, int priority, int flags, 22541544Seschrock uint32_t arc_flags, zbookmark_t *zb) 2255789Sahrens { 2256789Sahrens arc_buf_hdr_t *hdr = buf->b_hdr; 2257789Sahrens arc_callback_t *acb; 2258789Sahrens zio_t *rzio; 2259789Sahrens 2260789Sahrens /* this is a private buffer - no locking required */ 2261789Sahrens ASSERT3P(hdr->b_state, ==, arc.anon); 2262789Sahrens ASSERT(BUF_EMPTY(hdr)); 2263789Sahrens ASSERT(!HDR_IO_ERROR(hdr)); 22642237Smaybee ASSERT((hdr->b_flags & ARC_IO_IN_PROGRESS) == 0); 22652237Smaybee ASSERT(hdr->b_acb == 0); 2266789Sahrens acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP); 2267789Sahrens acb->acb_done = done; 2268789Sahrens acb->acb_private = private; 2269789Sahrens acb->acb_byteswap = (arc_byteswap_func_t *)-1; 2270789Sahrens hdr->b_acb = acb; 22711544Seschrock hdr->b_flags |= ARC_IO_IN_PROGRESS; 22721775Sbillm rzio = zio_write(pio, spa, checksum, compress, ncopies, txg, bp, 22731544Seschrock buf->b_data, hdr->b_size, arc_write_done, buf, priority, flags, zb); 2274789Sahrens 2275789Sahrens if (arc_flags & ARC_WAIT) 2276789Sahrens return (zio_wait(rzio)); 2277789Sahrens 2278789Sahrens ASSERT(arc_flags & ARC_NOWAIT); 2279789Sahrens zio_nowait(rzio); 2280789Sahrens 2281789Sahrens return (0); 2282789Sahrens } 2283789Sahrens 2284789Sahrens int 2285789Sahrens arc_free(zio_t *pio, spa_t *spa, uint64_t txg, blkptr_t *bp, 2286789Sahrens zio_done_func_t *done, void *private, uint32_t arc_flags) 2287789Sahrens { 2288789Sahrens arc_buf_hdr_t *ab; 2289789Sahrens kmutex_t *hash_lock; 2290789Sahrens zio_t *zio; 2291789Sahrens 2292789Sahrens /* 2293789Sahrens * If this buffer is in the cache, release it, so it 2294789Sahrens * can be re-used. 2295789Sahrens */ 2296789Sahrens ab = buf_hash_find(spa, BP_IDENTITY(bp), bp->blk_birth, &hash_lock); 2297789Sahrens if (ab != NULL) { 2298789Sahrens /* 2299789Sahrens * The checksum of blocks to free is not always 2300789Sahrens * preserved (eg. on the deadlist). However, if it is 2301789Sahrens * nonzero, it should match what we have in the cache. 2302789Sahrens */ 2303789Sahrens ASSERT(bp->blk_cksum.zc_word[0] == 0 || 2304789Sahrens ab->b_cksum0 == bp->blk_cksum.zc_word[0]); 23051990Smaybee if (ab->b_state != arc.anon) 23061990Smaybee arc_change_state(arc.anon, ab, hash_lock); 23072391Smaybee if (HDR_IO_IN_PROGRESS(ab)) { 23082391Smaybee /* 23092391Smaybee * This should only happen when we prefetch. 23102391Smaybee */ 23112391Smaybee ASSERT(ab->b_flags & ARC_PREFETCH); 23122391Smaybee ASSERT3U(ab->b_datacnt, ==, 1); 23132391Smaybee ab->b_flags |= ARC_FREED_IN_READ; 23142391Smaybee if (HDR_IN_HASH_TABLE(ab)) 23152391Smaybee buf_hash_remove(ab); 23162391Smaybee ab->b_arc_access = 0; 23172391Smaybee bzero(&ab->b_dva, sizeof (dva_t)); 23182391Smaybee ab->b_birth = 0; 23192391Smaybee ab->b_cksum0 = 0; 23202391Smaybee ab->b_buf->b_efunc = NULL; 23212391Smaybee ab->b_buf->b_private = NULL; 23222391Smaybee mutex_exit(hash_lock); 23232391Smaybee } else if (refcount_is_zero(&ab->b_refcnt)) { 2324789Sahrens mutex_exit(hash_lock); 23251544Seschrock arc_hdr_destroy(ab); 2326789Sahrens atomic_add_64(&arc.deleted, 1); 2327789Sahrens } else { 23281589Smaybee /* 23292391Smaybee * We still have an active reference on this 23302391Smaybee * buffer. This can happen, e.g., from 23312391Smaybee * dbuf_unoverride(). 23321589Smaybee */ 23332391Smaybee ASSERT(!HDR_IN_HASH_TABLE(ab)); 2334789Sahrens ab->b_arc_access = 0; 2335789Sahrens bzero(&ab->b_dva, sizeof (dva_t)); 2336789Sahrens ab->b_birth = 0; 2337789Sahrens ab->b_cksum0 = 0; 23381544Seschrock ab->b_buf->b_efunc = NULL; 23391544Seschrock ab->b_buf->b_private = NULL; 2340789Sahrens mutex_exit(hash_lock); 2341789Sahrens } 2342789Sahrens } 2343789Sahrens 2344789Sahrens zio = zio_free(pio, spa, txg, bp, done, private); 2345789Sahrens 2346789Sahrens if (arc_flags & ARC_WAIT) 2347789Sahrens return (zio_wait(zio)); 2348789Sahrens 2349789Sahrens ASSERT(arc_flags & ARC_NOWAIT); 2350789Sahrens zio_nowait(zio); 2351789Sahrens 2352789Sahrens return (0); 2353789Sahrens } 2354789Sahrens 2355789Sahrens void 2356789Sahrens arc_tempreserve_clear(uint64_t tempreserve) 2357789Sahrens { 2358789Sahrens atomic_add_64(&arc_tempreserve, -tempreserve); 2359789Sahrens ASSERT((int64_t)arc_tempreserve >= 0); 2360789Sahrens } 2361789Sahrens 2362789Sahrens int 2363789Sahrens arc_tempreserve_space(uint64_t tempreserve) 2364789Sahrens { 2365789Sahrens #ifdef ZFS_DEBUG 2366789Sahrens /* 2367789Sahrens * Once in a while, fail for no reason. Everything should cope. 2368789Sahrens */ 2369789Sahrens if (spa_get_random(10000) == 0) { 2370789Sahrens dprintf("forcing random failure\n"); 2371789Sahrens return (ERESTART); 2372789Sahrens } 2373789Sahrens #endif 2374982Smaybee if (tempreserve > arc.c/4 && !arc.no_grow) 2375982Smaybee arc.c = MIN(arc.c_max, tempreserve * 4); 2376982Smaybee if (tempreserve > arc.c) 2377982Smaybee return (ENOMEM); 2378982Smaybee 2379789Sahrens /* 2380982Smaybee * Throttle writes when the amount of dirty data in the cache 2381982Smaybee * gets too large. We try to keep the cache less than half full 2382982Smaybee * of dirty blocks so that our sync times don't grow too large. 2383982Smaybee * Note: if two requests come in concurrently, we might let them 2384982Smaybee * both succeed, when one of them should fail. Not a huge deal. 2385982Smaybee * 2386982Smaybee * XXX The limit should be adjusted dynamically to keep the time 2387982Smaybee * to sync a dataset fixed (around 1-5 seconds?). 2388789Sahrens */ 2389789Sahrens 2390982Smaybee if (tempreserve + arc_tempreserve + arc.anon->size > arc.c / 2 && 2391982Smaybee arc_tempreserve + arc.anon->size > arc.c / 4) { 2392789Sahrens dprintf("failing, arc_tempreserve=%lluK anon=%lluK " 2393789Sahrens "tempreserve=%lluK arc.c=%lluK\n", 2394789Sahrens arc_tempreserve>>10, arc.anon->lsize>>10, 2395789Sahrens tempreserve>>10, arc.c>>10); 2396789Sahrens return (ERESTART); 2397789Sahrens } 2398789Sahrens atomic_add_64(&arc_tempreserve, tempreserve); 2399789Sahrens return (0); 2400789Sahrens } 2401789Sahrens 2402789Sahrens void 2403789Sahrens arc_init(void) 2404789Sahrens { 2405789Sahrens mutex_init(&arc_reclaim_lock, NULL, MUTEX_DEFAULT, NULL); 2406789Sahrens mutex_init(&arc_reclaim_thr_lock, NULL, MUTEX_DEFAULT, NULL); 2407789Sahrens cv_init(&arc_reclaim_thr_cv, NULL, CV_DEFAULT, NULL); 2408789Sahrens 24092391Smaybee /* Convert seconds to clock ticks */ 24102638Sperrin arc_min_prefetch_lifespan = 1 * hz; 24112391Smaybee 2412789Sahrens /* Start out with 1/8 of all memory */ 2413789Sahrens arc.c = physmem * PAGESIZE / 8; 2414789Sahrens 2415789Sahrens #ifdef _KERNEL 2416789Sahrens /* 2417789Sahrens * On architectures where the physical memory can be larger 2418789Sahrens * than the addressable space (intel in 32-bit mode), we may 2419789Sahrens * need to limit the cache to 1/8 of VM size. 2420789Sahrens */ 2421789Sahrens arc.c = MIN(arc.c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8); 2422789Sahrens #endif 2423789Sahrens 2424982Smaybee /* set min cache to 1/32 of all memory, or 64MB, whichever is more */ 2425789Sahrens arc.c_min = MAX(arc.c / 4, 64<<20); 2426982Smaybee /* set max to 3/4 of all memory, or all but 1GB, whichever is more */ 2427789Sahrens if (arc.c * 8 >= 1<<30) 2428789Sahrens arc.c_max = (arc.c * 8) - (1<<30); 2429789Sahrens else 2430789Sahrens arc.c_max = arc.c_min; 2431789Sahrens arc.c_max = MAX(arc.c * 6, arc.c_max); 2432789Sahrens arc.c = arc.c_max; 2433789Sahrens arc.p = (arc.c >> 1); 2434789Sahrens 2435789Sahrens /* if kmem_flags are set, lets try to use less memory */ 2436789Sahrens if (kmem_debugging()) 2437789Sahrens arc.c = arc.c / 2; 2438789Sahrens if (arc.c < arc.c_min) 2439789Sahrens arc.c = arc.c_min; 2440789Sahrens 2441789Sahrens arc.anon = &ARC_anon; 24421544Seschrock arc.mru = &ARC_mru; 24431544Seschrock arc.mru_ghost = &ARC_mru_ghost; 24441544Seschrock arc.mfu = &ARC_mfu; 24451544Seschrock arc.mfu_ghost = &ARC_mfu_ghost; 24461544Seschrock arc.size = 0; 2447789Sahrens 24482688Smaybee arc.hits = 0; 24492688Smaybee arc.recycle_miss = 0; 24502688Smaybee arc.evict_skip = 0; 24512688Smaybee arc.mutex_miss = 0; 24522688Smaybee 2453*2856Snd150628 mutex_init(&arc.anon->mtx, NULL, MUTEX_DEFAULT, NULL); 2454*2856Snd150628 mutex_init(&arc.mru->mtx, NULL, MUTEX_DEFAULT, NULL); 2455*2856Snd150628 mutex_init(&arc.mru_ghost->mtx, NULL, MUTEX_DEFAULT, NULL); 2456*2856Snd150628 mutex_init(&arc.mfu->mtx, NULL, MUTEX_DEFAULT, NULL); 2457*2856Snd150628 mutex_init(&arc.mfu_ghost->mtx, NULL, MUTEX_DEFAULT, NULL); 2458*2856Snd150628 24591544Seschrock list_create(&arc.mru->list, sizeof (arc_buf_hdr_t), 2460789Sahrens offsetof(arc_buf_hdr_t, b_arc_node)); 24611544Seschrock list_create(&arc.mru_ghost->list, sizeof (arc_buf_hdr_t), 2462789Sahrens offsetof(arc_buf_hdr_t, b_arc_node)); 24631544Seschrock list_create(&arc.mfu->list, sizeof (arc_buf_hdr_t), 2464789Sahrens offsetof(arc_buf_hdr_t, b_arc_node)); 24651544Seschrock list_create(&arc.mfu_ghost->list, sizeof (arc_buf_hdr_t), 2466789Sahrens offsetof(arc_buf_hdr_t, b_arc_node)); 2467789Sahrens 2468789Sahrens buf_init(); 2469789Sahrens 2470789Sahrens arc_thread_exit = 0; 24711544Seschrock arc_eviction_list = NULL; 24721544Seschrock mutex_init(&arc_eviction_mtx, NULL, MUTEX_DEFAULT, NULL); 2473789Sahrens 2474789Sahrens (void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0, 2475789Sahrens TS_RUN, minclsyspri); 2476789Sahrens } 2477789Sahrens 2478789Sahrens void 2479789Sahrens arc_fini(void) 2480789Sahrens { 2481789Sahrens mutex_enter(&arc_reclaim_thr_lock); 2482789Sahrens arc_thread_exit = 1; 2483789Sahrens while (arc_thread_exit != 0) 2484789Sahrens cv_wait(&arc_reclaim_thr_cv, &arc_reclaim_thr_lock); 2485789Sahrens mutex_exit(&arc_reclaim_thr_lock); 2486789Sahrens 2487789Sahrens arc_flush(); 2488789Sahrens 2489789Sahrens arc_dead = TRUE; 2490789Sahrens 24911544Seschrock mutex_destroy(&arc_eviction_mtx); 2492789Sahrens mutex_destroy(&arc_reclaim_lock); 2493789Sahrens mutex_destroy(&arc_reclaim_thr_lock); 2494789Sahrens cv_destroy(&arc_reclaim_thr_cv); 2495789Sahrens 24961544Seschrock list_destroy(&arc.mru->list); 24971544Seschrock list_destroy(&arc.mru_ghost->list); 24981544Seschrock list_destroy(&arc.mfu->list); 24991544Seschrock list_destroy(&arc.mfu_ghost->list); 2500789Sahrens 2501*2856Snd150628 mutex_destroy(&arc.anon->mtx); 2502*2856Snd150628 mutex_destroy(&arc.mru->mtx); 2503*2856Snd150628 mutex_destroy(&arc.mru_ghost->mtx); 2504*2856Snd150628 mutex_destroy(&arc.mfu->mtx); 2505*2856Snd150628 mutex_destroy(&arc.mfu_ghost->mtx); 2506*2856Snd150628 2507789Sahrens buf_fini(); 2508789Sahrens } 2509