Lines Matching +full:heart +full:- +full:rate
9 * or https://opensource.org/licenses/CDDL-1.0.
39 * DVA-based Adjustable Replacement Cache
42 * based on the self-tuning, low overhead replacement cache
51 * subset of the blocks in the cache are un-evictable because we
79 * See also: "ARC: A Self-Tuning, Low Overhead Replacement Cache"
123 * - L2ARC buflist creation
124 * - L2ARC buflist eviction
125 * - L2ARC write completion, which walks L2ARC buflists
126 * - ARC header destruction, as it removes from L2ARC buflists
127 * - ARC header release, as it removes from L2ARC buflists
139 * stored in the l2arc_buf_hdr_t sub-structure of the arc_buf_hdr_t. A block
146 * caches data in two ways -- in a list of ARC buffers (arc_buf_t) and
151 * arc_buf_hdr_t. Since the b_pabd is a copy of the on-disk physical block,
152 * it will match its on-disk compression characteristics. This behavior can be
155 * uncompressed version of the on-disk data.
181 * +-----------+
184 * | L1- and |
186 * +-----------+
189 * +-----------+
192 * | b_buf +------------>+-----------+ arc_buf_t
193 * | b_pabd +-+ |b_next +---->+-----------+
194 * +-----------+ | |-----------| |b_next +-->NULL
195 * | |b_comp = T | +-----------+
196 * | |b_data +-+ |b_comp = F |
197 * | +-----------+ | |b_data +-+
198 * +->+------+ | +-----------+ |
200 * data | |<--------------+ | uncompressed
201 * +------+ compressed, | data
202 * shared +-->+------+
205 * +------+
223 * +-----------+
227 * +-----------+
230 * +-----------+
233 * | b_buf +------------>+---------+ arc_buf_t
234 * | | |b_next +---->+---------+
235 * | b_pabd +-+ |---------| |b_next +-->NULL
236 * +-----------+ | | | +---------+
237 * | |b_data +-+ | |
238 * | +---------+ | |b_data +-+
239 * +->+------+ | +---------+ |
242 * data +------+ | |
243 * ^ +->+------+ |
246 * | +------+ |
247 * +---------------------------------+
253 * with the transformed data and will memcpy the transformed on-disk block into
263 * to the on-disk block in the main data pool. This provides a significant
273 * possible to decrypt encrypted data (or vice-versa) if the keys aren't loaded.
279 * data is stored in hdr->b_crypt_hdr.b_rabd. An encrypted header, therefore,
391 * when reading a new block into the ARC, we will evict an equal-sized block
446 * data hits on target data/metadata rate.
456 * These tunables are Linux-specific
674 x = x - x / ARCSTAT_F_AVG_FACTOR + \
682 * There are several ARC variables that are critical to export as kstats --
703 #define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE)
704 #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS)
705 #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_FLAG_IO_ERROR)
706 #define HDR_PREFETCH(hdr) ((hdr)->b_flags & ARC_FLAG_PREFETCH)
708 ((hdr)->b_flags & ARC_FLAG_PRESCIENT_PREFETCH)
710 ((hdr)->b_flags & ARC_FLAG_COMPRESSED_ARC)
712 #define HDR_L2CACHE(hdr) ((hdr)->b_flags & ARC_FLAG_L2CACHE)
713 #define HDR_UNCACHED(hdr) ((hdr)->b_flags & ARC_FLAG_UNCACHED)
715 (((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) && \
716 ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR))
717 #define HDR_L2_WRITING(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITING)
718 #define HDR_L2_EVICTED(hdr) ((hdr)->b_flags & ARC_FLAG_L2_EVICTED)
719 #define HDR_L2_WRITE_HEAD(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD)
720 #define HDR_PROTECTED(hdr) ((hdr)->b_flags & ARC_FLAG_PROTECTED)
721 #define HDR_NOAUTH(hdr) ((hdr)->b_flags & ARC_FLAG_NOAUTH)
722 #define HDR_SHARED_DATA(hdr) ((hdr)->b_flags & ARC_FLAG_SHARED_DATA)
725 ((hdr)->b_flags & ARC_FLAG_BUFC_METADATA)
728 #define HDR_HAS_L1HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L1HDR)
729 #define HDR_HAS_L2HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)
732 (hdr)->b_crypt_hdr.b_rabd != NULL)
734 (HDR_PROTECTED(hdr) && DMU_OT_IS_ENCRYPTED((hdr)->b_crypt_hdr.b_ot))
736 (HDR_PROTECTED(hdr) && !DMU_OT_IS_ENCRYPTED((hdr)->b_crypt_hdr.b_ot))
739 #define HDR_COMPRESS_OFFSET (highbit64(ARC_FLAG_COMPRESS_0) - 1)
741 #define HDR_GET_COMPRESS(hdr) ((enum zio_compress)BF32_GET((hdr)->b_flags, \
743 #define HDR_SET_COMPRESS(hdr, cmp) BF32_SET((hdr)->b_flags, \
746 #define ARC_BUF_LAST(buf) ((buf)->b_next == NULL)
747 #define ARC_BUF_SHARED(buf) ((buf)->b_flags & ARC_BUF_FLAG_SHARED)
748 #define ARC_BUF_COMPRESSED(buf) ((buf)->b_flags & ARC_BUF_FLAG_COMPRESSED)
749 #define ARC_BUF_ENCRYPTED(buf) ((buf)->b_flags & ARC_BUF_FLAG_ENCRYPTED)
773 #define BUF_HASH_LOCK(idx) (&buf_hash_table.ht_locks[idx & (BUF_LOCKS-1)])
775 (BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
1011 return (cityhash4(spa, dva->dva_word[0], dva->dva_word[1], birth));
1015 ((hdr)->b_dva.dva_word[0] == 0 && \
1016 (hdr)->b_dva.dva_word[1] == 0)
1022 ((hdr)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \
1023 ((hdr)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \
1024 ((hdr)->b_birth == birth) && ((hdr)->b_spa == spa)
1029 hdr->b_dva.dva_word[0] = 0;
1030 hdr->b_dva.dva_word[1] = 0;
1031 hdr->b_birth = 0;
1045 hdr = hdr->b_hash_next) {
1066 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1071 ASSERT(!DVA_IS_EMPTY(&hdr->b_dva));
1072 ASSERT(hdr->b_birth != 0);
1083 fhdr = fhdr->b_hash_next, i++) {
1084 if (HDR_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr))
1088 hdr->b_hash_next = buf_hash_table.ht_table[idx];
1108 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1116 hdrp = &fhdr->b_hash_next;
1118 *hdrp = hdr->b_hash_next;
1119 hdr->b_hash_next = NULL;
1125 buf_hash_table.ht_table[idx]->b_hash_next == NULL)
1159 * Constructor callback - called when the cache is empty
1169 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
1170 zfs_refcount_create(&hdr->b_l1hdr.b_refcnt);
1172 mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
1174 multilist_link_init(&hdr->b_l1hdr.b_arc_node);
1175 list_link_init(&hdr->b_l2hdr.b_l2node);
1206 * Destructor callback - called when a cached buf is
1216 zfs_refcount_destroy(&hdr->b_l1hdr.b_refcnt);
1218 mutex_destroy(&hdr->b_l1hdr.b_freeze_lock);
1220 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
1254 * totalmem * sizeof(void*) / 8K (1MB per GB with 8-byte pointers).
1259 buf_hash_table.ht_mask = hsize - 1;
1286 for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--)
1287 *ct = (*ct >> 1) ^ (-(*ct & 1) & ZFS_CRC64_POLY);
1304 HDR_GET_PSIZE(buf->b_hdr) : HDR_GET_LSIZE(buf->b_hdr));
1310 return (HDR_GET_LSIZE(buf->b_hdr));
1330 return (HDR_NOAUTH(buf->b_hdr) != 0);
1337 arc_buf_hdr_t *hdr = buf->b_hdr;
1341 memcpy(salt, hdr->b_crypt_hdr.b_salt, ZIO_DATA_SALT_LEN);
1342 memcpy(iv, hdr->b_crypt_hdr.b_iv, ZIO_DATA_IV_LEN);
1343 memcpy(mac, hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN);
1344 *byteorder = (hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS) ?
1357 HDR_GET_COMPRESS(buf->b_hdr) : ZIO_COMPRESS_OFF);
1363 * as what's used to store it on-disk. Otherwise, this will be ZIO_COMPRESS_OFF.
1375 return (buf->b_hdr->b_complevel);
1381 boolean_t shared = (buf->b_data != NULL &&
1382 buf->b_hdr->b_l1hdr.b_pabd != NULL &&
1383 abd_is_linear(buf->b_hdr->b_l1hdr.b_pabd) &&
1384 buf->b_data == abd_to_buf(buf->b_hdr->b_l1hdr.b_pabd));
1385 IMPLY(shared, HDR_SHARED_DATA(buf->b_hdr));
1399 * is a no-op.
1407 mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1408 if (hdr->b_l1hdr.b_freeze_cksum != NULL) {
1409 kmem_free(hdr->b_l1hdr.b_freeze_cksum, sizeof (zio_cksum_t));
1410 hdr->b_l1hdr.b_freeze_cksum = NULL;
1412 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1423 ASSERT(hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY_OR_LOCKED(hdr));
1425 for (arc_buf_t *b = hdr->b_l1hdr.b_buf; b != NULL; b = b->b_next) {
1437 * or if the buf is compressed, this is a no-op.
1443 arc_buf_hdr_t *hdr = buf->b_hdr;
1454 mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1456 if (hdr->b_l1hdr.b_freeze_cksum == NULL || HDR_IO_ERROR(hdr)) {
1457 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1461 fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL, &zc);
1462 if (!ZIO_CHECKSUM_EQUAL(*hdr->b_l1hdr.b_freeze_cksum, zc))
1464 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1476 ASSERT(!BP_IS_EMBEDDED(zio->io_bp));
1477 VERIFY3U(BP_GET_PSIZE(zio->io_bp), ==, HDR_GET_PSIZE(hdr));
1492 return (zio_checksum_error_impl(zio->io_spa, zio->io_bp,
1493 BP_GET_CHECKSUM(zio->io_bp), zio->io_abd, zio->io_size,
1494 zio->io_offset, NULL) == 0);
1501 * on the hdr, this is a no-op (we only checksum uncompressed bufs).
1510 arc_buf_hdr_t *hdr = buf->b_hdr;
1512 mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1513 if (hdr->b_l1hdr.b_freeze_cksum != NULL || ARC_BUF_COMPRESSED(buf)) {
1514 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1520 hdr->b_l1hdr.b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t),
1522 fletcher_2_native(buf->b_data, arc_buf_size(buf), NULL,
1523 hdr->b_l1hdr.b_freeze_cksum);
1524 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1534 panic("Got SIGSEGV at address: 0x%lx\n", (long)si->si_addr);
1543 ASSERT0(mprotect(buf->b_data, arc_buf_size(buf),
1556 ASSERT0(mprotect(buf->b_data, arc_buf_size(buf),
1572 VERIFY3U(hdr->b_type, ==, type);
1579 return (HDR_ISTYPE_METADATA(buf->b_hdr) != 0);
1595 return ((uint32_t)-1);
1601 arc_buf_hdr_t *hdr = buf->b_hdr;
1603 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
1628 ASSERT(HDR_HAS_L1HDR(buf->b_hdr));
1635 * updated in a thread-safe way. When manipulating the flags either
1644 hdr->b_flags |= flags;
1651 hdr->b_flags &= ~flags;
1659 * thread-safe manner.
1690 arc_buf_hdr_t *hdr = buf->b_hdr;
1694 ASSERT3P(buf->b_data, !=, NULL);
1697 for (arc_buf_t *from = hdr->b_l1hdr.b_buf; from != NULL;
1698 from = from->b_next) {
1705 memcpy(buf->b_data, from->b_data, arc_buf_size(buf));
1717 EQUIV(!copied, hdr->b_l1hdr.b_freeze_cksum == NULL);
1724 * Allocates an ARC buf header that's in an evicted & L2-cached state.
1726 * which circumvent the regular disk->arc->l2arc path and instead come
1727 * into being in the reverse order, i.e. l2arc->arc.
1738 ASSERT(dev->l2ad_vdev != NULL);
1741 hdr->b_birth = birth;
1742 hdr->b_type = type;
1743 hdr->b_flags = 0;
1749 hdr->b_complevel = complevel;
1754 hdr->b_spa = spa_load_guid(dev->l2ad_vdev->vdev_spa);
1756 hdr->b_dva = dva;
1758 hdr->b_l2hdr.b_dev = dev;
1759 hdr->b_l2hdr.b_daddr = daddr;
1760 hdr->b_l2hdr.b_arcs_state = arcs_state;
1790 abd_t *abd = hdr->b_l1hdr.b_pabd;
1809 hdr->b_l1hdr.b_pabd, &abd, lsize, MIN(lsize, psize),
1810 hdr->b_complevel);
1816 abd_zero_off(abd, csize, psize - csize);
1824 if (hdr->b_crypt_hdr.b_ot == DMU_OT_OBJSET) {
1828 psize, hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);
1831 hdr->b_crypt_hdr.b_mac);
1857 boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);
1864 ret = spa_do_crypt_abd(B_FALSE, spa, zb, hdr->b_crypt_hdr.b_ot,
1865 B_FALSE, bswap, hdr->b_crypt_hdr.b_salt, hdr->b_crypt_hdr.b_iv,
1866 hdr->b_crypt_hdr.b_mac, HDR_GET_PSIZE(hdr), hdr->b_l1hdr.b_pabd,
1867 hdr->b_crypt_hdr.b_rabd, &no_crypt);
1872 abd_copy(hdr->b_l1hdr.b_pabd, hdr->b_crypt_hdr.b_rabd,
1892 hdr->b_l1hdr.b_pabd, cabd, HDR_GET_PSIZE(hdr),
1893 HDR_GET_LSIZE(hdr), &hdr->b_complevel);
1898 arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
1900 hdr->b_l1hdr.b_pabd = cabd;
1934 ret = arc_hdr_authenticate(hdr, spa, zb->zb_objset);
1937 } else if (HDR_HAS_RABD(hdr) && hdr->b_l1hdr.b_pabd == NULL) {
1948 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
1971 arc_buf_hdr_t *hdr = buf->b_hdr;
1974 ASSERT3U(hdr->b_crypt_hdr.b_ot, ==, DMU_OT_DNODE);
1976 ASSERT3PF(hdr->b_l1hdr.b_pabd, !=, NULL, "hdr %px buf %px", hdr, buf);
1978 zio_crypt_copy_dnode_bonus(hdr->b_l1hdr.b_pabd, buf->b_data,
1980 buf->b_flags &= ~ARC_BUF_FLAG_ENCRYPTED;
1981 buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
1995 * the correct-sized data buffer.
2002 arc_buf_hdr_t *hdr = buf->b_hdr;
2007 dmu_object_byteswap_t bswap = hdr->b_l1hdr.b_byteswap;
2010 ASSERT3P(buf->b_data, !=, NULL);
2025 abd_copy_to_buf(buf->b_data, hdr->b_crypt_hdr.b_rabd,
2054 * be decrypted in-place. This is necessary because there may
2059 * arises for other types to be decrypted in-place, they must
2068 ASSERT3U(hdr->b_crypt_hdr.b_ot, ==, DMU_OT_DNODE);
2087 abd_copy_to_buf(buf->b_data, hdr->b_l1hdr.b_pabd,
2103 buf->b_flags &= ~ARC_BUF_FLAG_SHARED;
2104 buf->b_data =
2114 arc_free_data_buf(hdr, buf->b_data, HDR_GET_PSIZE(hdr),
2116 buf->b_data =
2121 HDR_GET_LSIZE(hdr) - HDR_GET_PSIZE(hdr));
2128 buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
2140 abd_get_from_buf_struct(&dabd, buf->b_data,
2143 hdr->b_l1hdr.b_pabd, &dabd,
2145 &hdr->b_complevel);
2172 dmu_ot_byteswap[bswap].ob_func(buf->b_data, HDR_GET_LSIZE(hdr));
2204 spa_log_error(spa, zb, buf->b_hdr->b_birth);
2225 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2226 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2228 (void) zfs_refcount_add_many(&state->arcs_esize[type],
2233 if (hdr->b_l1hdr.b_pabd != NULL) {
2234 (void) zfs_refcount_add_many(&state->arcs_esize[type],
2238 (void) zfs_refcount_add_many(&state->arcs_esize[type],
2242 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2243 buf = buf->b_next) {
2246 (void) zfs_refcount_add_many(&state->arcs_esize[type],
2264 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2265 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2267 (void) zfs_refcount_remove_many(&state->arcs_esize[type],
2272 if (hdr->b_l1hdr.b_pabd != NULL) {
2273 (void) zfs_refcount_remove_many(&state->arcs_esize[type],
2277 (void) zfs_refcount_remove_many(&state->arcs_esize[type],
2281 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2282 buf = buf->b_next) {
2285 (void) zfs_refcount_remove_many(&state->arcs_esize[type],
2299 arc_state_t *state = hdr->b_l1hdr.b_state;
2304 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2305 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2308 if ((zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) &&
2310 /* We don't use the L2-only state list. */
2311 multilist_remove(&state->arcs_list[arc_buf_type(hdr)], hdr);
2325 arc_state_t *state = hdr->b_l1hdr.b_state;
2331 if ((cnt = zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) != 0)
2343 multilist_insert(&state->arcs_list[arc_buf_type(hdr)], hdr);
2359 arc_buf_hdr_t *hdr = ab->b_hdr;
2369 abi->abi_flags = hdr->b_flags;
2372 l1hdr = &hdr->b_l1hdr;
2373 state = l1hdr->b_state;
2376 l2hdr = &hdr->b_l2hdr;
2379 abi->abi_bufcnt = 0;
2380 for (arc_buf_t *buf = l1hdr->b_buf; buf; buf = buf->b_next)
2381 abi->abi_bufcnt++;
2382 abi->abi_access = l1hdr->b_arc_access;
2383 abi->abi_mru_hits = l1hdr->b_mru_hits;
2384 abi->abi_mru_ghost_hits = l1hdr->b_mru_ghost_hits;
2385 abi->abi_mfu_hits = l1hdr->b_mfu_hits;
2386 abi->abi_mfu_ghost_hits = l1hdr->b_mfu_ghost_hits;
2387 abi->abi_holds = zfs_refcount_count(&l1hdr->b_refcnt);
2391 abi->abi_l2arc_dattr = l2hdr->b_daddr;
2392 abi->abi_l2arc_hits = l2hdr->b_hits;
2395 abi->abi_state_type = state ? state->arcs_state : ARC_STATE_ANON;
2396 abi->abi_state_contents = arc_buf_type(hdr);
2397 abi->abi_size = arc_hdr_size(hdr);
2420 old_state = hdr->b_l1hdr.b_state;
2421 refcnt = zfs_refcount_count(&hdr->b_l1hdr.b_refcnt);
2422 update_old = (hdr->b_l1hdr.b_buf != NULL ||
2423 hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr));
2425 IMPLY(GHOST_STATE(old_state), hdr->b_l1hdr.b_buf == NULL);
2426 IMPLY(GHOST_STATE(new_state), hdr->b_l1hdr.b_buf == NULL);
2427 IMPLY(old_state == arc_anon, hdr->b_l1hdr.b_buf == NULL ||
2428 ARC_BUF_LAST(hdr->b_l1hdr.b_buf));
2451 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
2452 multilist_remove(&old_state->arcs_list[type],
2460 * moving to some L1-cached state (i.e. not l2c_only or
2465 multilist_insert(&new_state->arcs_list[type], hdr);
2487 &new_state->arcs_size[type],
2489 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2498 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2499 buf = buf->b_next) {
2512 &new_state->arcs_size[type],
2516 if (hdr->b_l1hdr.b_pabd != NULL) {
2518 &new_state->arcs_size[type],
2524 &new_state->arcs_size[type],
2533 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2545 &old_state->arcs_size[type],
2554 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2555 buf = buf->b_next) {
2568 &old_state->arcs_size[type],
2571 ASSERT(hdr->b_l1hdr.b_pabd != NULL ||
2574 if (hdr->b_l1hdr.b_pabd != NULL) {
2576 &old_state->arcs_size[type],
2582 &old_state->arcs_size[type],
2589 hdr->b_l1hdr.b_state = new_state;
2593 hdr->b_l2hdr.b_arcs_state = new_state->arcs_state;
2633 * very short-lived.
2654 ARCSTAT_INCR(arcstat_data_size, -space);
2657 ARCSTAT_INCR(arcstat_metadata_size, -space);
2660 ARCSTAT_INCR(arcstat_bonus_size, -space);
2663 ARCSTAT_INCR(arcstat_dnode_size, -space);
2666 ARCSTAT_INCR(arcstat_dbuf_size, -space);
2669 ARCSTAT_INCR(arcstat_hdr_size, -space);
2672 aggsum_add(&arc_sums.arcstat_l2_hdr_size, -space);
2675 ARCSTAT_INCR(arcstat_abd_chunk_waste_size, -space);
2680 ARCSTAT_INCR(arcstat_meta_used, -space);
2683 aggsum_add(&arc_sums.arcstat_size, -space);
2715 ASSERT3P(buf->b_hdr, ==, hdr);
2721 hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS &&
2741 VERIFY(hdr->b_type == ARC_BUFC_DATA ||
2742 hdr->b_type == ARC_BUFC_METADATA);
2748 buf->b_hdr = hdr;
2749 buf->b_data = NULL;
2750 buf->b_next = hdr->b_l1hdr.b_buf;
2751 buf->b_flags = 0;
2767 buf->b_flags |= ARC_BUF_FLAG_COMPRESSED;
2768 buf->b_flags |= ARC_BUF_FLAG_ENCRYPTED;
2772 buf->b_flags |= ARC_BUF_FLAG_COMPRESSED;
2788 * hdr -> buf instead of the usual buf -> hdr. First, the hdr can't be
2793 * need to be ABD-aware. It must be allocated via
2801 hdr->b_l1hdr.b_pabd != NULL &&
2802 abd_is_linear(hdr->b_l1hdr.b_pabd) &&
2803 !abd_is_linear_page(hdr->b_l1hdr.b_pabd);
2807 buf->b_data = abd_to_buf(hdr->b_l1hdr.b_pabd);
2808 buf->b_flags |= ARC_BUF_FLAG_SHARED;
2811 buf->b_data =
2815 VERIFY3P(buf->b_data, !=, NULL);
2817 hdr->b_l1hdr.b_buf = buf;
2892 arc_buf_hdr_t *hdr = buf->b_hdr;
2894 ASSERT3P(buf->b_data, !=, NULL);
2896 (void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
2897 (void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
2899 arc_loaned_bytes_update(-arc_buf_size(buf));
2906 arc_buf_hdr_t *hdr = buf->b_hdr;
2908 ASSERT3P(buf->b_data, !=, NULL);
2910 (void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
2911 (void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag);
2921 df->l2df_abd = abd;
2922 df->l2df_size = size;
2923 df->l2df_type = type;
2932 arc_state_t *state = hdr->b_l1hdr.b_state;
2937 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
2938 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2941 (void) zfs_refcount_remove_many(&state->arcs_esize[type],
2944 (void) zfs_refcount_remove_many(&state->arcs_size[type], size, hdr);
2953 l2arc_free_abd_on_write(hdr->b_crypt_hdr.b_rabd, size, type);
2955 l2arc_free_abd_on_write(hdr->b_l1hdr.b_pabd, size, type);
2968 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2978 &hdr->b_l1hdr.b_state->arcs_size[arc_buf_type(hdr)],
2980 hdr->b_l1hdr.b_pabd = abd_get_from_buf(buf->b_data, arc_buf_size(buf));
2981 abd_take_ownership_of_buf(hdr->b_l1hdr.b_pabd,
2984 buf->b_flags |= ARC_BUF_FLAG_SHARED;
2993 ARCSTAT_INCR(arcstat_overhead_size, -arc_buf_size(buf));
3000 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3008 &hdr->b_l1hdr.b_state->arcs_size[arc_buf_type(hdr)],
3011 abd_release_ownership_of_buf(hdr->b_l1hdr.b_pabd);
3012 abd_free(hdr->b_l1hdr.b_pabd);
3013 hdr->b_l1hdr.b_pabd = NULL;
3014 buf->b_flags &= ~ARC_BUF_FLAG_SHARED;
3020 ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr));
3021 ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
3036 arc_buf_t **bufp = &hdr->b_l1hdr.b_buf;
3045 *bufp = buf->b_next;
3054 bufp = &(*bufp)->b_next;
3057 buf->b_next = NULL;
3065 * Free up buf->b_data and pull the arc_buf_t off of the arc_buf_hdr_t's
3071 arc_buf_hdr_t *hdr = buf->b_hdr;
3078 if (buf->b_data != NULL) {
3093 arc_free_data_buf(hdr, buf->b_data, size, buf);
3094 ARCSTAT_INCR(arcstat_overhead_size, -size);
3096 buf->b_data = NULL;
3104 hdr->b_l1hdr.b_pabd != NULL && !HDR_IO_IN_PROGRESS(hdr)) {
3106 for (b = hdr->b_l1hdr.b_buf; b; b = b->b_next) {
3136 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3170 buf->b_hdr = NULL;
3187 ASSERT3P(hdr->b_crypt_hdr.b_rabd, ==, NULL);
3188 hdr->b_crypt_hdr.b_rabd = arc_get_data_abd(hdr, size, hdr,
3190 ASSERT3P(hdr->b_crypt_hdr.b_rabd, !=, NULL);
3194 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3195 hdr->b_l1hdr.b_pabd = arc_get_data_abd(hdr, size, hdr,
3197 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3210 ASSERT(hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr));
3223 arc_free_data_abd(hdr, hdr->b_crypt_hdr.b_rabd, size, hdr);
3225 arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd, size, hdr);
3229 hdr->b_crypt_hdr.b_rabd = NULL;
3230 ARCSTAT_INCR(arcstat_raw_size, -size);
3232 hdr->b_l1hdr.b_pabd = NULL;
3235 if (hdr->b_l1hdr.b_pabd == NULL && !HDR_HAS_RABD(hdr))
3236 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
3238 ARCSTAT_INCR(arcstat_compressed_size, -size);
3239 ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
3276 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3280 hdr->b_spa = spa;
3281 hdr->b_type = type;
3282 hdr->b_flags = 0;
3285 hdr->b_complevel = complevel;
3289 hdr->b_l1hdr.b_state = arc_anon;
3290 hdr->b_l1hdr.b_arc_access = 0;
3291 hdr->b_l1hdr.b_mru_hits = 0;
3292 hdr->b_l1hdr.b_mru_ghost_hits = 0;
3293 hdr->b_l1hdr.b_mfu_hits = 0;
3294 hdr->b_l1hdr.b_mfu_ghost_hits = 0;
3295 hdr->b_l1hdr.b_buf = NULL;
3297 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3305 * (hdr_l2only_cache) the fields necessary for the L1 cache - the smaller
3315 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
3334 nhdr->b_l1hdr.b_state = arc_l2c_only;
3337 ASSERT3P(nhdr->b_l1hdr.b_pabd, ==, NULL);
3340 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
3342 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3352 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3361 VERIFY3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3367 * The header has been reallocated so we need to re-insert it into any
3372 ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node));
3374 mutex_enter(&dev->l2ad_mtx);
3382 list_insert_after(&dev->l2ad_buflist, hdr, nhdr);
3383 list_remove(&dev->l2ad_buflist, hdr);
3385 mutex_exit(&dev->l2ad_mtx);
3395 (void) zfs_refcount_remove_many(&dev->l2ad_alloc,
3397 (void) zfs_refcount_add_many(&dev->l2ad_alloc,
3418 arc_buf_hdr_t *hdr = buf->b_hdr;
3422 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
3424 buf->b_flags |= (ARC_BUF_FLAG_COMPRESSED | ARC_BUF_FLAG_ENCRYPTED);
3426 hdr->b_crypt_hdr.b_dsobj = dsobj;
3427 hdr->b_crypt_hdr.b_ot = ot;
3428 hdr->b_l1hdr.b_byteswap = (byteorder == ZFS_HOST_BYTEORDER) ?
3434 memcpy(hdr->b_crypt_hdr.b_salt, salt, ZIO_DATA_SALT_LEN);
3436 memcpy(hdr->b_crypt_hdr.b_iv, iv, ZIO_DATA_IV_LEN);
3438 memcpy(hdr->b_crypt_hdr.b_mac, mac, ZIO_DATA_MAC_LEN);
3510 hdr->b_crypt_hdr.b_dsobj = dsobj;
3511 hdr->b_crypt_hdr.b_ot = ot;
3512 hdr->b_l1hdr.b_byteswap = (byteorder == ZFS_HOST_BYTEORDER) ?
3514 memcpy(hdr->b_crypt_hdr.b_salt, salt, ZIO_DATA_SALT_LEN);
3515 memcpy(hdr->b_crypt_hdr.b_iv, iv, ZIO_DATA_IV_LEN);
3516 memcpy(hdr->b_crypt_hdr.b_mac, mac, ZIO_DATA_MAC_LEN);
3538 arc_buf_contents_t type = hdr->b_type;
3551 lsize_s = -lsize;
3552 psize_s = -psize;
3553 asize_s = -asize;
3569 switch (hdr->b_l2hdr.b_arcs_state) {
3605 l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
3606 l2arc_dev_t *dev = l2hdr->b_dev;
3608 ASSERT(MUTEX_HELD(&dev->l2ad_mtx));
3611 list_remove(&dev->l2ad_buflist, hdr);
3614 if (dev->l2ad_vdev != NULL) {
3616 vdev_space_update(dev->l2ad_vdev, -asize, 0, 0);
3619 (void) zfs_refcount_remove_many(&dev->l2ad_alloc, arc_hdr_size(hdr),
3628 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3629 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
3635 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
3636 boolean_t buflist_held = MUTEX_HELD(&dev->l2ad_mtx);
3639 mutex_enter(&dev->l2ad_mtx);
3648 * want to re-destroy the header's L2 portion.
3659 mutex_exit(&dev->l2ad_mtx);
3674 while (hdr->b_l1hdr.b_buf != NULL)
3675 arc_buf_destroy_impl(hdr->b_l1hdr.b_buf);
3677 if (hdr->b_l1hdr.b_pabd != NULL)
3684 ASSERT3P(hdr->b_hash_next, ==, NULL);
3686 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3687 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
3689 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3700 arc_buf_hdr_t *hdr = buf->b_hdr;
3702 if (hdr->b_l1hdr.b_state == arc_anon) {
3703 ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);
3713 ASSERT3P(hdr, ==, buf->b_hdr);
3714 ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
3716 ASSERT3P(hdr->b_l1hdr.b_state, !=, arc_anon);
3717 ASSERT3P(buf->b_data, !=, NULL);
3729 * - arc_mru -> arc_mru_ghost
3730 * - arc_mfu -> arc_mfu_ghost
3731 * - arc_mru_ghost -> arc_l2c_only
3732 * - arc_mru_ghost -> deleted
3733 * - arc_mfu_ghost -> arc_l2c_only
3734 * - arc_mfu_ghost -> deleted
3735 * - arc_uncached -> deleted
3739 * progress at the same (or at least comparable) rate as from non-ghost states.
3742 * waiting for it. For non-ghost states it includes size of evicted data
3757 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
3758 ASSERT0(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt));
3761 state = hdr->b_l1hdr.b_state;
3782 ASSERT(hdr->b_l1hdr.b_pabd == NULL);
3790 * dropping from L1+L2 cached to L2-only,
3795 *real_evicted += HDR_FULL_SIZE - HDR_L2ONLY_SIZE;
3809 if ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) &&
3810 ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access <
3819 if (l2arc_write_eligible(hdr->b_spa, hdr)) {
3823 switch (state->arcs_state) {
3851 if (hdr->b_l1hdr.b_pabd != NULL)
3873 int64_t remaining = arc_free_memory() - arc_sys_free / 2;
3876 arc_need_free = MAX(-remaining, 0);
3879 MAX(-remaining, (int64_t)(aw->aew_count - arc_evict_count));
3912 * (only 'marker' will be removed and re-inserted).
3924 if (hdr->b_spa == 0)
3928 if (spa != 0 && hdr->b_spa != spa) {
3960 evict_count--;
3987 aw->aew_count <= arc_evict_count) {
3989 cv_broadcast(&aw->aew_cv);
3997 * if the average cached block is small), eviction can be on-CPU for
4016 marker->b_spa = 0;
4068 multilist_t *ml = &state->arcs_list[type];
4115 bytes_remaining = bytes - total_evicted;
4186 while (zfs_refcount_count(&state->arcs_esize[type]) != 0) {
4208 if (bytes > 0 && zfs_refcount_count(&state->arcs_esize[type]) > 0) {
4209 delta = MIN(zfs_refcount_count(&state->arcs_esize[type]),
4220 * decreasing it, plus a balance factor, controlling the decrease rate, used
4242 s = MIN(64 - s, 32);
4244 uint64_t ofrac = (1ULL << 32) - frac;
4248 up = (up << s) / (total >> (32 - s));
4251 down = (down << s) / (total >> (32 - s));
4254 return (frac + up - down);
4282 mrud = zfs_refcount_count(&arc_mru->arcs_size[ARC_BUFC_DATA]) +
4283 zfs_refcount_count(&arc_anon->arcs_size[ARC_BUFC_DATA]);
4284 mrum = zfs_refcount_count(&arc_mru->arcs_size[ARC_BUFC_METADATA]) +
4285 zfs_refcount_count(&arc_anon->arcs_size[ARC_BUFC_METADATA]);
4286 mfud = zfs_refcount_count(&arc_mfu->arcs_size[ARC_BUFC_DATA]);
4287 mfum = zfs_refcount_count(&arc_mfu->arcs_size[ARC_BUFC_METADATA]);
4293 ngrd = wmsum_value(&arc_mru_ghost->arcs_hits[ARC_BUFC_DATA]);
4294 uint64_t grd = ngrd - ogrd;
4296 ngrm = wmsum_value(&arc_mru_ghost->arcs_hits[ARC_BUFC_METADATA]);
4297 uint64_t grm = ngrm - ogrm;
4299 ngfd = wmsum_value(&arc_mfu_ghost->arcs_hits[ARC_BUFC_DATA]);
4300 uint64_t gfd = ngfd - ogfd;
4302 ngfm = wmsum_value(&arc_mfu_ghost->arcs_hits[ARC_BUFC_METADATA]);
4303 uint64_t gfm = ngfm - ogfm;
4314 int64_t wt = t - (asize - ac);
4322 int64_t nem = zfs_refcount_count(&arc_mru->arcs_size[ARC_BUFC_METADATA])
4323 + zfs_refcount_count(&arc_mfu->arcs_size[ARC_BUFC_METADATA])
4324 - zfs_refcount_count(&arc_mru->arcs_esize[ARC_BUFC_METADATA])
4325 - zfs_refcount_count(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]);
4331 prune = arc_mf(prune, nem - w * 3 / 4, w / 4);
4334 prune = MAX(prune, (dn - arc_dnode_limit) / sizeof (dnode_t) *
4342 e = MIN((int64_t)(asize - ac), (int64_t)(mrum - w));
4345 mrum -= bytes;
4346 asize -= bytes;
4350 e = MIN((int64_t)(asize - ac), (int64_t)(m - bytes - w));
4353 mfum -= bytes;
4354 asize -= bytes;
4357 wt -= m - total_evicted;
4359 e = MIN((int64_t)(asize - ac), (int64_t)(mrud - w));
4362 mrud -= bytes;
4363 asize -= bytes;
4366 e = asize - ac;
4368 mfud -= bytes;
4383 e = zfs_refcount_count(&arc_mru_ghost->arcs_size[ARC_BUFC_DATA]) -
4388 e = zfs_refcount_count(&arc_mru_ghost->arcs_size[ARC_BUFC_METADATA]) -
4393 e = zfs_refcount_count(&arc_mfu_ghost->arcs_size[ARC_BUFC_DATA]) -
4398 e = zfs_refcount_count(&arc_mfu_ghost->arcs_size[ARC_BUFC_METADATA]) -
4443 af->af_spa_guid = spa_guid;
4444 af->af_cache_level = level;
4445 taskq_init_ent(&af->af_tqent);
4446 list_link_init(&af->af_node);
4461 if (af->af_spa_guid == spa_guid &&
4462 af->af_cache_level == level) {
4476 uint64_t spa_guid = af->af_spa_guid;
4479 arc_async_flush_remove(spa_guid, af->af_cache_level);
4481 uint64_t elaspsed = NSEC2MSEC(gethrtime() - start_time);
4493 * It's OK if the spa is re-imported while this asynchronous flush is
4505 af, TQ_SLEEP, &af->af_tqent);
4509 * Check if a guid is still in-use as part of an async teardown task
4517 if (af->af_spa_guid == spa_guid) {
4546 to_free = MIN(to_free, c - arc_c_min);
4547 arc_c = c - to_free;
4596 /* reach upper limit of cache size on 32-bit */
4636 arc_ksp->ks_update(arc_ksp, KSTAT_READ);
4659 return ((zfs_refcount_count(&arc_uncached->arcs_esize[ARC_BUFC_DATA]) +
4660 zfs_refcount_count(&arc_uncached->arcs_esize[ARC_BUFC_METADATA]) &&
4661 ddi_get_lbolt() - arc_last_uncached_flush >
4712 cv_broadcast(&aw->aew_cv);
4732 * becoming implicitly blocked by a system-wide kmem reap -- which,
4801 can_free = arc_c - arc_c_min;
4802 to_free = (MAX(can_free, 0) >> arc_shrink_shift) - free_memory;
4906 int64_t over = aggsum_lower_bound(&arc_sums.arcstat_size) - arc_c -
4994 last_count = last->aew_count;
5071 arc_state_t *state = hdr->b_l1hdr.b_state;
5074 (void) zfs_refcount_add_many(&state->arcs_size[type], size,
5086 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
5087 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
5088 (void) zfs_refcount_add_many(&state->arcs_esize[type],
5122 arc_state_t *state = hdr->b_l1hdr.b_state;
5126 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
5127 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
5130 (void) zfs_refcount_remove_many(&state->arcs_esize[type],
5133 (void) zfs_refcount_remove_many(&state->arcs_size[type], size, tag);
5135 VERIFY3U(hdr->b_type, ==, type);
5187 if (hdr->b_l1hdr.b_state == arc_anon) {
5193 ASSERT0(hdr->b_l1hdr.b_arc_access);
5194 hdr->b_l1hdr.b_arc_access = now;
5204 } else if (hdr->b_l1hdr.b_state == arc_mru) {
5210 hdr->b_l1hdr.b_arc_access = now;
5213 hdr->b_l1hdr.b_mru_hits++;
5221 hdr->b_l1hdr.b_arc_access = now;
5229 if (ddi_time_after(now, hdr->b_l1hdr.b_arc_access +
5231 hdr->b_l1hdr.b_arc_access = now;
5235 } else if (hdr->b_l1hdr.b_state == arc_mru_ghost) {
5243 hdr->b_l1hdr.b_mru_ghost_hits++;
5245 hdr->b_l1hdr.b_arc_access = now;
5246 wmsum_add(&arc_mru_ghost->arcs_hits[arc_buf_type(hdr)],
5256 } else if (hdr->b_l1hdr.b_state == arc_mfu) {
5262 hdr->b_l1hdr.b_mfu_hits++;
5265 hdr->b_l1hdr.b_arc_access = now;
5266 } else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) {
5272 hdr->b_l1hdr.b_mfu_ghost_hits++;
5274 hdr->b_l1hdr.b_arc_access = now;
5275 wmsum_add(&arc_mfu_ghost->arcs_hits[arc_buf_type(hdr)],
5279 } else if (hdr->b_l1hdr.b_state == arc_uncached) {
5286 hdr->b_l1hdr.b_arc_access = now;
5287 } else if (hdr->b_l1hdr.b_state == arc_l2c_only) {
5292 hdr->b_l1hdr.b_arc_access = now;
5297 hdr->b_l1hdr.b_state);
5308 arc_buf_hdr_t *hdr = buf->b_hdr;
5315 if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr))
5321 if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr)) {
5327 ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
5328 hdr->b_l1hdr.b_state == arc_mfu ||
5329 hdr->b_l1hdr.b_state == arc_uncached);
5350 memcpy(arg, buf->b_data, arc_buf_size(buf));
5363 ASSERT(zio == NULL || zio->io_error != 0);
5366 ASSERT(zio == NULL || zio->io_error == 0);
5368 ASSERT(buf->b_data != NULL);
5392 blkptr_t *bp = zio->io_bp;
5393 arc_buf_hdr_t *hdr = zio->io_private;
5399 * The hdr was inserted into hash-table and removed from lists
5409 ASSERT3U(hdr->b_birth, ==, BP_GET_BIRTH(zio->io_bp));
5410 ASSERT3U(hdr->b_dva.dva_word[0], ==,
5411 BP_IDENTITY(zio->io_bp)->dva_word[0]);
5412 ASSERT3U(hdr->b_dva.dva_word[1], ==,
5413 BP_IDENTITY(zio->io_bp)->dva_word[1]);
5415 found = buf_hash_find(hdr->b_spa, zio->io_bp, &hash_lock);
5418 DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
5424 hdr->b_crypt_hdr.b_ot = BP_GET_TYPE(bp);
5425 hdr->b_crypt_hdr.b_dsobj = zio->io_bookmark.zb_objset;
5426 zio_crypt_decode_params_bp(bp, hdr->b_crypt_hdr.b_salt,
5427 hdr->b_crypt_hdr.b_iv);
5429 if (zio->io_error == 0) {
5433 tmpbuf = abd_borrow_buf_copy(zio->io_abd,
5436 hdr->b_crypt_hdr.b_mac);
5437 abd_return_buf(zio->io_abd, tmpbuf,
5441 hdr->b_crypt_hdr.b_mac);
5446 if (zio->io_error == 0) {
5448 if (BP_SHOULD_BYTESWAP(zio->io_bp)) {
5449 if (BP_GET_LEVEL(zio->io_bp) > 0) {
5450 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64;
5452 hdr->b_l1hdr.b_byteswap =
5453 DMU_OT_BYTESWAP(BP_GET_TYPE(zio->io_bp));
5456 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
5459 hdr->b_complevel = zio->io_prop.zp_complevel;
5467 callback_list = hdr->b_l1hdr.b_acb;
5469 hdr->b_l1hdr.b_acb = NULL;
5478 for (acb = callback_list; acb != NULL; acb = acb->acb_next) {
5483 if (!acb->acb_done || acb->acb_nobuf)
5488 if (zio->io_error != 0)
5491 int error = arc_buf_alloc_impl(hdr, zio->io_spa,
5492 &acb->acb_zb, acb->acb_private, acb->acb_encrypted,
5493 acb->acb_compressed, acb->acb_noauth, B_TRUE,
5494 &acb->acb_buf);
5497 * Assert non-speculative zios didn't fail because an
5500 ASSERT((zio->io_flags & ZIO_FLAG_SPECULATIVE) ||
5510 if ((zio->io_flags & ZIO_FLAG_SPECULATIVE) == 0) {
5511 spa_log_error(zio->io_spa, &acb->acb_zb,
5512 BP_GET_LOGICAL_BIRTH(zio->io_bp));
5515 zio->io_spa, NULL, &acb->acb_zb, zio, 0);
5534 zio->io_error = error;
5547 if (zio->io_error == 0) {
5548 arc_hdr_verify(hdr, zio->io_bp);
5551 if (hdr->b_l1hdr.b_state != arc_anon)
5565 if (acb->acb_done != NULL) {
5566 if (zio->io_error != 0 && acb->acb_buf != NULL) {
5572 arc_buf_destroy(acb->acb_buf,
5573 acb->acb_private);
5574 acb->acb_buf = NULL;
5576 acb->acb_done(zio, &zio->io_bookmark, zio->io_bp,
5577 acb->acb_buf, acb->acb_private);
5580 if (acb->acb_zio_dummy != NULL) {
5581 acb->acb_zio_dummy->io_error = zio->io_error;
5582 zio_nowait(acb->acb_zio_dummy);
5585 callback_list = acb->acb_prev;
5586 if (acb->acb_wait) {
5587 mutex_enter(&acb->acb_wait_lock);
5588 acb->acb_wait_error = zio->io_error;
5589 acb->acb_wait = B_FALSE;
5590 cv_signal(&acb->acb_wait_cv);
5591 mutex_exit(&acb->acb_wait_lock);
5619 arc_state_t *state = hdr->b_l1hdr.b_state;
5623 * more compile-time checking.
5625 switch (state->arcs_state) {
5659 * If a read request arrives for a block that has a read in-progress,
5660 * either wait for the in-progress read to complete (and return the
5720 (hdr->b_l1hdr.b_pabd != NULL && !encrypted_read))) {
5743 zio_t *head_zio = hdr->b_l1hdr.b_acb->acb_zio_head;
5745 if ((hdr->b_flags & ARC_FLAG_PRIO_ASYNC_READ) &&
5749 * an in-flight async read. Request that the
5781 acb->acb_done = done;
5782 acb->acb_private = private;
5783 acb->acb_compressed = compressed_read;
5784 acb->acb_encrypted = encrypted_read;
5785 acb->acb_noauth = noauth_read;
5786 acb->acb_nobuf = no_buf;
5788 acb->acb_wait = B_TRUE;
5789 mutex_init(&acb->acb_wait_lock, NULL,
5791 cv_init(&acb->acb_wait_cv, NULL,
5794 acb->acb_zb = *zb;
5796 acb->acb_zio_dummy = zio_null(pio,
5799 acb->acb_zio_head = head_zio;
5800 acb->acb_next = hdr->b_l1hdr.b_acb;
5801 hdr->b_l1hdr.b_acb->acb_prev = acb;
5802 hdr->b_l1hdr.b_acb = acb;
5811 mutex_enter(&acb->acb_wait_lock);
5812 while (acb->acb_wait) {
5813 cv_wait(&acb->acb_wait_cv,
5814 &acb->acb_wait_lock);
5816 rc = acb->acb_wait_error;
5817 mutex_exit(&acb->acb_wait_lock);
5818 mutex_destroy(&acb->acb_wait_lock);
5819 cv_destroy(&acb->acb_wait_cv);
5825 ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
5826 hdr->b_l1hdr.b_state == arc_mfu ||
5827 hdr->b_l1hdr.b_state == arc_uncached);
5847 spa_log_error(spa, zb, hdr->b_birth);
5912 hdr->b_dva = *BP_IDENTITY(bp);
5913 hdr->b_birth = BP_GET_BIRTH(bp);
5927 * L2-only (and thus didn't have an L1 hdr),
5935 if (GHOST_STATE(hdr->b_l1hdr.b_state)) {
5936 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
5940 &hdr->b_l1hdr.b_refcnt));
5941 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
5943 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
5957 acb->acb_wait = B_TRUE;
5958 mutex_init(&acb->acb_wait_lock, NULL,
5960 cv_init(&acb->acb_wait_cv, NULL, CV_DEFAULT,
5962 acb->acb_zio_head =
5963 hdr->b_l1hdr.b_acb->acb_zio_head;
5964 acb->acb_next = hdr->b_l1hdr.b_acb;
5965 hdr->b_l1hdr.b_acb->acb_prev = acb;
5966 hdr->b_l1hdr.b_acb = acb;
5968 mutex_enter(&acb->acb_wait_lock);
5969 while (acb->acb_wait) {
5970 cv_wait(&acb->acb_wait_cv,
5971 &acb->acb_wait_lock);
5973 mutex_exit(&acb->acb_wait_lock);
5974 mutex_destroy(&acb->acb_wait_lock);
5975 cv_destroy(&acb->acb_wait_cv);
6000 hdr_abd = hdr->b_crypt_hdr.b_rabd;
6003 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
6005 hdr_abd = hdr->b_l1hdr.b_pabd;
6026 ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state));
6029 acb->acb_done = done;
6030 acb->acb_private = private;
6031 acb->acb_compressed = compressed_read;
6032 acb->acb_encrypted = encrypted_read;
6033 acb->acb_noauth = noauth_read;
6034 acb->acb_zb = *zb;
6036 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
6037 hdr->b_l1hdr.b_acb = acb;
6040 (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) {
6041 devw = hdr->b_l2hdr.b_dev->l2ad_writing;
6042 addr = hdr->b_l2hdr.b_daddr;
6054 * the read IO is still in-flight.
6086 spa->spa_l2cache.sav_count > 0;
6105 hdr->b_l2hdr.b_hits++;
6109 cb->l2rcb_hdr = hdr;
6110 cb->l2rcb_bp = *bp;
6111 cb->l2rcb_zb = *zb;
6112 cb->l2rcb_flags = zio_flags;
6129 cb->l2rcb_abd = abd;
6135 addr + asize <= vd->vdev_psize -
6153 acb->acb_zio_head = rzio;
6190 * faulted cache device - that's also a miss.)
6209 acb->acb_zio_head = rzio;
6235 zio->io_error = rc;
6247 p->p_pfunc = func;
6248 p->p_private = private;
6249 list_link_init(&p->p_node);
6250 zfs_refcount_create(&p->p_refcnt);
6253 zfs_refcount_add(&p->p_refcnt, &arc_prune_list);
6266 if (zfs_refcount_remove(&p->p_refcnt, &arc_prune_list) > 0)
6273 ASSERT0(zfs_refcount_count(&p->p_refcnt));
6274 zfs_refcount_destroy(&p->p_refcnt);
6286 arc_prune_func_t *func = ap->p_pfunc;
6289 func(ap->p_adjust, ap->p_private);
6291 (void) zfs_refcount_remove(&ap->p_refcnt, func);
6313 if (zfs_refcount_count(&ap->p_refcnt) >= 2)
6316 zfs_refcount_add(&ap->p_refcnt, ap->p_pfunc);
6317 ap->p_adjust = adjust;
6320 (void) zfs_refcount_remove(&ap->p_refcnt, ap->p_pfunc);
6346 * (i.e. prefetch) or has some other reference (i.e. a dedup-ed,
6347 * dmu_sync-ed block). A block may also have a reference if it is
6348 * part of a dedup-ed, dmu_synced write. The dmu_sync() function would
6365 zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
6384 arc_buf_hdr_t *hdr = buf->b_hdr;
6399 if (hdr->b_l1hdr.b_state == arc_anon) {
6404 ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);
6406 ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1);
6407 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
6409 hdr->b_l1hdr.b_arc_access = 0;
6429 arc_state_t *state = hdr->b_l1hdr.b_state;
6434 ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), >, 0);
6437 mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx);
6450 mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx);
6456 if (hdr->b_l1hdr.b_buf != buf || !ARC_BUF_LAST(buf)) {
6458 uint64_t spa = hdr->b_spa;
6464 VERIFY3U(hdr->b_type, ==, type);
6466 ASSERT(hdr->b_l1hdr.b_buf != buf || buf->b_next != NULL);
6470 ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf);
6487 ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf);
6506 abd_copy_from_buf(hdr->b_l1hdr.b_pabd,
6507 buf->b_data, psize);
6509 VERIFY3P(lastbuf->b_data, !=, NULL);
6524 ASSERT(hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr));
6527 (void) zfs_refcount_remove_many(&state->arcs_size[type],
6530 if (zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
6533 &state->arcs_esize[type],
6547 compress, hdr->b_complevel, type);
6548 ASSERT3P(nhdr->b_l1hdr.b_buf, ==, NULL);
6549 ASSERT0(zfs_refcount_count(&nhdr->b_l1hdr.b_refcnt));
6550 VERIFY3U(nhdr->b_type, ==, type);
6553 nhdr->b_l1hdr.b_buf = buf;
6554 (void) zfs_refcount_add(&nhdr->b_l1hdr.b_refcnt, tag);
6555 buf->b_hdr = nhdr;
6557 (void) zfs_refcount_add_many(&arc_anon->arcs_size[type],
6560 ASSERT(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt) == 1);
6562 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
6564 hdr->b_l1hdr.b_mru_hits = 0;
6565 hdr->b_l1hdr.b_mru_ghost_hits = 0;
6566 hdr->b_l1hdr.b_mfu_hits = 0;
6567 hdr->b_l1hdr.b_mfu_ghost_hits = 0;
6569 hdr->b_l1hdr.b_arc_access = 0;
6580 return (buf->b_data != NULL &&
6581 buf->b_hdr->b_l1hdr.b_state == arc_anon);
6588 return (zfs_refcount_count(&buf->b_hdr->b_l1hdr.b_refcnt));
6595 arc_write_callback_t *callback = zio->io_private;
6596 arc_buf_t *buf = callback->awcb_buf;
6597 arc_buf_hdr_t *hdr = buf->b_hdr;
6598 blkptr_t *bp = zio->io_bp;
6603 ASSERT(!zfs_refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt));
6604 ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
6611 if (zio->io_flags & ZIO_FLAG_REEXECUTED) {
6614 if (hdr->b_l1hdr.b_pabd != NULL) {
6626 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
6631 callback->awcb_ready(zio, buf, callback->awcb_private);
6634 ASSERT(zio->io_flags & ZIO_FLAG_REEXECUTED);
6646 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64;
6648 hdr->b_l1hdr.b_byteswap =
6652 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
6656 hdr->b_crypt_hdr.b_ot = BP_GET_TYPE(bp);
6657 hdr->b_crypt_hdr.b_dsobj = zio->io_bookmark.zb_objset;
6658 zio_crypt_decode_params_bp(bp, hdr->b_crypt_hdr.b_salt,
6659 hdr->b_crypt_hdr.b_iv);
6660 zio_crypt_decode_mac_bp(bp, hdr->b_crypt_hdr.b_mac);
6671 buf->b_flags &= ~ARC_BUF_FLAG_ENCRYPTED;
6673 buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
6675 buf->b_flags &= ~ARC_BUF_FLAG_ENCRYPTED;
6676 buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED;
6691 hdr->b_complevel = zio->io_prop.zp_complevel;
6693 if (zio->io_error != 0 || psize == 0)
6715 abd_copy(hdr->b_crypt_hdr.b_rabd, zio->io_abd, psize);
6728 abd_copy(hdr->b_crypt_hdr.b_rabd, zio->io_abd, psize);
6733 abd_copy(hdr->b_l1hdr.b_pabd, zio->io_abd, psize);
6735 ASSERT3U(zio->io_orig_size, ==, arc_hdr_size(hdr));
6737 abd_copy_from_buf(hdr->b_l1hdr.b_pabd, buf->b_data,
6741 ASSERT3P(buf->b_data, ==, abd_to_buf(zio->io_orig_abd));
6742 ASSERT3U(zio->io_orig_size, ==, arc_buf_size(buf));
6743 ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);
6757 arc_write_callback_t *callback = zio->io_private;
6758 arc_buf_t *buf = callback->awcb_buf;
6760 callback->awcb_children_ready(zio, buf, callback->awcb_private);
6766 arc_write_callback_t *callback = zio->io_private;
6767 arc_buf_t *buf = callback->awcb_buf;
6768 arc_buf_hdr_t *hdr = buf->b_hdr;
6770 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
6772 if (zio->io_error == 0) {
6773 arc_hdr_verify(hdr, zio->io_bp);
6775 if (BP_IS_HOLE(zio->io_bp) || BP_IS_EMBEDDED(zio->io_bp)) {
6778 hdr->b_dva = *BP_IDENTITY(zio->io_bp);
6779 hdr->b_birth = BP_GET_BIRTH(zio->io_bp);
6786 * If the block to be written was all-zero or compressed enough to be
6795 ASSERT3U(zio->io_error, ==, 0);
6803 * sync-to-convergence, because we remove
6806 if (zio->io_flags & ZIO_FLAG_IO_REWRITE) {
6807 if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
6811 &exists->b_l1hdr.b_refcnt));
6817 } else if (zio->io_flags & ZIO_FLAG_NOPWRITE) {
6819 ASSERT(zio->io_prop.zp_nopwrite);
6820 if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
6825 ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
6826 ASSERT(ARC_BUF_LAST(hdr->b_l1hdr.b_buf));
6827 ASSERT(hdr->b_l1hdr.b_state == arc_anon);
6828 ASSERT(BP_GET_DEDUP(zio->io_bp));
6829 ASSERT(BP_GET_LEVEL(zio->io_bp) == 0);
6835 if (exists == NULL && hdr->b_l1hdr.b_state == arc_anon)
6843 callback->awcb_done(zio, buf, callback->awcb_private);
6845 abd_free(zio->io_abd);
6857 arc_buf_hdr_t *hdr = buf->b_hdr;
6866 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
6867 ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
6877 localprop.zp_complevel = hdr->b_complevel;
6879 (hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS) ?
6881 memcpy(localprop.zp_salt, hdr->b_crypt_hdr.b_salt,
6883 memcpy(localprop.zp_iv, hdr->b_crypt_hdr.b_iv,
6885 memcpy(localprop.zp_mac, hdr->b_crypt_hdr.b_mac,
6890 MIN(localprop.zp_copies, SPA_DVAS_PER_BP - 1);
6896 localprop.zp_complevel = hdr->b_complevel;
6900 callback->awcb_ready = ready;
6901 callback->awcb_children_ready = children_ready;
6902 callback->awcb_done = done;
6903 callback->awcb_private = private;
6904 callback->awcb_buf = buf;
6910 if (hdr->b_l1hdr.b_pabd != NULL) {
6923 VERIFY3P(buf->b_data, !=, NULL);
6933 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
6936 abd_get_from_buf(buf->b_data, HDR_GET_LSIZE(hdr)),
6947 atomic_add_64(&arc_tempreserve, -reserve);
6981 (zfs_refcount_count(&arc_anon->arcs_size[ARC_BUFC_DATA]) +
6982 zfs_refcount_count(&arc_anon->arcs_size[ARC_BUFC_METADATA]) -
7018 &arc_anon->arcs_esize[ARC_BUFC_METADATA]);
7020 zfs_refcount_count(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
7041 data->value.ui64 =
7042 zfs_refcount_count(&state->arcs_size[ARC_BUFC_DATA]);
7043 metadata->value.ui64 =
7044 zfs_refcount_count(&state->arcs_size[ARC_BUFC_METADATA]);
7045 size->value.ui64 = data->value.ui64 + metadata->value.ui64;
7046 evict_data->value.ui64 =
7047 zfs_refcount_count(&state->arcs_esize[ARC_BUFC_DATA]);
7048 evict_metadata->value.ui64 =
7049 zfs_refcount_count(&state->arcs_esize[ARC_BUFC_METADATA]);
7055 arc_stats_t *as = ksp->ks_data;
7060 as->arcstat_hits.value.ui64 =
7062 as->arcstat_iohits.value.ui64 =
7064 as->arcstat_misses.value.ui64 =
7066 as->arcstat_demand_data_hits.value.ui64 =
7068 as->arcstat_demand_data_iohits.value.ui64 =
7070 as->arcstat_demand_data_misses.value.ui64 =
7072 as->arcstat_demand_metadata_hits.value.ui64 =
7074 as->arcstat_demand_metadata_iohits.value.ui64 =
7076 as->arcstat_demand_metadata_misses.value.ui64 =
7078 as->arcstat_prefetch_data_hits.value.ui64 =
7080 as->arcstat_prefetch_data_iohits.value.ui64 =
7082 as->arcstat_prefetch_data_misses.value.ui64 =
7084 as->arcstat_prefetch_metadata_hits.value.ui64 =
7086 as->arcstat_prefetch_metadata_iohits.value.ui64 =
7088 as->arcstat_prefetch_metadata_misses.value.ui64 =
7090 as->arcstat_mru_hits.value.ui64 =
7092 as->arcstat_mru_ghost_hits.value.ui64 =
7094 as->arcstat_mfu_hits.value.ui64 =
7096 as->arcstat_mfu_ghost_hits.value.ui64 =
7098 as->arcstat_uncached_hits.value.ui64 =
7100 as->arcstat_deleted.value.ui64 =
7102 as->arcstat_mutex_miss.value.ui64 =
7104 as->arcstat_access_skip.value.ui64 =
7106 as->arcstat_evict_skip.value.ui64 =
7108 as->arcstat_evict_not_enough.value.ui64 =
7110 as->arcstat_evict_l2_cached.value.ui64 =
7112 as->arcstat_evict_l2_eligible.value.ui64 =
7114 as->arcstat_evict_l2_eligible_mfu.value.ui64 =
7116 as->arcstat_evict_l2_eligible_mru.value.ui64 =
7118 as->arcstat_evict_l2_ineligible.value.ui64 =
7120 as->arcstat_evict_l2_skip.value.ui64 =
7122 as->arcstat_hash_elements.value.ui64 =
7123 as->arcstat_hash_elements_max.value.ui64 =
7125 as->arcstat_hash_collisions.value.ui64 =
7127 as->arcstat_hash_chains.value.ui64 =
7129 as->arcstat_size.value.ui64 =
7131 as->arcstat_compressed_size.value.ui64 =
7133 as->arcstat_uncompressed_size.value.ui64 =
7135 as->arcstat_overhead_size.value.ui64 =
7137 as->arcstat_hdr_size.value.ui64 =
7139 as->arcstat_data_size.value.ui64 =
7141 as->arcstat_metadata_size.value.ui64 =
7143 as->arcstat_dbuf_size.value.ui64 =
7146 as->arcstat_other_size.value.ui64 =
7153 &as->arcstat_anon_size,
7154 &as->arcstat_anon_data,
7155 &as->arcstat_anon_metadata,
7156 &as->arcstat_anon_evictable_data,
7157 &as->arcstat_anon_evictable_metadata);
7159 &as->arcstat_mru_size,
7160 &as->arcstat_mru_data,
7161 &as->arcstat_mru_metadata,
7162 &as->arcstat_mru_evictable_data,
7163 &as->arcstat_mru_evictable_metadata);
7165 &as->arcstat_mru_ghost_size,
7166 &as->arcstat_mru_ghost_data,
7167 &as->arcstat_mru_ghost_metadata,
7168 &as->arcstat_mru_ghost_evictable_data,
7169 &as->arcstat_mru_ghost_evictable_metadata);
7171 &as->arcstat_mfu_size,
7172 &as->arcstat_mfu_data,
7173 &as->arcstat_mfu_metadata,
7174 &as->arcstat_mfu_evictable_data,
7175 &as->arcstat_mfu_evictable_metadata);
7177 &as->arcstat_mfu_ghost_size,
7178 &as->arcstat_mfu_ghost_data,
7179 &as->arcstat_mfu_ghost_metadata,
7180 &as->arcstat_mfu_ghost_evictable_data,
7181 &as->arcstat_mfu_ghost_evictable_metadata);
7183 &as->arcstat_uncached_size,
7184 &as->arcstat_uncached_data,
7185 &as->arcstat_uncached_metadata,
7186 &as->arcstat_uncached_evictable_data,
7187 &as->arcstat_uncached_evictable_metadata);
7189 as->arcstat_dnode_size.value.ui64 =
7191 as->arcstat_bonus_size.value.ui64 =
7193 as->arcstat_l2_hits.value.ui64 =
7195 as->arcstat_l2_misses.value.ui64 =
7197 as->arcstat_l2_prefetch_asize.value.ui64 =
7199 as->arcstat_l2_mru_asize.value.ui64 =
7201 as->arcstat_l2_mfu_asize.value.ui64 =
7203 as->arcstat_l2_bufc_data_asize.value.ui64 =
7205 as->arcstat_l2_bufc_metadata_asize.value.ui64 =
7207 as->arcstat_l2_feeds.value.ui64 =
7209 as->arcstat_l2_rw_clash.value.ui64 =
7211 as->arcstat_l2_read_bytes.value.ui64 =
7213 as->arcstat_l2_write_bytes.value.ui64 =
7215 as->arcstat_l2_writes_sent.value.ui64 =
7217 as->arcstat_l2_writes_done.value.ui64 =
7219 as->arcstat_l2_writes_error.value.ui64 =
7221 as->arcstat_l2_writes_lock_retry.value.ui64 =
7223 as->arcstat_l2_evict_lock_retry.value.ui64 =
7225 as->arcstat_l2_evict_reading.value.ui64 =
7227 as->arcstat_l2_evict_l1cached.value.ui64 =
7229 as->arcstat_l2_free_on_write.value.ui64 =
7231 as->arcstat_l2_abort_lowmem.value.ui64 =
7233 as->arcstat_l2_cksum_bad.value.ui64 =
7235 as->arcstat_l2_io_error.value.ui64 =
7237 as->arcstat_l2_lsize.value.ui64 =
7239 as->arcstat_l2_psize.value.ui64 =
7241 as->arcstat_l2_hdr_size.value.ui64 =
7243 as->arcstat_l2_log_blk_writes.value.ui64 =
7245 as->arcstat_l2_log_blk_asize.value.ui64 =
7247 as->arcstat_l2_log_blk_count.value.ui64 =
7249 as->arcstat_l2_rebuild_success.value.ui64 =
7251 as->arcstat_l2_rebuild_abort_unsupported.value.ui64 =
7253 as->arcstat_l2_rebuild_abort_io_errors.value.ui64 =
7255 as->arcstat_l2_rebuild_abort_dh_errors.value.ui64 =
7257 as->arcstat_l2_rebuild_abort_cksum_lb_errors.value.ui64 =
7259 as->arcstat_l2_rebuild_abort_lowmem.value.ui64 =
7261 as->arcstat_l2_rebuild_size.value.ui64 =
7263 as->arcstat_l2_rebuild_asize.value.ui64 =
7265 as->arcstat_l2_rebuild_bufs.value.ui64 =
7267 as->arcstat_l2_rebuild_bufs_precached.value.ui64 =
7269 as->arcstat_l2_rebuild_log_blks.value.ui64 =
7271 as->arcstat_memory_throttle_count.value.ui64 =
7273 as->arcstat_memory_direct_count.value.ui64 =
7275 as->arcstat_memory_indirect_count.value.ui64 =
7278 as->arcstat_memory_all_bytes.value.ui64 =
7280 as->arcstat_memory_free_bytes.value.ui64 =
7282 as->arcstat_memory_available_bytes.value.i64 =
7285 as->arcstat_prune.value.ui64 =
7287 as->arcstat_meta_used.value.ui64 =
7289 as->arcstat_async_upgrade_sync.value.ui64 =
7291 as->arcstat_predictive_prefetch.value.ui64 =
7293 as->arcstat_demand_hit_predictive_prefetch.value.ui64 =
7295 as->arcstat_demand_iohit_predictive_prefetch.value.ui64 =
7297 as->arcstat_prescient_prefetch.value.ui64 =
7299 as->arcstat_demand_hit_prescient_prefetch.value.ui64 =
7301 as->arcstat_demand_iohit_prescient_prefetch.value.ui64 =
7303 as->arcstat_raw_size.value.ui64 =
7305 as->arcstat_cached_only_in_progress.value.ui64 =
7307 as->arcstat_abd_chunk_waste_size.value.ui64 =
7345 return ((unsigned int)buf_hash(hdr->b_spa, &hdr->b_dva, hdr->b_birth) %
7367 * updated manually. Non-zero zfs_* values which differ from the currently set
7375 /* Valid range: 32M - <arc_c_max> */
7384 /* Valid range: 64M - <all physical memory> */
7395 /* Valid range: 0 - <all physical memory> */
7400 /* Valid range: 1 - N */
7404 /* Valid range: 1 - N */
7407 arc_no_grow_shift = MIN(arc_no_grow_shift, arc_shrink_shift -1);
7410 /* Valid range: 1 - N ms */
7414 /* Valid range: 1 - N ms */
7420 /* Valid range: 0 - 100 */
7426 /* Valid range: 0 - <all physical memory> */
7446 arc_state_multilist_init(&arc_mru->arcs_list[ARC_BUFC_METADATA],
7448 arc_state_multilist_init(&arc_mru->arcs_list[ARC_BUFC_DATA],
7450 arc_state_multilist_init(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
7452 arc_state_multilist_init(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
7454 arc_state_multilist_init(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
7456 arc_state_multilist_init(&arc_mfu->arcs_list[ARC_BUFC_DATA],
7458 arc_state_multilist_init(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
7460 arc_state_multilist_init(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
7462 arc_state_multilist_init(&arc_uncached->arcs_list[ARC_BUFC_METADATA],
7464 arc_state_multilist_init(&arc_uncached->arcs_list[ARC_BUFC_DATA],
7471 arc_state_multilist_init(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA],
7473 arc_state_multilist_init(&arc_l2c_only->arcs_list[ARC_BUFC_DATA],
7478 * any ARC state. The markers will be pre-allocated so as to minimize
7483 zfs_refcount_create(&arc_anon->arcs_esize[ARC_BUFC_METADATA]);
7484 zfs_refcount_create(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
7485 zfs_refcount_create(&arc_mru->arcs_esize[ARC_BUFC_METADATA]);
7486 zfs_refcount_create(&arc_mru->arcs_esize[ARC_BUFC_DATA]);
7487 zfs_refcount_create(&arc_mru_ghost->arcs_esize[ARC_BUFC_METADATA]);
7488 zfs_refcount_create(&arc_mru_ghost->arcs_esize[ARC_BUFC_DATA]);
7489 zfs_refcount_create(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]);
7490 zfs_refcount_create(&arc_mfu->arcs_esize[ARC_BUFC_DATA]);
7491 zfs_refcount_create(&arc_mfu_ghost->arcs_esize[ARC_BUFC_METADATA]);
7492 zfs_refcount_create(&arc_mfu_ghost->arcs_esize[ARC_BUFC_DATA]);
7493 zfs_refcount_create(&arc_l2c_only->arcs_esize[ARC_BUFC_METADATA]);
7494 zfs_refcount_create(&arc_l2c_only->arcs_esize[ARC_BUFC_DATA]);
7495 zfs_refcount_create(&arc_uncached->arcs_esize[ARC_BUFC_METADATA]);
7496 zfs_refcount_create(&arc_uncached->arcs_esize[ARC_BUFC_DATA]);
7498 zfs_refcount_create(&arc_anon->arcs_size[ARC_BUFC_DATA]);
7499 zfs_refcount_create(&arc_anon->arcs_size[ARC_BUFC_METADATA]);
7500 zfs_refcount_create(&arc_mru->arcs_size[ARC_BUFC_DATA]);
7501 zfs_refcount_create(&arc_mru->arcs_size[ARC_BUFC_METADATA]);
7502 zfs_refcount_create(&arc_mru_ghost->arcs_size[ARC_BUFC_DATA]);
7503 zfs_refcount_create(&arc_mru_ghost->arcs_size[ARC_BUFC_METADATA]);
7504 zfs_refcount_create(&arc_mfu->arcs_size[ARC_BUFC_DATA]);
7505 zfs_refcount_create(&arc_mfu->arcs_size[ARC_BUFC_METADATA]);
7506 zfs_refcount_create(&arc_mfu_ghost->arcs_size[ARC_BUFC_DATA]);
7507 zfs_refcount_create(&arc_mfu_ghost->arcs_size[ARC_BUFC_METADATA]);
7508 zfs_refcount_create(&arc_l2c_only->arcs_size[ARC_BUFC_DATA]);
7509 zfs_refcount_create(&arc_l2c_only->arcs_size[ARC_BUFC_METADATA]);
7510 zfs_refcount_create(&arc_uncached->arcs_size[ARC_BUFC_DATA]);
7511 zfs_refcount_create(&arc_uncached->arcs_size[ARC_BUFC_METADATA]);
7513 wmsum_init(&arc_mru_ghost->arcs_hits[ARC_BUFC_DATA], 0);
7514 wmsum_init(&arc_mru_ghost->arcs_hits[ARC_BUFC_METADATA], 0);
7515 wmsum_init(&arc_mfu_ghost->arcs_hits[ARC_BUFC_DATA], 0);
7516 wmsum_init(&arc_mfu_ghost->arcs_hits[ARC_BUFC_METADATA], 0);
7617 arc_anon->arcs_state = ARC_STATE_ANON;
7618 arc_mru->arcs_state = ARC_STATE_MRU;
7619 arc_mru_ghost->arcs_state = ARC_STATE_MRU_GHOST;
7620 arc_mfu->arcs_state = ARC_STATE_MFU;
7621 arc_mfu_ghost->arcs_state = ARC_STATE_MFU_GHOST;
7622 arc_l2c_only->arcs_state = ARC_STATE_L2C_ONLY;
7623 arc_uncached->arcs_state = ARC_STATE_UNCACHED;
7629 zfs_refcount_destroy(&arc_anon->arcs_esize[ARC_BUFC_METADATA]);
7630 zfs_refcount_destroy(&arc_anon->arcs_esize[ARC_BUFC_DATA]);
7631 zfs_refcount_destroy(&arc_mru->arcs_esize[ARC_BUFC_METADATA]);
7632 zfs_refcount_destroy(&arc_mru->arcs_esize[ARC_BUFC_DATA]);
7633 zfs_refcount_destroy(&arc_mru_ghost->arcs_esize[ARC_BUFC_METADATA]);
7634 zfs_refcount_destroy(&arc_mru_ghost->arcs_esize[ARC_BUFC_DATA]);
7635 zfs_refcount_destroy(&arc_mfu->arcs_esize[ARC_BUFC_METADATA]);
7636 zfs_refcount_destroy(&arc_mfu->arcs_esize[ARC_BUFC_DATA]);
7637 zfs_refcount_destroy(&arc_mfu_ghost->arcs_esize[ARC_BUFC_METADATA]);
7638 zfs_refcount_destroy(&arc_mfu_ghost->arcs_esize[ARC_BUFC_DATA]);
7639 zfs_refcount_destroy(&arc_l2c_only->arcs_esize[ARC_BUFC_METADATA]);
7640 zfs_refcount_destroy(&arc_l2c_only->arcs_esize[ARC_BUFC_DATA]);
7641 zfs_refcount_destroy(&arc_uncached->arcs_esize[ARC_BUFC_METADATA]);
7642 zfs_refcount_destroy(&arc_uncached->arcs_esize[ARC_BUFC_DATA]);
7644 zfs_refcount_destroy(&arc_anon->arcs_size[ARC_BUFC_DATA]);
7645 zfs_refcount_destroy(&arc_anon->arcs_size[ARC_BUFC_METADATA]);
7646 zfs_refcount_destroy(&arc_mru->arcs_size[ARC_BUFC_DATA]);
7647 zfs_refcount_destroy(&arc_mru->arcs_size[ARC_BUFC_METADATA]);
7648 zfs_refcount_destroy(&arc_mru_ghost->arcs_size[ARC_BUFC_DATA]);
7649 zfs_refcount_destroy(&arc_mru_ghost->arcs_size[ARC_BUFC_METADATA]);
7650 zfs_refcount_destroy(&arc_mfu->arcs_size[ARC_BUFC_DATA]);
7651 zfs_refcount_destroy(&arc_mfu->arcs_size[ARC_BUFC_METADATA]);
7652 zfs_refcount_destroy(&arc_mfu_ghost->arcs_size[ARC_BUFC_DATA]);
7653 zfs_refcount_destroy(&arc_mfu_ghost->arcs_size[ARC_BUFC_METADATA]);
7654 zfs_refcount_destroy(&arc_l2c_only->arcs_size[ARC_BUFC_DATA]);
7655 zfs_refcount_destroy(&arc_l2c_only->arcs_size[ARC_BUFC_METADATA]);
7656 zfs_refcount_destroy(&arc_uncached->arcs_size[ARC_BUFC_DATA]);
7657 zfs_refcount_destroy(&arc_uncached->arcs_size[ARC_BUFC_METADATA]);
7659 multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]);
7660 multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
7661 multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]);
7662 multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
7663 multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]);
7664 multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
7665 multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]);
7666 multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
7667 multilist_destroy(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA]);
7668 multilist_destroy(&arc_l2c_only->arcs_list[ARC_BUFC_DATA]);
7669 multilist_destroy(&arc_uncached->arcs_list[ARC_BUFC_METADATA]);
7670 multilist_destroy(&arc_uncached->arcs_list[ARC_BUFC_DATA]);
7672 wmsum_fini(&arc_mru_ghost->arcs_hits[ARC_BUFC_DATA]);
7673 wmsum_fini(&arc_mru_ghost->arcs_hits[ARC_BUFC_METADATA]);
7674 wmsum_fini(&arc_mfu_ghost->arcs_hits[ARC_BUFC_DATA]);
7675 wmsum_fini(&arc_mfu_ghost->arcs_hits[ARC_BUFC_METADATA]);
7811 * If zfs_arc_max is non-zero at init, meaning it was set in the kernel
7837 * 32-bit fixed point fractions of metadata from total ARC size,
7879 arc_ksp->ks_data = &arc_stats;
7880 arc_ksp->ks_update = arc_kstat_update;
7960 (void) zfs_refcount_remove(&p->p_refcnt, &arc_prune_list);
7961 zfs_refcount_destroy(&p->p_refcnt);
8008 * The level 2 ARC (L2ARC) is a cache layer in-between main memory and disk.
8012 * include short-stroked disks, solid state disks, and other media with
8015 * +-----------------------+
8017 * +-----------------------+
8024 * +---------------+ |
8026 * +---------------+ |
8031 * +-------+ +-------+
8034 * +-------+ +-------+
8035 * +=========+ .-----.
8036 * : L2ARC : |-_____-|
8038 * +=========+ `-_____-'
8058 * It does this by periodically scanning buffers from the eviction-end of
8069 * head --> tail
8070 * +---------------------+----------+
8071 * ARC_mfu |:::::#:::::::::::::::|o#o###o###|-->. # already on L2ARC
8072 * +---------------------+----------+ | o L2ARC eligible
8073 * ARC_mru |:#:::::::::::::::::::|#o#ooo####|-->| : ARC buffer
8074 * +---------------------+----------+ |
8079 * l2arc write hand <--[oooo]--'
8112 * 6. Writes to the L2ARC devices are grouped and sent in-sequence, so that
8162 * main ARC buffers. There are 2 linked-lists of log blocks headed by
8164 * time-wise and offset-wise interleaved, but that is an optimization rather
8170 * which contains our top-level reference structures. We update it each
8183 * ||L2 dev|....|lb |bufs |lb |bufs |lb |bufs |lb |bufs |lb |---(empty)---|
8185 * |+------+ ...--\-------/ \-----/--\------/ / |
8186 * | \--------------/ \--------------/ |
8195 * incurring a large amount of I/O round-trip latency. Having two lists
8199 * On-device data structures:
8215 * <--|bufs |lb |bufs |lb | |bufs |lb |bufs |lb |-->
8218 * <<nextwrite>> may overwrite this blk and/or its bufs --'
8235 * birth TXG uniquely identify a block in space and time - once created,
8257 if (hdr->b_spa != spa_guid || HDR_HAS_L2HDR(hdr) ||
8285 if (dev->l2ad_vdev->vdev_has_trim && l2arc_trim_ahead > 0) {
8299 size = MIN(size, (dev->l2ad_end - dev->l2ad_start) / 4);
8301 size = P2ROUNDUP(size, 1ULL << dev->l2ad_vdev->vdev_ashift);
8313 * If the ARC lists are busy, increase our write rate; if the
8315 * how much we previously wrote - if it was more than half of
8336 return (dev->l2ad_vdev == NULL || vdev_is_dead(dev->l2ad_vdev) ||
8337 dev->l2ad_rebuild || dev->l2ad_trim_all ||
8338 dev->l2ad_spa == NULL || dev->l2ad_spa->spa_is_exporting);
8365 /* loop around the list looking for a non-faulted vdev */
8397 spa_config_enter(next->l2ad_spa, SCL_L2ARC, next, RW_READER);
8413 ASSERT3P(df->l2df_abd, !=, NULL);
8414 abd_free(df->l2df_abd);
8437 cb = zio->io_private;
8439 dev = cb->l2wcb_dev;
8440 l2dhdr = dev->l2ad_dev_hdr;
8442 head = cb->l2wcb_head;
8444 buflist = &dev->l2ad_buflist;
8453 mutex_enter(&dev->l2ad_mtx);
8480 mutex_exit(&dev->l2ad_mtx);
8495 * state while in-flight due to our ARC_FLAG_L2_WRITING
8501 * Skipped - drop L2ARC entry and mark the header as no
8504 if (zio->io_error != 0) {
8506 * Error - drop L2ARC entry.
8514 ASSERT(dev->l2ad_vdev != NULL);
8517 vdev_psize_to_asize(dev->l2ad_vdev, psize);
8518 (void) zfs_refcount_remove_many(&dev->l2ad_alloc,
8537 while ((abd_buf = list_remove_tail(&cb->l2wcb_abd_list)) != NULL) {
8538 abd_free(abd_buf->abd);
8540 if (zio->io_error != 0) {
8541 lb_ptr_buf = list_remove_head(&dev->l2ad_lbptr_list);
8547 L2BLK_GET_PSIZE((lb_ptr_buf->lb_ptr)->lbp_prop);
8549 ARCSTAT_INCR(arcstat_l2_log_blk_asize, -asize);
8551 zfs_refcount_remove_many(&dev->l2ad_lb_asize, asize,
8553 (void) zfs_refcount_remove(&dev->l2ad_lb_count,
8555 kmem_free(lb_ptr_buf->lb_ptr,
8560 list_destroy(&cb->l2wcb_abd_list);
8562 if (zio->io_error != 0) {
8570 lb_ptr_buf = list_head(&dev->l2ad_lbptr_list);
8580 dev->l2ad_dev_hdr_asize);
8582 memset(&l2dhdr->dh_start_lbps[i], 0,
8587 memcpy(&l2dhdr->dh_start_lbps[i], lb_ptr_buf->lb_ptr,
8589 lb_ptr_buf = list_next(&dev->l2ad_lbptr_list,
8598 mutex_exit(&dev->l2ad_mtx);
8600 ASSERT(dev->l2ad_vdev != NULL);
8601 vdev_space_update(dev->l2ad_vdev, -bytes_dropped, 0, 0);
8612 spa_t *spa = zio->io_spa;
8613 arc_buf_hdr_t *hdr = cb->l2rcb_hdr;
8614 blkptr_t *bp = zio->io_bp;
8626 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
8641 ret = spa_do_crypt_abd(B_FALSE, spa, &cb->l2rcb_zb,
8644 hdr->b_l1hdr.b_pabd, &no_crypt);
8656 arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
8658 hdr->b_l1hdr.b_pabd = eabd;
8659 zio->io_abd = eabd;
8676 hdr->b_l1hdr.b_pabd, cabd, HDR_GET_PSIZE(hdr),
8677 HDR_GET_LSIZE(hdr), &hdr->b_complevel);
8683 arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
8685 hdr->b_l1hdr.b_pabd = cabd;
8686 zio->io_abd = cabd;
8687 zio->io_size = HDR_GET_LSIZE(hdr);
8705 l2arc_read_callback_t *cb = zio->io_private;
8709 boolean_t using_rdata = (BP_IS_ENCRYPTED(&cb->l2rcb_bp) &&
8710 (cb->l2rcb_flags & ZIO_FLAG_RAW_ENCRYPT));
8712 ASSERT3P(zio->io_vd, !=, NULL);
8713 ASSERT(zio->io_flags & ZIO_FLAG_DONT_PROPAGATE);
8715 spa_config_exit(zio->io_spa, SCL_L2ARC, zio->io_vd);
8718 hdr = cb->l2rcb_hdr;
8729 if (cb->l2rcb_abd != NULL) {
8730 ASSERT3U(arc_hdr_size(hdr), <, zio->io_size);
8731 if (zio->io_error == 0) {
8733 abd_copy(hdr->b_crypt_hdr.b_rabd,
8734 cb->l2rcb_abd, arc_hdr_size(hdr));
8736 abd_copy(hdr->b_l1hdr.b_pabd,
8737 cb->l2rcb_abd, arc_hdr_size(hdr));
8744 * - free the temporary buffer
8745 * - point zio to the real ARC buffer
8746 * - set zio size accordingly
8747 * These are required because zio is either re-used for
8752 abd_free(cb->l2rcb_abd);
8753 zio->io_size = zio->io_orig_size = arc_hdr_size(hdr);
8757 zio->io_abd = zio->io_orig_abd =
8758 hdr->b_crypt_hdr.b_rabd;
8760 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
8761 zio->io_abd = zio->io_orig_abd = hdr->b_l1hdr.b_pabd;
8765 ASSERT3P(zio->io_abd, !=, NULL);
8770 ASSERT(zio->io_abd == hdr->b_l1hdr.b_pabd ||
8771 (HDR_HAS_RABD(hdr) && zio->io_abd == hdr->b_crypt_hdr.b_rabd));
8772 zio->io_bp_copy = cb->l2rcb_bp; /* XXX fix in L2ARC 2.0 */
8773 zio->io_bp = &zio->io_bp_copy; /* XXX fix in L2ARC 2.0 */
8774 zio->io_prop.zp_complevel = hdr->b_complevel;
8786 if (valid_cksum && tfm_error == 0 && zio->io_error == 0 &&
8789 zio->io_private = hdr;
8796 if (zio->io_error != 0) {
8799 zio->io_error = SET_ERROR(EIO);
8809 if (zio->io_waiter == NULL) {
8812 hdr->b_crypt_hdr.b_rabd : hdr->b_l1hdr.b_pabd;
8814 ASSERT(!pio || pio->io_child_type == ZIO_CHILD_LOGICAL);
8816 zio = zio_read(pio, zio->io_spa, zio->io_bp,
8817 abd, zio->io_size, arc_read_done,
8818 hdr, zio->io_priority, cb->l2rcb_flags,
8819 &cb->l2rcb_zb);
8826 for (struct arc_callback *acb = hdr->b_l1hdr.b_acb;
8827 acb != NULL; acb = acb->acb_next)
8828 acb->acb_zio_head = zio;
8860 ml = &arc_mfu->arcs_list[ARC_BUFC_METADATA];
8863 ml = &arc_mru->arcs_list[ARC_BUFC_METADATA];
8866 ml = &arc_mfu->arcs_list[ARC_BUFC_DATA];
8869 ml = &arc_mru->arcs_list[ARC_BUFC_DATA];
8876 * Return a randomly-selected sublist. This is acceptable
8894 if (dev->l2ad_log_entries == 0) {
8897 ASSERT(dev->l2ad_vdev != NULL);
8902 dev->l2ad_log_entries - 1) /
8903 dev->l2ad_log_entries;
8905 return (vdev_psize_to_asize(dev->l2ad_vdev,
8924 vdev_t *vd = dev->l2ad_vdev;
8928 ASSERT(dev->l2ad_spa != NULL || all);
8930 buflist = &dev->l2ad_buflist;
8934 if (dev->l2ad_hand + distance > dev->l2ad_end) {
8944 taddr = dev->l2ad_end;
8946 taddr = dev->l2ad_hand + distance;
8956 if (dev->l2ad_first) {
8967 if (vd->vdev_has_trim && dev->l2ad_evict < taddr &&
8977 spa_config_exit(dev->l2ad_spa, SCL_L2ARC, dev);
8979 dev->l2ad_evict - VDEV_LABEL_START_SIZE,
8980 taddr - dev->l2ad_evict);
8981 spa_config_enter(dev->l2ad_spa, SCL_L2ARC, dev,
8993 dev->l2ad_evict = MAX(dev->l2ad_evict, taddr);
8998 mutex_enter(&dev->l2ad_mtx);
9005 for (lb_ptr_buf = list_tail(&dev->l2ad_lbptr_list); lb_ptr_buf;
9008 lb_ptr_buf_prev = list_prev(&dev->l2ad_lbptr_list, lb_ptr_buf);
9012 (lb_ptr_buf->lb_ptr)->lbp_prop);
9019 if (!all && l2arc_log_blkptr_valid(dev, lb_ptr_buf->lb_ptr)) {
9023 vdev_space_update(vd, -asize, 0, 0);
9024 ARCSTAT_INCR(arcstat_l2_log_blk_asize, -asize);
9026 zfs_refcount_remove_many(&dev->l2ad_lb_asize, asize,
9028 (void) zfs_refcount_remove(&dev->l2ad_lb_count,
9030 list_remove(&dev->l2ad_lbptr_list, lb_ptr_buf);
9031 kmem_free(lb_ptr_buf->lb_ptr,
9053 mutex_exit(&dev->l2ad_mtx);
9068 if (!all && (hdr->b_l2hdr.b_daddr >= dev->l2ad_evict ||
9069 hdr->b_l2hdr.b_daddr < dev->l2ad_hand)) {
9088 ASSERT(hdr->b_l1hdr.b_state != arc_l2c_only);
9104 mutex_exit(&dev->l2ad_mtx);
9116 dev->l2ad_hand = dev->l2ad_start;
9117 dev->l2ad_evict = dev->l2ad_start;
9118 dev->l2ad_first = B_FALSE;
9128 ASSERT3U(dev->l2ad_hand + distance, <=, dev->l2ad_end);
9129 if (!dev->l2ad_first)
9130 ASSERT3U(dev->l2ad_hand, <=, dev->l2ad_evict);
9144 abd_t *cabd = NULL, *eabd = NULL, *to_write = hdr->b_l1hdr.b_pabd;
9149 boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);
9167 abd_copy(to_write, hdr->b_crypt_hdr.b_rabd, psize);
9168 abd_zero_off(to_write, psize, asize - psize);
9176 abd_copy(to_write, hdr->b_l1hdr.b_pabd, size);
9178 abd_zero_off(to_write, size, asize - size);
9185 size, MIN(size, psize), hdr->b_complevel);
9188 * We can't re-compress the block into the original
9196 abd_zero_off(cabd, csize, asize - csize);
9205 * made it to this point, the key to re-encrypt
9209 ret = spa_keystore_lookup_key(spa, hdr->b_crypt_hdr.b_dsobj,
9214 ret = zio_do_crypt_abd(B_TRUE, &dck->dck_key,
9215 hdr->b_crypt_hdr.b_ot, bswap, hdr->b_crypt_hdr.b_salt,
9216 hdr->b_crypt_hdr.b_iv, mac, psize, to_write, eabd,
9225 abd_zero_off(eabd, psize, asize - psize);
9228 ASSERT0(memcmp(mac, hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN));
9238 ASSERT3P(to_write, !=, hdr->b_l1hdr.b_pabd);
9259 cb = zio->io_private;
9260 if (cb->l2rcb_abd != NULL)
9261 abd_free(cb->l2rcb_abd);
9270 * The headroom_boost is an in-out parameter used to maintain headroom boost
9286 l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr;
9288 ASSERT3P(dev->l2ad_vdev, !=, NULL);
9363 ASSERT(hdr->b_l1hdr.b_pabd != NULL ||
9366 uint64_t asize = vdev_psize_to_asize(dev->l2ad_vdev,
9410 to_write = hdr->b_crypt_hdr.b_rabd;
9415 to_write = hdr->b_l1hdr.b_pabd;
9432 hdr->b_l2hdr.b_dev = dev;
9433 hdr->b_l2hdr.b_daddr = dev->l2ad_hand;
9434 hdr->b_l2hdr.b_hits = 0;
9435 hdr->b_l2hdr.b_arcs_state =
9436 hdr->b_l1hdr.b_state->arcs_state;
9442 (void) zfs_refcount_add_many(&dev->l2ad_alloc,
9445 vdev_space_update(dev->l2ad_vdev, asize, 0, 0);
9447 mutex_enter(&dev->l2ad_mtx);
9454 list_insert_head(&dev->l2ad_buflist, head);
9456 list_insert_head(&dev->l2ad_buflist, hdr);
9457 mutex_exit(&dev->l2ad_mtx);
9465 cb->l2wcb_dev = dev;
9466 cb->l2wcb_head = head;
9467 list_create(&cb->l2wcb_abd_list,
9474 wzio = zio_write_phys(pio, dev->l2ad_vdev,
9475 dev->l2ad_hand, asize, to_write,
9480 DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev,
9486 dev->l2ad_hand += asize;
9521 if (dev->l2ad_evict != l2dhdr->dh_evict)
9527 if (!dev->l2ad_first)
9528 ASSERT3U(dev->l2ad_hand, <=, dev->l2ad_evict);
9534 dev->l2ad_writing = B_TRUE;
9536 dev->l2ad_writing = B_FALSE;
9559 * heart of the L2ARC.
9597 * doing so the next spa to feed from: dev->l2ad_spa. This
9608 spa = dev->l2ad_spa;
9612 * If the pool is read-only then force the feed thread to
9676 if (dev->l2ad_vdev == vd)
9687 l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr;
9688 uint64_t l2dhdr_asize = dev->l2ad_dev_hdr_asize;
9689 spa_t *spa = dev->l2ad_spa;
9707 if (dev->l2ad_end < l2arc_rebuild_blocks_min_l2size) {
9708 dev->l2ad_log_entries = 0;
9710 dev->l2ad_log_entries = MIN((dev->l2ad_end -
9711 dev->l2ad_start) >> SPA_MAXBLOCKSHIFT,
9718 if (l2arc_dev_hdr_read(dev) == 0 && dev->l2ad_log_entries > 0) {
9732 dev->l2ad_log_ent_idx = 0;
9733 dev->l2ad_log_blk_payload_asize = 0;
9734 dev->l2ad_log_blk_payload_start = 0;
9743 dev->l2ad_rebuild = B_TRUE;
9756 dev->l2ad_trim_all = B_TRUE;
9780 adddev->l2ad_spa = spa;
9781 adddev->l2ad_vdev = vd;
9783 l2dhdr_asize = adddev->l2ad_dev_hdr_asize =
9784 MAX(sizeof (*adddev->l2ad_dev_hdr), 1 << vd->vdev_ashift);
9785 adddev->l2ad_start = VDEV_LABEL_START_SIZE + l2dhdr_asize;
9786 adddev->l2ad_end = VDEV_LABEL_START_SIZE + vdev_get_min_asize(vd);
9787 ASSERT3U(adddev->l2ad_start, <, adddev->l2ad_end);
9788 adddev->l2ad_hand = adddev->l2ad_start;
9789 adddev->l2ad_evict = adddev->l2ad_start;
9790 adddev->l2ad_first = B_TRUE;
9791 adddev->l2ad_writing = B_FALSE;
9792 adddev->l2ad_trim_all = B_FALSE;
9793 list_link_init(&adddev->l2ad_node);
9794 adddev->l2ad_dev_hdr = kmem_zalloc(l2dhdr_asize, KM_SLEEP);
9796 mutex_init(&adddev->l2ad_mtx, NULL, MUTEX_DEFAULT, NULL);
9801 list_create(&adddev->l2ad_buflist, sizeof (arc_buf_hdr_t),
9808 list_create(&adddev->l2ad_lbptr_list, sizeof (l2arc_lb_ptr_buf_t),
9811 vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand);
9812 zfs_refcount_create(&adddev->l2ad_alloc);
9813 zfs_refcount_create(&adddev->l2ad_lb_asize);
9814 zfs_refcount_create(&adddev->l2ad_lb_count);
9853 * When onlining the cache device (ie offline->online without exporting
9855 * vdev_reopen() -> vdev_open() -> l2arc_rebuild_vdev()
9876 l2arc_dev_t *remdev = rva->rva_l2arc_dev;
9883 list_destroy(&remdev->l2ad_buflist);
9884 ASSERT(list_is_empty(&remdev->l2ad_lbptr_list));
9885 list_destroy(&remdev->l2ad_lbptr_list);
9886 mutex_destroy(&remdev->l2ad_mtx);
9887 zfs_refcount_destroy(&remdev->l2ad_alloc);
9888 zfs_refcount_destroy(&remdev->l2ad_lb_asize);
9889 zfs_refcount_destroy(&remdev->l2ad_lb_count);
9890 kmem_free(remdev->l2ad_dev_hdr, remdev->l2ad_dev_hdr_asize);
9893 uint64_t elaspsed = NSEC2MSEC(gethrtime() - start_time);
9896 (u_longlong_t)rva->rva_spa_gid,
9897 (u_longlong_t)rva->rva_vdev_gid,
9901 if (rva->rva_async)
9902 arc_async_flush_remove(rva->rva_spa_gid, 2);
9912 spa_t *spa = vd->vdev_spa;
9913 boolean_t asynchronous = spa->spa_state == POOL_STATE_EXPORTED ||
9914 spa->spa_state == POOL_STATE_DESTROYED;
9927 rva->rva_l2arc_dev = remdev;
9928 rva->rva_spa_gid = spa_load_guid(spa);
9929 rva->rva_vdev_gid = remdev->l2ad_vdev->vdev_guid;
9935 remdev->l2ad_rebuild_cancel = B_TRUE;
9936 if (remdev->l2ad_rebuild_began == B_TRUE) {
9937 while (remdev->l2ad_rebuild == B_TRUE)
9941 rva->rva_async = asynchronous;
9954 remdev->l2ad_spa = NULL;
9955 remdev->l2ad_vdev = NULL;
9964 arc_async_flush_t *af = arc_async_flush_add(rva->rva_spa_gid, 2);
9967 TQ_SLEEP, &af->af_tqent);
10043 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) {
10045 l2arc_vdev_get(spa->spa_l2cache.sav_vdevs[i]);
10051 if (dev->l2ad_rebuild && !dev->l2ad_rebuild_cancel) {
10052 dev->l2ad_rebuild_began = B_TRUE;
10064 spa->spa_export_thread == curthread);
10066 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) {
10068 l2arc_vdev_get(spa->spa_l2cache.sav_vdevs[i]);
10072 dev->l2ad_rebuild_cancel = B_TRUE;
10075 for (int i = 0; i < spa->spa_l2cache.sav_count; i++) {
10077 l2arc_vdev_get(spa->spa_l2cache.sav_vdevs[i]);
10081 if (dev->l2ad_rebuild_began == B_TRUE) {
10082 while (dev->l2ad_rebuild == B_TRUE) {
10099 VERIFY(dev->l2ad_rebuild);
10102 dev->l2ad_rebuild_began = B_FALSE;
10103 dev->l2ad_rebuild = B_FALSE;
10123 vdev_t *vd = dev->l2ad_vdev;
10124 spa_t *spa = vd->vdev_spa;
10126 l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr;
10139 * that a spa_unload or device remove can be initiated - this is
10150 dev->l2ad_evict = MAX(l2dhdr->dh_evict, dev->l2ad_start);
10151 dev->l2ad_hand = MAX(l2dhdr->dh_start_lbps[0].lbp_daddr +
10152 L2BLK_GET_PSIZE((&l2dhdr->dh_start_lbps[0])->lbp_prop),
10153 dev->l2ad_start);
10154 dev->l2ad_first = !!(l2dhdr->dh_flags & L2ARC_DEV_HDR_EVICT_FIRST);
10156 vd->vdev_trim_action_time = l2dhdr->dh_trim_action_time;
10157 vd->vdev_trim_state = l2dhdr->dh_trim_state;
10167 memcpy(lbps, l2dhdr->dh_start_lbps, sizeof (lbps));
10184 * online the L2ARC dev at a later time (or re-import the pool)
10203 uint64_t asize = L2BLK_GET_PSIZE((&lbps[0])->lbp_prop);
10211 lb_ptr_buf->lb_ptr = kmem_zalloc(sizeof (l2arc_log_blkptr_t),
10213 memcpy(lb_ptr_buf->lb_ptr, &lbps[0],
10215 mutex_enter(&dev->l2ad_mtx);
10216 list_insert_tail(&dev->l2ad_lbptr_list, lb_ptr_buf);
10219 zfs_refcount_add_many(&dev->l2ad_lb_asize, asize, lb_ptr_buf);
10220 zfs_refcount_add(&dev->l2ad_lb_count, lb_ptr_buf);
10221 mutex_exit(&dev->l2ad_mtx);
10230 * -----|||----|||---|||----|||
10232 * ---|||---|||----|||---|||
10238 * (0). Only log blocks (0)-(3) should be restored. We check
10247 lbps[0].lbp_payload_start, dev->l2ad_evict) &&
10248 !dev->l2ad_first)
10254 if (dev->l2ad_rebuild_cancel) {
10279 lbps[1] = this_lb->lb_prev_lbp;
10299 (u_longlong_t)zfs_refcount_count(&dev->l2ad_lb_count));
10304 } else if (err == 0 && zfs_refcount_count(&dev->l2ad_lb_count) > 0) {
10308 (u_longlong_t)zfs_refcount_count(&dev->l2ad_lb_count));
10309 } else if (err == 0 && zfs_refcount_count(&dev->l2ad_lb_count) == 0) {
10312 * in the device header points to invalid/non-present log
10317 memset(l2dhdr, 0, dev->l2ad_dev_hdr_asize);
10322 (u_longlong_t)zfs_refcount_count(&dev->l2ad_lb_count));
10341 l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr;
10342 const uint64_t l2dhdr_asize = dev->l2ad_dev_hdr_asize;
10345 guid = spa_guid(dev->l2ad_vdev->vdev_spa);
10349 err = zio_wait(zio_read_phys(NULL, dev->l2ad_vdev,
10361 (u_longlong_t)dev->l2ad_vdev->vdev_guid);
10365 if (l2dhdr->dh_magic == BSWAP_64(L2ARC_DEV_HDR_MAGIC))
10368 if (l2dhdr->dh_magic != L2ARC_DEV_HDR_MAGIC ||
10369 l2dhdr->dh_spa_guid != guid ||
10370 l2dhdr->dh_vdev_guid != dev->l2ad_vdev->vdev_guid ||
10371 l2dhdr->dh_version != L2ARC_PERSISTENT_VERSION ||
10372 l2dhdr->dh_log_entries != dev->l2ad_log_entries ||
10373 l2dhdr->dh_end != dev->l2ad_end ||
10374 !l2arc_range_check_overlap(dev->l2ad_start, dev->l2ad_end,
10375 l2dhdr->dh_evict) ||
10376 (l2dhdr->dh_trim_state != VDEV_TRIM_COMPLETE &&
10436 this_io = l2arc_log_blk_fetch(dev->l2ad_vdev, this_lbp,
10445 * Start issuing IO for the next log block early - this
10449 *next_io = l2arc_log_blk_fetch(dev->l2ad_vdev, next_lbp,
10458 (u_longlong_t)this_lbp->lbp_daddr,
10459 (u_longlong_t)dev->l2ad_vdev->vdev_guid);
10467 asize = L2BLK_GET_PSIZE((this_lbp)->lbp_prop);
10469 if (!ZIO_CHECKSUM_EQUAL(cksum, this_lbp->lbp_cksum)) {
10473 (u_longlong_t)this_lbp->lbp_daddr,
10474 (u_longlong_t)dev->l2ad_vdev->vdev_guid,
10475 (u_longlong_t)dev->l2ad_hand,
10476 (u_longlong_t)dev->l2ad_evict);
10482 switch (L2BLK_GET_COMPRESS((this_lbp)->lbp_prop)) {
10491 L2BLK_GET_COMPRESS((this_lbp)->lbp_prop),
10505 if (this_lb->lb_magic == BSWAP_64(L2ARC_LOG_BLK_MAGIC))
10507 if (this_lb->lb_magic != L2ARC_LOG_BLK_MAGIC) {
10512 /* Abort an in-flight fetch I/O in case of error */
10531 uint64_t log_entries = dev->l2ad_log_entries;
10540 for (int i = log_entries - 1; i >= 0; i--) {
10548 * HEAD <------ (time) ------ TAIL
10549 * direction +-----+-----+-----+-----+-----+ direction
10551 * fill +-----+-----+-----+-----+-----+
10559 * l2arc_feed_thread() as dev->l2ad_rebuild is set to true.
10561 size += L2BLK_GET_LSIZE((&lb->lb_entries[i])->le_prop);
10562 asize += vdev_psize_to_asize(dev->l2ad_vdev,
10563 L2BLK_GET_PSIZE((&lb->lb_entries[i])->le_prop));
10564 l2arc_hdr_restore(&lb->lb_entries[i], dev);
10589 arc_buf_contents_t type = L2BLK_GET_TYPE((le)->le_prop);
10590 uint64_t asize = vdev_psize_to_asize(dev->l2ad_vdev,
10591 L2BLK_GET_PSIZE((le)->le_prop));
10598 hdr = arc_buf_alloc_l2only(L2BLK_GET_LSIZE((le)->le_prop), type,
10599 dev, le->le_dva, le->le_daddr,
10600 L2BLK_GET_PSIZE((le)->le_prop), asize, le->le_birth,
10601 L2BLK_GET_COMPRESS((le)->le_prop), le->le_complevel,
10602 L2BLK_GET_PROTECTED((le)->le_prop),
10603 L2BLK_GET_PREFETCH((le)->le_prop),
10604 L2BLK_GET_STATE((le)->le_prop));
10611 vdev_space_update(dev->l2ad_vdev, asize, 0, 0);
10613 mutex_enter(&dev->l2ad_mtx);
10614 list_insert_tail(&dev->l2ad_buflist, hdr);
10615 (void) zfs_refcount_add_many(&dev->l2ad_alloc, arc_hdr_size(hdr), hdr);
10616 mutex_exit(&dev->l2ad_mtx);
10630 exists->b_l2hdr.b_dev = dev;
10631 exists->b_l2hdr.b_daddr = le->le_daddr;
10632 exists->b_l2hdr.b_arcs_state =
10633 L2BLK_GET_STATE((le)->le_prop);
10636 mutex_enter(&dev->l2ad_mtx);
10637 list_insert_tail(&dev->l2ad_buflist, exists);
10638 (void) zfs_refcount_add_many(&dev->l2ad_alloc,
10640 mutex_exit(&dev->l2ad_mtx);
10642 vdev_space_update(dev->l2ad_vdev, asize, 0, 0);
10670 asize = L2BLK_GET_PSIZE((lbp)->lbp_prop);
10674 cb->l2rcb_abd = abd_get_from_buf(lb, asize);
10675 pio = zio_root(vd->vdev_spa, l2arc_blk_fetch_done, cb,
10677 (void) zio_nowait(zio_read_phys(pio, vd, lbp->lbp_daddr, asize,
10678 cb->l2rcb_abd, ZIO_CHECKSUM_OFF, NULL, NULL,
10701 l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr;
10702 const uint64_t l2dhdr_asize = dev->l2ad_dev_hdr_asize;
10706 VERIFY(spa_config_held(dev->l2ad_spa, SCL_STATE_ALL, RW_READER));
10708 l2dhdr->dh_magic = L2ARC_DEV_HDR_MAGIC;
10709 l2dhdr->dh_version = L2ARC_PERSISTENT_VERSION;
10710 l2dhdr->dh_spa_guid = spa_guid(dev->l2ad_vdev->vdev_spa);
10711 l2dhdr->dh_vdev_guid = dev->l2ad_vdev->vdev_guid;
10712 l2dhdr->dh_log_entries = dev->l2ad_log_entries;
10713 l2dhdr->dh_evict = dev->l2ad_evict;
10714 l2dhdr->dh_start = dev->l2ad_start;
10715 l2dhdr->dh_end = dev->l2ad_end;
10716 l2dhdr->dh_lb_asize = zfs_refcount_count(&dev->l2ad_lb_asize);
10717 l2dhdr->dh_lb_count = zfs_refcount_count(&dev->l2ad_lb_count);
10718 l2dhdr->dh_flags = 0;
10719 l2dhdr->dh_trim_action_time = dev->l2ad_vdev->vdev_trim_action_time;
10720 l2dhdr->dh_trim_state = dev->l2ad_vdev->vdev_trim_state;
10721 if (dev->l2ad_first)
10722 l2dhdr->dh_flags |= L2ARC_DEV_HDR_EVICT_FIRST;
10726 err = zio_wait(zio_write_phys(NULL, dev->l2ad_vdev,
10735 (u_longlong_t)dev->l2ad_vdev->vdev_guid);
10748 l2arc_log_blk_phys_t *lb = &dev->l2ad_log_blk;
10749 l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr;
10756 VERIFY3S(dev->l2ad_log_ent_idx, ==, dev->l2ad_log_entries);
10759 abd_buf->abd = abd_get_from_buf(lb, sizeof (*lb));
10761 lb_ptr_buf->lb_ptr = kmem_zalloc(sizeof (l2arc_log_blkptr_t), KM_SLEEP);
10764 lb->lb_prev_lbp = l2dhdr->dh_start_lbps[1];
10765 lb->lb_magic = L2ARC_LOG_BLK_MAGIC;
10772 list_insert_tail(&cb->l2wcb_abd_list, abd_buf);
10776 abd_buf->abd, &abd, sizeof (*lb),
10778 dev->l2ad_vdev->vdev_ashift,
10779 dev->l2ad_vdev->vdev_ashift, sizeof (*lb)), 0);
10783 asize = vdev_psize_to_asize(dev->l2ad_vdev, psize);
10790 l2dhdr->dh_start_lbps[1] = l2dhdr->dh_start_lbps[0];
10791 l2dhdr->dh_start_lbps[0].lbp_daddr = dev->l2ad_hand;
10792 l2dhdr->dh_start_lbps[0].lbp_payload_asize =
10793 dev->l2ad_log_blk_payload_asize;
10794 l2dhdr->dh_start_lbps[0].lbp_payload_start =
10795 dev->l2ad_log_blk_payload_start;
10797 (&l2dhdr->dh_start_lbps[0])->lbp_prop, sizeof (*lb));
10799 (&l2dhdr->dh_start_lbps[0])->lbp_prop, asize);
10801 (&l2dhdr->dh_start_lbps[0])->lbp_prop,
10805 abd_zero_off(abd, psize, asize - psize);
10807 (&l2dhdr->dh_start_lbps[0])->lbp_prop,
10813 (&l2dhdr->dh_start_lbps[0])->lbp_prop,
10819 &l2dhdr->dh_start_lbps[0].lbp_cksum);
10821 abd_free(abd_buf->abd);
10824 abd_buf->abd = abd;
10825 wzio = zio_write_phys(pio, dev->l2ad_vdev, dev->l2ad_hand,
10826 asize, abd_buf->abd, ZIO_CHECKSUM_OFF, NULL, NULL,
10828 DTRACE_PROBE2(l2arc__write, vdev_t *, dev->l2ad_vdev, zio_t *, wzio);
10831 dev->l2ad_hand += asize;
10832 vdev_space_update(dev->l2ad_vdev, asize, 0, 0);
10838 memcpy(lb_ptr_buf->lb_ptr, &l2dhdr->dh_start_lbps[0],
10840 mutex_enter(&dev->l2ad_mtx);
10841 list_insert_head(&dev->l2ad_lbptr_list, lb_ptr_buf);
10844 zfs_refcount_add_many(&dev->l2ad_lb_asize, asize, lb_ptr_buf);
10845 zfs_refcount_add(&dev->l2ad_lb_count, lb_ptr_buf);
10846 mutex_exit(&dev->l2ad_mtx);
10853 dev->l2ad_log_blk_payload_asize / asize);
10856 dev->l2ad_log_ent_idx = 0;
10857 dev->l2ad_log_blk_payload_asize = 0;
10858 dev->l2ad_log_blk_payload_start = 0;
10871 uint64_t asize = L2BLK_GET_PSIZE((lbp)->lbp_prop);
10872 uint64_t end = lbp->lbp_daddr + asize - 1;
10873 uint64_t start = lbp->lbp_payload_start;
10878 * - it fits entirely (including its payload) between l2ad_start and
10880 * - it has a valid size
10881 * - neither the log block itself nor part of its payload was evicted
10890 * --------------------------||||
10897 l2arc_range_check_overlap(start, end, dev->l2ad_hand) ||
10898 l2arc_range_check_overlap(start, end, dev->l2ad_evict) ||
10899 l2arc_range_check_overlap(dev->l2ad_hand, dev->l2ad_evict, start) ||
10900 l2arc_range_check_overlap(dev->l2ad_hand, dev->l2ad_evict, end);
10902 return (start >= dev->l2ad_start && end <= dev->l2ad_end &&
10904 (!evicted || dev->l2ad_first));
10916 l2arc_log_blk_phys_t *lb = &dev->l2ad_log_blk;
10919 if (dev->l2ad_log_entries == 0)
10922 int index = dev->l2ad_log_ent_idx++;
10924 ASSERT3S(index, <, dev->l2ad_log_entries);
10927 le = &lb->lb_entries[index];
10929 le->le_dva = hdr->b_dva;
10930 le->le_birth = hdr->b_birth;
10931 le->le_daddr = hdr->b_l2hdr.b_daddr;
10933 dev->l2ad_log_blk_payload_start = le->le_daddr;
10934 L2BLK_SET_LSIZE((le)->le_prop, HDR_GET_LSIZE(hdr));
10935 L2BLK_SET_PSIZE((le)->le_prop, HDR_GET_PSIZE(hdr));
10936 L2BLK_SET_COMPRESS((le)->le_prop, HDR_GET_COMPRESS(hdr));
10937 le->le_complevel = hdr->b_complevel;
10938 L2BLK_SET_TYPE((le)->le_prop, hdr->b_type);
10939 L2BLK_SET_PROTECTED((le)->le_prop, !!(HDR_PROTECTED(hdr)));
10940 L2BLK_SET_PREFETCH((le)->le_prop, !!(HDR_PREFETCH(hdr)));
10941 L2BLK_SET_STATE((le)->le_prop, hdr->b_l2hdr.b_arcs_state);
10943 dev->l2ad_log_blk_payload_asize += vdev_psize_to_asize(dev->l2ad_vdev,
10946 return (dev->l2ad_log_ent_idx == dev->l2ad_log_entries);
10950 * Checks whether a given L2ARC device address sits in a time-sequential
10954 * bottom -- Lower end of the range to check (written to earlier).
10955 * top -- Upper end of the range to check (written to later).
10956 * check -- The address for which we want to determine if it sits in
10959 * The 3-way conditional below represents the following cases:
10962 * <check>--------+-------------------+
10965 * |---------------<bottom>============<top>--------------|
10967 * bottom > top: Looped-around case:
10968 * <check>--------+------------------+
10971 * |===============<top>---------------<bottom>===========|
10974 * +---------------+---------<check>
11059 "Percent of ARC size allowed for L2ARC-only headers");