Lines Matching full:hdr
208 * arc_buf_hdr_t is cached. If the hdr is cached then the ARC allocates a new
210 * existing uncompressed arc_buf_t, decompresses the hdr's b_pabd buffer into a
211 * new data buffer, or shares the hdr's b_pabd buffer, depending on whether the
212 * hdr is compressed and the desired compression characteristics of the
215 * the last buffer in the hdr's b_buf list, however a shared compressed buf can
216 * be anywhere in the hdr's list.
218 * The diagram below shows an example of an uncompressed ARC hdr that is
249 * Writing to the ARC requires that the ARC first discard the hdr's b_pabd
256 * buffers which have been released (and hence have their own hdr, if there
257 * were originally other readers of the buf's original hdr). This ensures that
258 * the ARC only needs to update a single buf and its hdr after a write occurs.
279 * data is stored in hdr->b_crypt_hdr.b_rabd. An encrypted header, therefore,
703 #define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE)
704 #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS)
705 #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_FLAG_IO_ERROR)
706 #define HDR_PREFETCH(hdr) ((hdr)->b_flags & ARC_FLAG_PREFETCH)
707 #define HDR_PRESCIENT_PREFETCH(hdr) \
708 ((hdr)->b_flags & ARC_FLAG_PRESCIENT_PREFETCH)
709 #define HDR_COMPRESSION_ENABLED(hdr) \
710 ((hdr)->b_flags & ARC_FLAG_COMPRESSED_ARC)
712 #define HDR_L2CACHE(hdr) ((hdr)->b_flags & ARC_FLAG_L2CACHE)
713 #define HDR_UNCACHED(hdr) ((hdr)->b_flags & ARC_FLAG_UNCACHED)
714 #define HDR_L2_READING(hdr) \
715 (((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) && \
716 ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR))
717 #define HDR_L2_WRITING(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITING)
718 #define HDR_L2_EVICTED(hdr) ((hdr)->b_flags & ARC_FLAG_L2_EVICTED)
719 #define HDR_L2_WRITE_HEAD(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD)
720 #define HDR_PROTECTED(hdr) ((hdr)->b_flags & ARC_FLAG_PROTECTED)
721 #define HDR_NOAUTH(hdr) ((hdr)->b_flags & ARC_FLAG_NOAUTH)
722 #define HDR_SHARED_DATA(hdr) ((hdr)->b_flags & ARC_FLAG_SHARED_DATA)
724 #define HDR_ISTYPE_METADATA(hdr) \
725 ((hdr)->b_flags & ARC_FLAG_BUFC_METADATA)
726 #define HDR_ISTYPE_DATA(hdr) (!HDR_ISTYPE_METADATA(hdr))
728 #define HDR_HAS_L1HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L1HDR)
729 #define HDR_HAS_L2HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)
730 #define HDR_HAS_RABD(hdr) \
731 (HDR_HAS_L1HDR(hdr) && HDR_PROTECTED(hdr) && \
732 (hdr)->b_crypt_hdr.b_rabd != NULL)
733 #define HDR_ENCRYPTED(hdr) \
734 (HDR_PROTECTED(hdr) && DMU_OT_IS_ENCRYPTED((hdr)->b_crypt_hdr.b_ot))
735 #define HDR_AUTHENTICATED(hdr) \
736 (HDR_PROTECTED(hdr) && !DMU_OT_IS_ENCRYPTED((hdr)->b_crypt_hdr.b_ot))
741 #define HDR_GET_COMPRESS(hdr) ((enum zio_compress)BF32_GET((hdr)->b_flags, \
743 #define HDR_SET_COMPRESS(hdr, cmp) BF32_SET((hdr)->b_flags, \
774 #define HDR_LOCK(hdr) \
775 (BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
858 ARC_FILL_LOCKED = 1 << 0, /* hdr lock is held */
890 static void arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size,
901 static inline void arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
902 static inline void arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
907 static void l2arc_hdr_arcstats_update(arc_buf_hdr_t *hdr, boolean_t incr,
912 #define l2arc_hdr_arcstats_increment(hdr) \
913 l2arc_hdr_arcstats_update((hdr), B_TRUE, B_FALSE)
914 #define l2arc_hdr_arcstats_decrement(hdr) \
915 l2arc_hdr_arcstats_update((hdr), B_FALSE, B_FALSE)
916 #define l2arc_hdr_arcstats_increment_state(hdr) \
917 l2arc_hdr_arcstats_update((hdr), B_TRUE, B_TRUE)
918 #define l2arc_hdr_arcstats_decrement_state(hdr) \
919 l2arc_hdr_arcstats_update((hdr), B_FALSE, B_TRUE)
1014 #define HDR_EMPTY(hdr) \
1015 ((hdr)->b_dva.dva_word[0] == 0 && \
1016 (hdr)->b_dva.dva_word[1] == 0)
1018 #define HDR_EMPTY_OR_LOCKED(hdr) \
1019 (HDR_EMPTY(hdr) || MUTEX_HELD(HDR_LOCK(hdr)))
1021 #define HDR_EQUAL(spa, dva, birth, hdr) \
1022 ((hdr)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \
1023 ((hdr)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \
1024 ((hdr)->b_birth == birth) && ((hdr)->b_spa == spa)
1027 buf_discard_identity(arc_buf_hdr_t *hdr)
1029 hdr->b_dva.dva_word[0] = 0;
1030 hdr->b_dva.dva_word[1] = 0;
1031 hdr->b_birth = 0;
1041 arc_buf_hdr_t *hdr;
1044 for (hdr = buf_hash_table.ht_table[idx]; hdr != NULL;
1045 hdr = hdr->b_hash_next) {
1046 if (HDR_EQUAL(spa, dva, birth, hdr)) {
1048 return (hdr);
1064 buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp)
1066 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1071 ASSERT(!DVA_IS_EMPTY(&hdr->b_dva));
1072 ASSERT(hdr->b_birth != 0);
1073 ASSERT(!HDR_IN_HASH_TABLE(hdr));
1084 if (HDR_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr))
1088 hdr->b_hash_next = buf_hash_table.ht_table[idx];
1089 buf_hash_table.ht_table[idx] = hdr;
1090 arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
1105 buf_hash_remove(arc_buf_hdr_t *hdr)
1108 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth);
1111 ASSERT(HDR_IN_HASH_TABLE(hdr));
1114 while ((fhdr = *hdrp) != hdr) {
1118 *hdrp = hdr->b_hash_next;
1119 hdr->b_hash_next = NULL;
1120 arc_hdr_clear_flags(hdr, ARC_FLAG_IN_HASH_TABLE);
1166 arc_buf_hdr_t *hdr = vbuf;
1168 memset(hdr, 0, HDR_FULL_SIZE);
1169 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
1170 zfs_refcount_create(&hdr->b_l1hdr.b_refcnt);
1172 mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
1174 multilist_link_init(&hdr->b_l1hdr.b_arc_node);
1175 list_link_init(&hdr->b_l2hdr.b_l2node);
1185 arc_buf_hdr_t *hdr = vbuf;
1187 memset(hdr, 0, HDR_L2ONLY_SIZE);
1213 arc_buf_hdr_t *hdr = vbuf;
1215 ASSERT(HDR_EMPTY(hdr));
1216 zfs_refcount_destroy(&hdr->b_l1hdr.b_refcnt);
1218 mutex_destroy(&hdr->b_l1hdr.b_freeze_lock);
1220 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
1228 arc_buf_hdr_t *hdr = vbuf;
1230 ASSERT(HDR_EMPTY(hdr));
1337 arc_buf_hdr_t *hdr = buf->b_hdr;
1339 ASSERT(HDR_PROTECTED(hdr));
1341 memcpy(salt, hdr->b_crypt_hdr.b_salt, ZIO_DATA_SALT_LEN);
1342 memcpy(iv, hdr->b_crypt_hdr.b_iv, ZIO_DATA_IV_LEN);
1343 memcpy(mac, hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN);
1344 *byteorder = (hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS) ?
1366 arc_hdr_get_compress(arc_buf_hdr_t *hdr)
1368 return (HDR_COMPRESSION_ENABLED(hdr) ?
1369 HDR_GET_COMPRESS(hdr) : ZIO_COMPRESS_OFF);
1390 * It would be nice to assert arc_can_share() too, but the "hdr isn't
1402 arc_cksum_free(arc_buf_hdr_t *hdr)
1405 ASSERT(HDR_HAS_L1HDR(hdr));
1407 mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1408 if (hdr->b_l1hdr.b_freeze_cksum != NULL) {
1409 kmem_free(hdr->b_l1hdr.b_freeze_cksum, sizeof (zio_cksum_t));
1410 hdr->b_l1hdr.b_freeze_cksum = NULL;
1412 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1417 * Return true iff at least one of the bufs on hdr is not compressed.
1421 arc_hdr_has_uncompressed_buf(arc_buf_hdr_t *hdr)
1423 ASSERT(hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY_OR_LOCKED(hdr));
1425 for (arc_buf_t *b = hdr->b_l1hdr.b_buf; b != NULL; b = b->b_next) {
1436 * matches the checksum that is stored in the hdr. If there is no checksum,
1443 arc_buf_hdr_t *hdr = buf->b_hdr;
1452 ASSERT(HDR_HAS_L1HDR(hdr));
1454 mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1456 if (hdr->b_l1hdr.b_freeze_cksum == NULL || HDR_IO_ERROR(hdr)) {
1457 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1462 if (!ZIO_CHECKSUM_EQUAL(*hdr->b_l1hdr.b_freeze_cksum, zc))
1464 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1474 arc_cksum_is_equal(arc_buf_hdr_t *hdr, zio_t *zio)
1477 VERIFY3U(BP_GET_PSIZE(zio->io_bp), ==, HDR_GET_PSIZE(hdr));
1499 * checksum and attaches it to the buf's hdr so that we can ensure that the buf
1501 * on the hdr, this is a no-op (we only checksum uncompressed bufs).
1510 arc_buf_hdr_t *hdr = buf->b_hdr;
1511 ASSERT(HDR_HAS_L1HDR(hdr));
1512 mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
1513 if (hdr->b_l1hdr.b_freeze_cksum != NULL || ARC_BUF_COMPRESSED(buf)) {
1514 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1520 hdr->b_l1hdr.b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t),
1523 hdr->b_l1hdr.b_freeze_cksum);
1524 mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
1564 arc_buf_type(arc_buf_hdr_t *hdr)
1567 if (HDR_ISTYPE_METADATA(hdr)) {
1572 VERIFY3U(hdr->b_type, ==, type);
1601 arc_buf_hdr_t *hdr = buf->b_hdr;
1603 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
1604 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
1614 ASSERT(HDR_HAS_L1HDR(hdr));
1615 arc_cksum_free(hdr);
1636 * the hash_lock must be held or the hdr must be undiscoverable. This
1641 arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)
1643 ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
1644 hdr->b_flags |= flags;
1648 arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags)
1650 ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
1651 hdr->b_flags &= ~flags;
1662 arc_hdr_set_compress(arc_buf_hdr_t *hdr, enum zio_compress cmp)
1664 ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
1671 if (!zfs_compressed_arc_enabled || HDR_GET_PSIZE(hdr) == 0) {
1672 arc_hdr_clear_flags(hdr, ARC_FLAG_COMPRESSED_ARC);
1673 ASSERT(!HDR_COMPRESSION_ENABLED(hdr));
1675 arc_hdr_set_flags(hdr, ARC_FLAG_COMPRESSED_ARC);
1676 ASSERT(HDR_COMPRESSION_ENABLED(hdr));
1679 HDR_SET_COMPRESS(hdr, cmp);
1680 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, cmp);
1684 * Looks for another buf on the same hdr which has the data decompressed, copies
1690 arc_buf_hdr_t *hdr = buf->b_hdr;
1693 ASSERT(HDR_HAS_L1HDR(hdr));
1697 for (arc_buf_t *from = hdr->b_l1hdr.b_buf; from != NULL;
1714 * checksum on the hdr either.
1717 EQUIV(!copied, hdr->b_l1hdr.b_freeze_cksum == NULL);
1735 arc_buf_hdr_t *hdr;
1740 hdr = kmem_cache_alloc(hdr_l2only_cache, KM_SLEEP);
1741 hdr->b_birth = birth;
1742 hdr->b_type = type;
1743 hdr->b_flags = 0;
1744 arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L2HDR);
1745 HDR_SET_LSIZE(hdr, size);
1746 HDR_SET_PSIZE(hdr, psize);
1747 HDR_SET_L2SIZE(hdr, asize);
1748 arc_hdr_set_compress(hdr, compress);
1749 hdr->b_complevel = complevel;
1751 arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);
1753 arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
1754 hdr->b_spa = spa_load_guid(dev->l2ad_vdev->vdev_spa);
1756 hdr->b_dva = dva;
1758 hdr->b_l2hdr.b_dev = dev;
1759 hdr->b_l2hdr.b_daddr = daddr;
1760 hdr->b_l2hdr.b_arcs_state = arcs_state;
1762 return (hdr);
1769 arc_hdr_size(arc_buf_hdr_t *hdr)
1773 if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF &&
1774 HDR_GET_PSIZE(hdr) > 0) {
1775 size = HDR_GET_PSIZE(hdr);
1777 ASSERT3U(HDR_GET_LSIZE(hdr), !=, 0);
1778 size = HDR_GET_LSIZE(hdr);
1784 arc_hdr_authenticate(arc_buf_hdr_t *hdr, spa_t *spa, uint64_t dsobj)
1788 uint64_t lsize = HDR_GET_LSIZE(hdr);
1789 uint64_t psize = HDR_GET_PSIZE(hdr);
1790 abd_t *abd = hdr->b_l1hdr.b_pabd;
1793 ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
1794 ASSERT(HDR_AUTHENTICATED(hdr));
1805 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
1806 !HDR_COMPRESSION_ENABLED(hdr)) {
1808 csize = zio_compress_data(HDR_GET_COMPRESS(hdr),
1809 hdr->b_l1hdr.b_pabd, &abd, lsize, MIN(lsize, psize),
1810 hdr->b_complevel);
1824 if (hdr->b_crypt_hdr.b_ot == DMU_OT_OBJSET) {
1825 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF);
1828 psize, hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);
1831 hdr->b_crypt_hdr.b_mac);
1835 arc_hdr_clear_flags(hdr, ARC_FLAG_NOAUTH);
1852 arc_hdr_decrypt(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb)
1857 boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);
1859 ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
1860 ASSERT(HDR_ENCRYPTED(hdr));
1862 arc_hdr_alloc_abd(hdr, 0);
1864 ret = spa_do_crypt_abd(B_FALSE, spa, zb, hdr->b_crypt_hdr.b_ot,
1865 B_FALSE, bswap, hdr->b_crypt_hdr.b_salt, hdr->b_crypt_hdr.b_iv,
1866 hdr->b_crypt_hdr.b_mac, HDR_GET_PSIZE(hdr), hdr->b_l1hdr.b_pabd,
1867 hdr->b_crypt_hdr.b_rabd, &no_crypt);
1872 abd_copy(hdr->b_l1hdr.b_pabd, hdr->b_crypt_hdr.b_rabd,
1873 HDR_GET_PSIZE(hdr));
1881 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
1882 !HDR_COMPRESSION_ENABLED(hdr)) {
1889 cabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr, 0);
1891 ret = zio_decompress_data(HDR_GET_COMPRESS(hdr),
1892 hdr->b_l1hdr.b_pabd, cabd, HDR_GET_PSIZE(hdr),
1893 HDR_GET_LSIZE(hdr), &hdr->b_complevel);
1898 arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
1899 arc_hdr_size(hdr), hdr);
1900 hdr->b_l1hdr.b_pabd = cabd;
1906 arc_hdr_free_abd(hdr, B_FALSE);
1908 arc_free_data_buf(hdr, cabd, arc_hdr_size(hdr), hdr);
1919 arc_fill_hdr_crypt(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, spa_t *spa,
1924 ASSERT(HDR_PROTECTED(hdr));
1929 if (HDR_NOAUTH(hdr) && !noauth) {
1934 ret = arc_hdr_authenticate(hdr, spa, zb->zb_objset);
1937 } else if (HDR_HAS_RABD(hdr) && hdr->b_l1hdr.b_pabd == NULL) {
1943 ret = arc_hdr_decrypt(hdr, spa, zb);
1948 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
1971 arc_buf_hdr_t *hdr = buf->b_hdr;
1973 ASSERT(HDR_ENCRYPTED(hdr));
1974 ASSERT3U(hdr->b_crypt_hdr.b_ot, ==, DMU_OT_DNODE);
1975 ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
1976 ASSERT3PF(hdr->b_l1hdr.b_pabd, !=, NULL, "hdr %px buf %px", hdr, buf);
1978 zio_crypt_copy_dnode_bonus(hdr->b_l1hdr.b_pabd, buf->b_data,
1987 * the hdr and update the hdr's b_freeze_cksum if necessary. If the buf and hdr
1992 * buf with uncompressed data. You can't request a compressed buf on a hdr with
2002 arc_buf_hdr_t *hdr = buf->b_hdr;
2004 (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF);
2007 dmu_object_byteswap_t bswap = hdr->b_l1hdr.b_byteswap;
2008 kmutex_t *hash_lock = (flags & ARC_FILL_LOCKED) ? NULL : HDR_LOCK(hdr);
2013 IMPLY(encrypted, HDR_ENCRYPTED(hdr));
2024 ASSERT(HDR_HAS_RABD(hdr));
2025 abd_copy_to_buf(buf->b_data, hdr->b_crypt_hdr.b_rabd,
2026 HDR_GET_PSIZE(hdr));
2036 if (HDR_PROTECTED(hdr)) {
2037 error = arc_fill_hdr_crypt(hdr, hash_lock, spa,
2044 arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
2067 if (HDR_ENCRYPTED(hdr) && ARC_BUF_ENCRYPTED(buf)) {
2068 ASSERT3U(hdr->b_crypt_hdr.b_ot, ==, DMU_OT_DNODE);
2076 /* Compute the hdr's checksum if necessary */
2087 abd_copy_to_buf(buf->b_data, hdr->b_l1hdr.b_pabd,
2095 * If the buf is sharing its data with the hdr, unlink it and
2105 arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);
2106 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
2109 ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr));
2114 arc_free_data_buf(hdr, buf->b_data, HDR_GET_PSIZE(hdr),
2117 arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf);
2121 HDR_GET_LSIZE(hdr) - HDR_GET_PSIZE(hdr));
2133 * bite the bullet and decompress the data from the hdr.
2141 HDR_GET_LSIZE(hdr));
2142 error = zio_decompress_data(HDR_GET_COMPRESS(hdr),
2143 hdr->b_l1hdr.b_pabd, &dabd,
2144 HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr),
2145 &hdr->b_complevel);
2154 "hdr %px, compress %d, psize %d, lsize %d",
2155 hdr, arc_hdr_get_compress(hdr),
2156 HDR_GET_PSIZE(hdr), HDR_GET_LSIZE(hdr));
2159 arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
2170 ASSERT(!HDR_SHARED_DATA(hdr));
2172 dmu_ot_byteswap[bswap].ob_func(buf->b_data, HDR_GET_LSIZE(hdr));
2175 /* Compute the hdr's checksum if necessary */
2214 * We account for the space used by the hdr and the arc buf individually
2218 arc_evictable_space_increment(arc_buf_hdr_t *hdr, arc_state_t *state)
2220 arc_buf_contents_t type = arc_buf_type(hdr);
2222 ASSERT(HDR_HAS_L1HDR(hdr));
2225 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2226 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2227 ASSERT(!HDR_HAS_RABD(hdr));
2229 HDR_GET_LSIZE(hdr), hdr);
2233 if (hdr->b_l1hdr.b_pabd != NULL) {
2235 arc_hdr_size(hdr), hdr);
2237 if (HDR_HAS_RABD(hdr)) {
2239 HDR_GET_PSIZE(hdr), hdr);
2242 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2253 * We account for the space used by the hdr and the arc buf individually
2257 arc_evictable_space_decrement(arc_buf_hdr_t *hdr, arc_state_t *state)
2259 arc_buf_contents_t type = arc_buf_type(hdr);
2261 ASSERT(HDR_HAS_L1HDR(hdr));
2264 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2265 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2266 ASSERT(!HDR_HAS_RABD(hdr));
2268 HDR_GET_LSIZE(hdr), hdr);
2272 if (hdr->b_l1hdr.b_pabd != NULL) {
2274 arc_hdr_size(hdr), hdr);
2276 if (HDR_HAS_RABD(hdr)) {
2278 HDR_GET_PSIZE(hdr), hdr);
2281 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2291 * Add a reference to this hdr indicating that someone is actively
2297 add_reference(arc_buf_hdr_t *hdr, const void *tag)
2299 arc_state_t *state = hdr->b_l1hdr.b_state;
2301 ASSERT(HDR_HAS_L1HDR(hdr));
2302 if (!HDR_EMPTY(hdr) && !MUTEX_HELD(HDR_LOCK(hdr))) {
2304 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2305 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
2308 if ((zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) &&
2311 multilist_remove(&state->arcs_list[arc_buf_type(hdr)], hdr);
2312 arc_evictable_space_decrement(hdr, state);
2317 * Remove a reference from this hdr. When the reference transitions from
2318 * 1 to 0 and we're not anonymous, then we add this hdr to the arc_state_t's
2322 remove_reference(arc_buf_hdr_t *hdr, const void *tag)
2325 arc_state_t *state = hdr->b_l1hdr.b_state;
2327 ASSERT(HDR_HAS_L1HDR(hdr));
2328 ASSERT(state == arc_anon || MUTEX_HELD(HDR_LOCK(hdr)));
2331 if ((cnt = zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) != 0)
2335 arc_hdr_destroy(hdr);
2338 if (state == arc_uncached && !HDR_PREFETCH(hdr)) {
2339 arc_change_state(arc_anon, hdr);
2340 arc_hdr_destroy(hdr);
2343 multilist_insert(&state->arcs_list[arc_buf_type(hdr)], hdr);
2344 arc_evictable_space_increment(hdr, state);
2359 arc_buf_hdr_t *hdr = ab->b_hdr;
2366 if (hdr == NULL)
2369 abi->abi_flags = hdr->b_flags;
2371 if (HDR_HAS_L1HDR(hdr)) {
2372 l1hdr = &hdr->b_l1hdr;
2375 if (HDR_HAS_L2HDR(hdr))
2376 l2hdr = &hdr->b_l2hdr;
2396 abi->abi_state_contents = arc_buf_type(hdr);
2397 abi->abi_size = arc_hdr_size(hdr);
2405 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr)
2410 arc_buf_contents_t type = arc_buf_type(hdr);
2413 * We almost always have an L1 hdr here, since we call arc_hdr_realloc()
2415 * L1 hdr doesn't always exist when we change state to arc_anon before
2416 * destroying a header, in which case reallocating to add the L1 hdr is
2419 if (HDR_HAS_L1HDR(hdr)) {
2420 old_state = hdr->b_l1hdr.b_state;
2421 refcnt = zfs_refcount_count(&hdr->b_l1hdr.b_refcnt);
2422 update_old = (hdr->b_l1hdr.b_buf != NULL ||
2423 hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr));
2425 IMPLY(GHOST_STATE(old_state), hdr->b_l1hdr.b_buf == NULL);
2426 IMPLY(GHOST_STATE(new_state), hdr->b_l1hdr.b_buf == NULL);
2427 IMPLY(old_state == arc_anon, hdr->b_l1hdr.b_buf == NULL ||
2428 ARC_BUF_LAST(hdr->b_l1hdr.b_buf));
2440 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
2449 ASSERT(HDR_HAS_L1HDR(hdr));
2451 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
2453 hdr);
2454 arc_evictable_space_decrement(hdr, old_state);
2464 ASSERT(HDR_HAS_L1HDR(hdr));
2465 multilist_insert(&new_state->arcs_list[type], hdr);
2466 arc_evictable_space_increment(hdr, new_state);
2470 ASSERT(!HDR_EMPTY(hdr));
2471 if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr))
2472 buf_hash_remove(hdr);
2477 ASSERT(HDR_HAS_L1HDR(hdr));
2488 HDR_GET_LSIZE(hdr), hdr);
2489 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2490 ASSERT(!HDR_HAS_RABD(hdr));
2498 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2503 * block with the hdr, the owner of the
2504 * reference belongs to the hdr. Only
2516 if (hdr->b_l1hdr.b_pabd != NULL) {
2519 arc_hdr_size(hdr), hdr);
2522 if (HDR_HAS_RABD(hdr)) {
2525 HDR_GET_PSIZE(hdr), hdr);
2531 ASSERT(HDR_HAS_L1HDR(hdr));
2533 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2534 ASSERT(!HDR_HAS_RABD(hdr));
2546 HDR_GET_LSIZE(hdr), hdr);
2554 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL;
2559 * block with the hdr, the owner of the
2560 * reference belongs to the hdr. Only
2571 ASSERT(hdr->b_l1hdr.b_pabd != NULL ||
2572 HDR_HAS_RABD(hdr));
2574 if (hdr->b_l1hdr.b_pabd != NULL) {
2577 arc_hdr_size(hdr), hdr);
2580 if (HDR_HAS_RABD(hdr)) {
2583 HDR_GET_PSIZE(hdr), hdr);
2588 if (HDR_HAS_L1HDR(hdr)) {
2589 hdr->b_l1hdr.b_state = new_state;
2591 if (HDR_HAS_L2HDR(hdr) && new_state != arc_l2c_only) {
2592 l2arc_hdr_arcstats_decrement_state(hdr);
2593 hdr->b_l2hdr.b_arcs_state = new_state->arcs_state;
2594 l2arc_hdr_arcstats_increment_state(hdr);
2687 * Given a hdr and a buf, returns whether that buf can share its b_data buffer
2688 * with the hdr's b_pabd.
2691 arc_can_share(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2694 * The criteria for sharing a hdr's data are:
2696 * 2. the hdr's compression matches the buf's compression
2697 * 3. the hdr doesn't need to be byteswapped
2698 * 4. the hdr isn't already being shared
2699 * 5. the buf is either compressed or it is the last buf in the hdr list
2702 * bufs must be the final buf in the hdr's b_buf list. Reading this, you
2705 * a shared uncompressed buf anyway (because the hdr must be compressed
2715 ASSERT3P(buf->b_hdr, ==, hdr);
2717 arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF;
2721 hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS &&
2722 !HDR_SHARED_DATA(hdr) &&
2727 * Allocate a buf for this hdr. If you care about the data that's in the hdr,
2732 arc_buf_alloc_impl(arc_buf_hdr_t *hdr, spa_t *spa, const zbookmark_phys_t *zb,
2739 ASSERT(HDR_HAS_L1HDR(hdr));
2740 ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);
2741 VERIFY(hdr->b_type == ARC_BUFC_DATA ||
2742 hdr->b_type == ARC_BUFC_METADATA);
2748 buf->b_hdr = hdr;
2750 buf->b_next = hdr->b_l1hdr.b_buf;
2753 add_reference(hdr, tag);
2756 * We're about to change the hdr's b_flags. We must either
2759 ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
2762 * Only honor requests for compressed bufs if the hdr is actually
2771 arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF) {
2782 * If the hdr's data can be shared then we share the data buffer and
2783 * set the appropriate bit in the hdr's b_flags to indicate the hdr is
2788 * hdr -> buf instead of the usual buf -> hdr. First, the hdr can't be
2790 * an arc_write() then the hdr's data buffer will be released when the
2792 * Second, the hdr's ABD must be linear so that the buf's user doesn't
2799 boolean_t can_share = arc_can_share(hdr, buf) &&
2800 !HDR_L2_WRITING(hdr) &&
2801 hdr->b_l1hdr.b_pabd != NULL &&
2802 abd_is_linear(hdr->b_l1hdr.b_pabd) &&
2803 !abd_is_linear_page(hdr->b_l1hdr.b_pabd);
2807 buf->b_data = abd_to_buf(hdr->b_l1hdr.b_pabd);
2809 arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);
2812 arc_get_data_buf(hdr, arc_buf_size(buf), buf);
2817 hdr->b_l1hdr.b_buf = buf;
2820 * If the user wants the data from the hdr, we need to either copy or
2892 arc_buf_hdr_t *hdr = buf->b_hdr;
2895 ASSERT(HDR_HAS_L1HDR(hdr));
2896 (void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
2897 (void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
2906 arc_buf_hdr_t *hdr = buf->b_hdr;
2909 ASSERT(HDR_HAS_L1HDR(hdr));
2910 (void) zfs_refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
2911 (void) zfs_refcount_remove(&hdr->b_l1hdr.b_refcnt, tag);
2930 arc_hdr_free_on_write(arc_buf_hdr_t *hdr, boolean_t free_rdata)
2932 arc_state_t *state = hdr->b_l1hdr.b_state;
2933 arc_buf_contents_t type = arc_buf_type(hdr);
2934 uint64_t size = (free_rdata) ? HDR_GET_PSIZE(hdr) : arc_hdr_size(hdr);
2937 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
2938 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
2942 size, hdr);
2944 (void) zfs_refcount_remove_many(&state->arcs_size[type], size, hdr);
2953 l2arc_free_abd_on_write(hdr->b_crypt_hdr.b_rabd, size, type);
2955 l2arc_free_abd_on_write(hdr->b_l1hdr.b_pabd, size, type);
2960 * Share the arc_buf_t's data with the hdr. Whenever we are sharing the
2961 * data buffer, we transfer the refcount ownership to the hdr and update
2965 arc_share_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf)
2967 ASSERT(arc_can_share(hdr, buf));
2968 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
2970 ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
2974 * refcount ownership to the hdr since it always owns
2978 &hdr->b_l1hdr.b_state->arcs_size[arc_buf_type(hdr)],
2979 arc_hdr_size(hdr), buf, hdr);
2980 hdr->b_l1hdr.b_pabd = abd_get_from_buf(buf->b_data, arc_buf_size(buf));
2981 abd_take_ownership_of_buf(hdr->b_l1hdr.b_pabd,
2982 HDR_ISTYPE_METADATA(hdr));
2983 arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA);
2987 * Since we've transferred ownership to the hdr we need
2991 ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr));
2992 ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));
2997 arc_unshare_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf)
3000 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3001 ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
3008 &hdr->b_l1hdr.b_state->arcs_size[arc_buf_type(hdr)],
3009 arc_hdr_size(hdr), hdr, buf);
3010 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
3011 abd_release_ownership_of_buf(hdr->b_l1hdr.b_pabd);
3012 abd_free(hdr->b_l1hdr.b_pabd);
3013 hdr->b_l1hdr.b_pabd = NULL;
3018 * the arc buf and the hdr, count it as overhead.
3020 ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr));
3021 ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
3026 * Remove an arc_buf_t from the hdr's buf list and return the last
3031 arc_buf_remove(arc_buf_hdr_t *hdr, arc_buf_t *buf)
3033 ASSERT(HDR_HAS_L1HDR(hdr));
3034 ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
3036 arc_buf_t **bufp = &hdr->b_l1hdr.b_buf;
3040 * Remove the buf from the hdr list and locate the last
3071 arc_buf_hdr_t *hdr = buf->b_hdr;
3075 * sharing this with the hdr. If we are sharing it with the hdr, the
3076 * hdr is responsible for doing the free.
3080 * We're about to change the hdr's b_flags. We must either
3083 ASSERT(HDR_EMPTY_OR_LOCKED(hdr));
3089 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA);
3093 arc_free_data_buf(hdr, buf->b_data, size, buf);
3103 if (ARC_BUF_ENCRYPTED(buf) && HDR_HAS_RABD(hdr) &&
3104 hdr->b_l1hdr.b_pabd != NULL && !HDR_IO_IN_PROGRESS(hdr)) {
3106 for (b = hdr->b_l1hdr.b_buf; b; b = b->b_next) {
3111 arc_hdr_free_abd(hdr, B_TRUE);
3115 arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);
3120 * hdr, then reassign the hdr's b_pabd to share it with the new
3122 * the last one on the hdr's buffer list.
3133 /* hdr is uncompressed so can't have compressed buf */
3136 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3137 arc_hdr_free_abd(hdr, B_FALSE);
3141 * last buffer and the hdr. The data would have
3143 * ownership to the hdr since it's now being shared.
3145 arc_share_buf(hdr, lastbuf);
3147 } else if (HDR_SHARED_DATA(hdr)) {
3153 * we rely on the hdr's compression flags to determine
3158 arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF);
3163 * this hdr.
3165 if (!arc_hdr_has_uncompressed_buf(hdr)) {
3166 arc_cksum_free(hdr);
3175 arc_hdr_alloc_abd(arc_buf_hdr_t *hdr, int alloc_flags)
3180 ASSERT3U(HDR_GET_LSIZE(hdr), >, 0);
3181 ASSERT(HDR_HAS_L1HDR(hdr));
3182 ASSERT(!HDR_SHARED_DATA(hdr) || alloc_rdata);
3183 IMPLY(alloc_rdata, HDR_PROTECTED(hdr));
3186 size = HDR_GET_PSIZE(hdr);
3187 ASSERT3P(hdr->b_crypt_hdr.b_rabd, ==, NULL);
3188 hdr->b_crypt_hdr.b_rabd = arc_get_data_abd(hdr, size, hdr,
3190 ASSERT3P(hdr->b_crypt_hdr.b_rabd, !=, NULL);
3193 size = arc_hdr_size(hdr);
3194 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3195 hdr->b_l1hdr.b_pabd = arc_get_data_abd(hdr, size, hdr,
3197 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
3201 ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr));
3205 arc_hdr_free_abd(arc_buf_hdr_t *hdr, boolean_t free_rdata)
3207 uint64_t size = (free_rdata) ? HDR_GET_PSIZE(hdr) : arc_hdr_size(hdr);
3209 ASSERT(HDR_HAS_L1HDR(hdr));
3210 ASSERT(hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr));
3211 IMPLY(free_rdata, HDR_HAS_RABD(hdr));
3214 * If the hdr is currently being written to the l2arc then
3219 if (HDR_L2_WRITING(hdr)) {
3220 arc_hdr_free_on_write(hdr, free_rdata);
3223 arc_free_data_abd(hdr, hdr->b_crypt_hdr.b_rabd, size, hdr);
3225 arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd, size, hdr);
3229 hdr->b_crypt_hdr.b_rabd = NULL;
3232 hdr->b_l1hdr.b_pabd = NULL;
3235 if (hdr->b_l1hdr.b_pabd == NULL && !HDR_HAS_RABD(hdr))
3236 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
3239 ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr));
3269 arc_buf_hdr_t *hdr;
3272 hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE);
3274 ASSERT(HDR_EMPTY(hdr));
3276 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3278 HDR_SET_PSIZE(hdr, psize);
3279 HDR_SET_LSIZE(hdr, lsize);
3280 hdr->b_spa = spa;
3281 hdr->b_type = type;
3282 hdr->b_flags = 0;
3283 arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L1HDR);
3284 arc_hdr_set_compress(hdr, compression_type);
3285 hdr->b_complevel = complevel;
3287 arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);
3289 hdr->b_l1hdr.b_state = arc_anon;
3290 hdr->b_l1hdr.b_arc_access = 0;
3291 hdr->b_l1hdr.b_mru_hits = 0;
3292 hdr->b_l1hdr.b_mru_ghost_hits = 0;
3293 hdr->b_l1hdr.b_mfu_hits = 0;
3294 hdr->b_l1hdr.b_mfu_ghost_hits = 0;
3295 hdr->b_l1hdr.b_buf = NULL;
3297 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3299 return (hdr);
3310 arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
3312 ASSERT(HDR_HAS_L2HDR(hdr));
3315 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
3322 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
3323 buf_hash_remove(hdr);
3325 memcpy(nhdr, hdr, HDR_L2ONLY_SIZE);
3338 ASSERT(!HDR_HAS_RABD(hdr));
3340 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
3342 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3352 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3360 VERIFY(!HDR_L2_WRITING(hdr));
3361 VERIFY3P(hdr->b_l1hdr.b_pabd, ==, NULL);
3362 ASSERT(!HDR_HAS_RABD(hdr));
3372 ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node));
3382 list_insert_after(&dev->l2ad_buflist, hdr, nhdr);
3383 list_remove(&dev->l2ad_buflist, hdr);
3396 arc_hdr_size(hdr), hdr);
3400 buf_discard_identity(hdr);
3401 kmem_cache_free(old, hdr);
3418 arc_buf_hdr_t *hdr = buf->b_hdr;
3421 ASSERT(HDR_HAS_L1HDR(hdr));
3422 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
3425 arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);
3426 hdr->b_crypt_hdr.b_dsobj = dsobj;
3427 hdr->b_crypt_hdr.b_ot = ot;
3428 hdr->b_l1hdr.b_byteswap = (byteorder == ZFS_HOST_BYTEORDER) ?
3430 if (!arc_hdr_has_uncompressed_buf(hdr))
3431 arc_cksum_free(hdr);
3434 memcpy(hdr->b_crypt_hdr.b_salt, salt, ZIO_DATA_SALT_LEN);
3436 memcpy(hdr->b_crypt_hdr.b_iv, iv, ZIO_DATA_IV_LEN);
3438 memcpy(hdr->b_crypt_hdr.b_mac, mac, ZIO_DATA_MAC_LEN);
3449 arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), size, size,
3453 VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_FALSE, B_FALSE,
3473 arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize,
3477 VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_FALSE,
3482 * To ensure that the hdr has the correct data in it if we call
3484 * it's easiest if we just set up sharing between the buf and the hdr.
3486 arc_share_buf(hdr, buf);
3497 arc_buf_hdr_t *hdr;
3507 hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize, B_TRUE,
3510 hdr->b_crypt_hdr.b_dsobj = dsobj;
3511 hdr->b_crypt_hdr.b_ot = ot;
3512 hdr->b_l1hdr.b_byteswap = (byteorder == ZFS_HOST_BYTEORDER) ?
3514 memcpy(hdr->b_crypt_hdr.b_salt, salt, ZIO_DATA_SALT_LEN);
3515 memcpy(hdr->b_crypt_hdr.b_iv, iv, ZIO_DATA_IV_LEN);
3516 memcpy(hdr->b_crypt_hdr.b_mac, mac, ZIO_DATA_MAC_LEN);
3524 VERIFY0(arc_buf_alloc_impl(hdr, spa, NULL, tag, B_TRUE, B_TRUE,
3532 l2arc_hdr_arcstats_update(arc_buf_hdr_t *hdr, boolean_t incr,
3535 uint64_t lsize = HDR_GET_LSIZE(hdr);
3536 uint64_t psize = HDR_GET_PSIZE(hdr);
3537 uint64_t asize = HDR_GET_L2SIZE(hdr);
3538 arc_buf_contents_t type = hdr->b_type;
3557 if (HDR_PREFETCH(hdr)) {
3569 switch (hdr->b_l2hdr.b_arcs_state) {
3603 arc_hdr_l2hdr_destroy(arc_buf_hdr_t *hdr)
3605 l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
3609 ASSERT(HDR_HAS_L2HDR(hdr));
3611 list_remove(&dev->l2ad_buflist, hdr);
3613 l2arc_hdr_arcstats_decrement(hdr);
3615 uint64_t asize = HDR_GET_L2SIZE(hdr);
3619 (void) zfs_refcount_remove_many(&dev->l2ad_alloc, arc_hdr_size(hdr),
3620 hdr);
3621 arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);
3625 arc_hdr_destroy(arc_buf_hdr_t *hdr)
3627 if (HDR_HAS_L1HDR(hdr)) {
3628 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
3629 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
3631 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3632 ASSERT(!HDR_IN_HASH_TABLE(hdr));
3634 if (HDR_HAS_L2HDR(hdr)) {
3635 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
3650 if (HDR_HAS_L2HDR(hdr)) {
3652 if (!HDR_EMPTY(hdr))
3653 buf_discard_identity(hdr);
3655 arc_hdr_l2hdr_destroy(hdr);
3668 if (!HDR_EMPTY(hdr))
3669 buf_discard_identity(hdr);
3671 if (HDR_HAS_L1HDR(hdr)) {
3672 arc_cksum_free(hdr);
3674 while (hdr->b_l1hdr.b_buf != NULL)
3675 arc_buf_destroy_impl(hdr->b_l1hdr.b_buf);
3677 if (hdr->b_l1hdr.b_pabd != NULL)
3678 arc_hdr_free_abd(hdr, B_FALSE);
3680 if (HDR_HAS_RABD(hdr))
3681 arc_hdr_free_abd(hdr, B_TRUE);
3684 ASSERT3P(hdr->b_hash_next, ==, NULL);
3685 if (HDR_HAS_L1HDR(hdr)) {
3686 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
3687 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
3689 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
3691 kmem_cache_free(hdr_full_cache, hdr);
3693 kmem_cache_free(hdr_l2only_cache, hdr);
3700 arc_buf_hdr_t *hdr = buf->b_hdr;
3702 if (hdr->b_l1hdr.b_state == arc_anon) {
3703 ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);
3705 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3706 VERIFY0(remove_reference(hdr, tag));
3710 kmutex_t *hash_lock = HDR_LOCK(hdr);
3713 ASSERT3P(hdr, ==, buf->b_hdr);
3714 ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
3715 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
3716 ASSERT3P(hdr->b_l1hdr.b_state, !=, arc_anon);
3720 (void) remove_reference(hdr, tag);
3747 arc_evict_hdr(arc_buf_hdr_t *hdr, uint64_t *real_evicted)
3751 uint_t min_lifetime = HDR_PRESCIENT_PREFETCH(hdr) ?
3754 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
3755 ASSERT(HDR_HAS_L1HDR(hdr));
3756 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
3757 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
3758 ASSERT0(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt));
3761 state = hdr->b_l1hdr.b_state;
3771 if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) {
3777 bytes_evicted += HDR_GET_LSIZE(hdr);
3779 DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr);
3781 if (HDR_HAS_L2HDR(hdr)) {
3782 ASSERT(hdr->b_l1hdr.b_pabd == NULL);
3783 ASSERT(!HDR_HAS_RABD(hdr));
3788 arc_change_state(arc_l2c_only, hdr);
3793 (void) arc_hdr_realloc(hdr, hdr_full_cache,
3797 arc_change_state(arc_anon, hdr);
3798 arc_hdr_destroy(hdr);
3809 if ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) &&
3810 ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access <
3816 if (HDR_HAS_L2HDR(hdr)) {
3817 ARCSTAT_INCR(arcstat_evict_l2_cached, HDR_GET_LSIZE(hdr));
3819 if (l2arc_write_eligible(hdr->b_spa, hdr)) {
3821 HDR_GET_LSIZE(hdr));
3827 HDR_GET_LSIZE(hdr));
3832 HDR_GET_LSIZE(hdr));
3839 HDR_GET_LSIZE(hdr));
3843 bytes_evicted += arc_hdr_size(hdr);
3844 *real_evicted += arc_hdr_size(hdr);
3847 * If this hdr is being evicted and has a compressed buffer then we
3851 if (hdr->b_l1hdr.b_pabd != NULL)
3852 arc_hdr_free_abd(hdr, B_FALSE);
3854 if (HDR_HAS_RABD(hdr))
3855 arc_hdr_free_abd(hdr, B_TRUE);
3857 arc_change_state(evicted_state, hdr);
3858 DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr);
3860 arc_hdr_destroy(hdr);
3863 ASSERT(HDR_IN_HASH_TABLE(hdr));
3889 arc_buf_hdr_t *hdr;
3897 for (hdr = multilist_sublist_prev(mls, marker); likely(hdr != NULL);
3898 hdr = multilist_sublist_prev(mls, marker)) {
3904 * forward. Since we're not holding hdr's hash lock, we
3905 * must be very careful and not remove 'hdr' from the
3907 * 'hdr' as not being on a sublist when they call the
3924 if (hdr->b_spa == 0)
3928 if (spa != 0 && hdr->b_spa != spa) {
3933 hash_lock = HDR_LOCK(hdr);
3948 uint64_t evicted = arc_evict_hdr(hdr, &revicted);
4925 arc_get_data_abd(arc_buf_hdr_t *hdr, uint64_t size, const void *tag,
4928 arc_buf_contents_t type = arc_buf_type(hdr);
4930 arc_get_data_impl(hdr, size, tag, alloc_flags);
4938 arc_get_data_buf(arc_buf_hdr_t *hdr, uint64_t size, const void *tag)
4940 arc_buf_contents_t type = arc_buf_type(hdr);
4942 arc_get_data_impl(hdr, size, tag, 0);
5039 arc_get_data_impl(arc_buf_hdr_t *hdr, uint64_t size, const void *tag,
5060 arc_buf_contents_t type = arc_buf_type(hdr);
5071 arc_state_t *state = hdr->b_l1hdr.b_state;
5086 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
5087 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
5095 arc_free_data_abd(arc_buf_hdr_t *hdr, abd_t *abd, uint64_t size,
5098 arc_free_data_impl(hdr, size, tag);
5103 arc_free_data_buf(arc_buf_hdr_t *hdr, void *buf, uint64_t size, const void *tag)
5105 arc_buf_contents_t type = arc_buf_type(hdr);
5107 arc_free_data_impl(hdr, size, tag);
5120 arc_free_data_impl(arc_buf_hdr_t *hdr, uint64_t size, const void *tag)
5122 arc_state_t *state = hdr->b_l1hdr.b_state;
5123 arc_buf_contents_t type = arc_buf_type(hdr);
5126 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
5127 ASSERT(zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
5135 VERIFY3U(hdr->b_type, ==, type);
5148 arc_access(arc_buf_hdr_t *hdr, arc_flags_t arc_flags, boolean_t hit)
5150 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
5151 ASSERT(HDR_HAS_L1HDR(hdr));
5156 boolean_t was_prefetch = HDR_PREFETCH(hdr);
5161 HDR_PRESCIENT_PREFETCH(hdr), prescient, predictive,
5164 if (HDR_HAS_L2HDR(hdr))
5165 l2arc_hdr_arcstats_decrement_state(hdr);
5167 arc_hdr_clear_flags(hdr,
5170 arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH);
5172 if (HDR_HAS_L2HDR(hdr))
5173 l2arc_hdr_arcstats_increment_state(hdr);
5177 arc_hdr_set_flags(hdr, ARC_FLAG_PRESCIENT_PREFETCH);
5184 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
5187 if (hdr->b_l1hdr.b_state == arc_anon) {
5193 ASSERT0(hdr->b_l1hdr.b_arc_access);
5194 hdr->b_l1hdr.b_arc_access = now;
5195 if (HDR_UNCACHED(hdr)) {
5198 hdr);
5201 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
5203 arc_change_state(new_state, hdr);
5204 } else if (hdr->b_l1hdr.b_state == arc_mru) {
5209 if (HDR_IO_IN_PROGRESS(hdr)) {
5210 hdr->b_l1hdr.b_arc_access = now;
5213 hdr->b_l1hdr.b_mru_hits++;
5221 hdr->b_l1hdr.b_arc_access = now;
5229 if (ddi_time_after(now, hdr->b_l1hdr.b_arc_access +
5231 hdr->b_l1hdr.b_arc_access = now;
5232 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
5233 arc_change_state(arc_mfu, hdr);
5235 } else if (hdr->b_l1hdr.b_state == arc_mru_ghost) {
5243 hdr->b_l1hdr.b_mru_ghost_hits++;
5245 hdr->b_l1hdr.b_arc_access = now;
5246 wmsum_add(&arc_mru_ghost->arcs_hits[arc_buf_type(hdr)],
5247 arc_hdr_size(hdr));
5250 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
5253 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
5255 arc_change_state(new_state, hdr);
5256 } else if (hdr->b_l1hdr.b_state == arc_mfu) {
5261 if (!HDR_IO_IN_PROGRESS(hdr)) {
5262 hdr->b_l1hdr.b_mfu_hits++;
5265 hdr->b_l1hdr.b_arc_access = now;
5266 } else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) {
5272 hdr->b_l1hdr.b_mfu_ghost_hits++;
5274 hdr->b_l1hdr.b_arc_access = now;
5275 wmsum_add(&arc_mfu_ghost->arcs_hits[arc_buf_type(hdr)],
5276 arc_hdr_size(hdr));
5277 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
5278 arc_change_state(arc_mfu, hdr);
5279 } else if (hdr->b_l1hdr.b_state == arc_uncached) {
5284 if (!HDR_IO_IN_PROGRESS(hdr))
5286 hdr->b_l1hdr.b_arc_access = now;
5287 } else if (hdr->b_l1hdr.b_state == arc_l2c_only) {
5292 hdr->b_l1hdr.b_arc_access = now;
5293 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
5294 arc_change_state(arc_mru, hdr);
5297 hdr->b_l1hdr.b_state);
5308 arc_buf_hdr_t *hdr = buf->b_hdr;
5315 if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr))
5318 kmutex_t *hash_lock = HDR_LOCK(hdr);
5321 if (hdr->b_l1hdr.b_state == arc_anon || HDR_EMPTY(hdr)) {
5327 ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
5328 hdr->b_l1hdr.b_state == arc_mfu ||
5329 hdr->b_l1hdr.b_state == arc_uncached);
5331 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
5332 arc_access(hdr, 0, B_TRUE);
5337 !HDR_ISTYPE_METADATA(hdr), data, metadata, hits);
5373 arc_hdr_verify(arc_buf_hdr_t *hdr, blkptr_t *bp)
5376 ASSERT3U(HDR_GET_PSIZE(hdr), ==, 0);
5377 ASSERT3U(arc_hdr_get_compress(hdr), ==, ZIO_COMPRESS_OFF);
5379 if (HDR_COMPRESSION_ENABLED(hdr)) {
5380 ASSERT3U(arc_hdr_get_compress(hdr), ==,
5383 ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp));
5384 ASSERT3U(HDR_GET_PSIZE(hdr), ==, BP_GET_PSIZE(bp));
5385 ASSERT3U(!!HDR_PROTECTED(hdr), ==, BP_IS_PROTECTED(bp));
5393 arc_buf_hdr_t *hdr = zio->io_private;
5399 * The hdr was inserted into hash-table and removed from lists
5406 if (HDR_IN_HASH_TABLE(hdr)) {
5409 ASSERT3U(hdr->b_birth, ==, BP_GET_BIRTH(zio->io_bp));
5410 ASSERT3U(hdr->b_dva.dva_word[0], ==,
5412 ASSERT3U(hdr->b_dva.dva_word[1], ==,
5415 found = buf_hash_find(hdr->b_spa, zio->io_bp, &hash_lock);
5417 ASSERT((found == hdr &&
5418 DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) ||
5419 (found == hdr && HDR_L2_READING(hdr)));
5424 hdr->b_crypt_hdr.b_ot = BP_GET_TYPE(bp);
5425 hdr->b_crypt_hdr.b_dsobj = zio->io_bookmark.zb_objset;
5426 zio_crypt_decode_params_bp(bp, hdr->b_crypt_hdr.b_salt,
5427 hdr->b_crypt_hdr.b_iv);
5436 hdr->b_crypt_hdr.b_mac);
5441 hdr->b_crypt_hdr.b_mac);
5450 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64;
5452 hdr->b_l1hdr.b_byteswap =
5456 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
5458 if (!HDR_L2_READING(hdr)) {
5459 hdr->b_complevel = zio->io_prop.zp_complevel;
5463 arc_hdr_clear_flags(hdr, ARC_FLAG_L2_EVICTED);
5464 if (l2arc_noprefetch && HDR_PREFETCH(hdr))
5465 arc_hdr_clear_flags(hdr, ARC_FLAG_L2CACHE);
5467 callback_list = hdr->b_l1hdr.b_acb;
5469 hdr->b_l1hdr.b_acb = NULL;
5491 int error = arc_buf_alloc_impl(hdr, zio->io_spa,
5529 * the hdr will not be anonymous, because
5540 * because the only way for multiple threads to find this hdr is
5542 * callbacks, the hdr is not anonymous. If it were anonymous,
5548 arc_hdr_verify(hdr, zio->io_bp);
5550 arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR);
5551 if (hdr->b_l1hdr.b_state != arc_anon)
5552 arc_change_state(arc_anon, hdr);
5553 if (HDR_IN_HASH_TABLE(hdr))
5554 buf_hash_remove(hdr);
5557 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5558 (void) remove_reference(hdr, hdr);
5606 arc_buf_hdr_t *hdr = NULL;
5614 hdr = buf_hash_find(guid, bp, &hash_lock);
5615 if (hdr == NULL)
5618 if (HDR_HAS_L1HDR(hdr)) {
5619 arc_state_t *state = hdr->b_l1hdr.b_state;
5642 if (HDR_HAS_L2HDR(hdr))
5673 arc_buf_hdr_t *hdr = NULL;
5709 hdr = buf_hash_find(guid, bp, &hash_lock);
5719 if (hdr != NULL && HDR_HAS_L1HDR(hdr) && (HDR_HAS_RABD(hdr) ||
5720 (hdr->b_l1hdr.b_pabd != NULL && !encrypted_read))) {
5721 boolean_t is_data = !HDR_ISTYPE_METADATA(hdr);
5735 if (HDR_IO_IN_PROGRESS(hdr)) {
5743 zio_t *head_zio = hdr->b_l1hdr.b_acb->acb_zio_head;
5745 if ((hdr->b_flags & ARC_FLAG_PRIO_ASYNC_READ) &&
5754 arc_buf_hdr_t *, hdr);
5758 DTRACE_PROBE1(arc__iohit, arc_buf_hdr_t *, hdr);
5759 arc_access(hdr, *arc_flags, B_FALSE);
5800 acb->acb_next = hdr->b_l1hdr.b_acb;
5801 hdr->b_l1hdr.b_acb->acb_prev = acb;
5802 hdr->b_l1hdr.b_acb = acb;
5825 ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
5826 hdr->b_l1hdr.b_state == arc_mfu ||
5827 hdr->b_l1hdr.b_state == arc_uncached);
5829 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
5830 arc_access(hdr, *arc_flags, B_TRUE);
5836 rc = arc_buf_alloc_impl(hdr, spa, zb, private,
5847 spa_log_error(spa, zb, hdr->b_birth);
5856 (void) remove_reference(hdr, private);
5902 if (hdr == NULL) {
5908 hdr = arc_hdr_alloc(guid, psize, lsize,
5912 hdr->b_dva = *BP_IDENTITY(bp);
5913 hdr->b_birth = BP_GET_BIRTH(bp);
5914 exists = buf_hash_insert(hdr, &hash_lock);
5919 buf_discard_identity(hdr);
5920 arc_hdr_destroy(hdr);
5927 * L2-only (and thus didn't have an L1 hdr),
5928 * we realloc the header to add an L1 hdr.
5930 if (!HDR_HAS_L1HDR(hdr)) {
5931 hdr = arc_hdr_realloc(hdr, hdr_l2only_cache,
5935 if (GHOST_STATE(hdr->b_l1hdr.b_state)) {
5936 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
5937 ASSERT(!HDR_HAS_RABD(hdr));
5938 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
5940 &hdr->b_l1hdr.b_refcnt));
5941 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
5943 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL);
5945 } else if (HDR_IO_IN_PROGRESS(hdr)) {
5963 hdr->b_l1hdr.b_acb->acb_zio_head;
5964 acb->acb_next = hdr->b_l1hdr.b_acb;
5965 hdr->b_l1hdr.b_acb->acb_prev = acb;
5966 hdr->b_l1hdr.b_acb = acb;
5981 arc_hdr_set_flags(hdr, ARC_FLAG_UNCACHED);
5992 add_reference(hdr, hdr);
5994 arc_access(hdr, *arc_flags, B_FALSE);
5995 arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
5996 arc_hdr_alloc_abd(hdr, alloc_flags);
5998 ASSERT(HDR_HAS_RABD(hdr));
5999 size = HDR_GET_PSIZE(hdr);
6000 hdr_abd = hdr->b_crypt_hdr.b_rabd;
6003 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
6004 size = arc_hdr_size(hdr);
6005 hdr_abd = hdr->b_l1hdr.b_pabd;
6007 if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF) {
6023 arc_hdr_set_flags(hdr, ARC_FLAG_NOAUTH);
6025 arc_hdr_set_flags(hdr, ARC_FLAG_INDIRECT);
6026 ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state));
6036 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
6037 hdr->b_l1hdr.b_acb = acb;
6039 if (HDR_HAS_L2HDR(hdr) &&
6040 (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) {
6041 devw = hdr->b_l2hdr.b_dev->l2ad_writing;
6042 addr = hdr->b_l2hdr.b_daddr;
6058 arc_hdr_set_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);
6060 arc_hdr_clear_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ);
6066 ASSERT3U(HDR_GET_LSIZE(hdr), ==, lsize);
6074 DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr,
6079 demand, prefetch, !HDR_ISTYPE_METADATA(hdr), data,
6097 if (HDR_HAS_L2HDR(hdr) &&
6098 !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr)) {
6103 DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr);
6105 hdr->b_l2hdr.b_hits++;
6109 cb->l2rcb_hdr = hdr;
6119 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
6120 !HDR_COMPRESSION_ENABLED(hdr) &&
6121 HDR_GET_PSIZE(hdr) != 0) {
6122 size = HDR_GET_PSIZE(hdr);
6128 HDR_ISTYPE_METADATA(hdr));
6144 ASSERT3U(arc_hdr_get_compress(hdr), !=,
6161 HDR_GET_PSIZE(hdr));
6177 arc_buf_hdr_t *, hdr);
6179 if (HDR_L2_WRITING(hdr))
6201 arc_buf_hdr_t *, hdr);
6208 arc_read_done, hdr, priority, zio_flags, zb);
6334 arc_buf_hdr_t *hdr;
6340 hdr = buf_hash_find(guid, bp, &hash_lock);
6341 if (hdr == NULL)
6350 * without the dedup flag set. This would have left the hdr in the MRU
6362 * this hdr, then we don't destroy the hdr.
6364 if (!HDR_HAS_L1HDR(hdr) ||
6365 zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
6366 arc_change_state(arc_anon, hdr);
6367 arc_hdr_destroy(hdr);
6379 * a new hdr for the buffer.
6384 arc_buf_hdr_t *hdr = buf->b_hdr;
6392 ASSERT(HDR_HAS_L1HDR(hdr));
6399 if (hdr->b_l1hdr.b_state == arc_anon) {
6400 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
6401 ASSERT(!HDR_IN_HASH_TABLE(hdr));
6402 ASSERT(!HDR_HAS_L2HDR(hdr));
6404 ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);
6406 ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1);
6407 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
6409 hdr->b_l1hdr.b_arc_access = 0;
6413 * have a hdr that is not empty.
6415 buf_discard_identity(hdr);
6421 kmutex_t *hash_lock = HDR_LOCK(hdr);
6429 arc_state_t *state = hdr->b_l1hdr.b_state;
6430 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
6434 ASSERT3S(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt), >, 0);
6436 if (HDR_HAS_L2HDR(hdr)) {
6437 mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx);
6447 if (HDR_HAS_L2HDR(hdr))
6448 arc_hdr_l2hdr_destroy(hdr);
6450 mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx);
6456 if (hdr->b_l1hdr.b_buf != buf || !ARC_BUF_LAST(buf)) {
6458 uint64_t spa = hdr->b_spa;
6459 uint64_t psize = HDR_GET_PSIZE(hdr);
6460 uint64_t lsize = HDR_GET_LSIZE(hdr);
6461 boolean_t protected = HDR_PROTECTED(hdr);
6462 enum zio_compress compress = arc_hdr_get_compress(hdr);
6463 arc_buf_contents_t type = arc_buf_type(hdr);
6464 VERIFY3U(hdr->b_type, ==, type);
6466 ASSERT(hdr->b_l1hdr.b_buf != buf || buf->b_next != NULL);
6467 VERIFY3S(remove_reference(hdr, tag), >, 0);
6470 ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf);
6475 * Pull the data off of this hdr and attach it to
6476 * a new anonymous hdr. Also find the last buffer
6477 * in the hdr's buffer list.
6479 arc_buf_t *lastbuf = arc_buf_remove(hdr, buf);
6483 * If the current arc_buf_t and the hdr are sharing their data
6487 ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf);
6494 arc_unshare_buf(hdr, buf);
6497 * Now we need to recreate the hdr's b_pabd. Since we
6502 if (arc_can_share(hdr, lastbuf)) {
6503 arc_share_buf(hdr, lastbuf);
6505 arc_hdr_alloc_abd(hdr, 0);
6506 abd_copy_from_buf(hdr->b_l1hdr.b_pabd,
6510 } else if (HDR_SHARED_DATA(hdr)) {
6516 * we rely on the hdr's compression flags to determine
6520 arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF);
6524 ASSERT(hdr->b_l1hdr.b_pabd != NULL || HDR_HAS_RABD(hdr));
6530 if (zfs_refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
6541 if (!arc_hdr_has_uncompressed_buf(hdr))
6542 arc_cksum_free(hdr);
6547 compress, hdr->b_complevel, type);
6560 ASSERT(zfs_refcount_count(&hdr->b_l1hdr.b_refcnt) == 1);
6561 /* protected by hash lock, or hdr is on arc_anon */
6562 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
6563 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
6564 hdr->b_l1hdr.b_mru_hits = 0;
6565 hdr->b_l1hdr.b_mru_ghost_hits = 0;
6566 hdr->b_l1hdr.b_mfu_hits = 0;
6567 hdr->b_l1hdr.b_mfu_ghost_hits = 0;
6568 arc_change_state(arc_anon, hdr);
6569 hdr->b_l1hdr.b_arc_access = 0;
6572 buf_discard_identity(hdr);
6597 arc_buf_hdr_t *hdr = buf->b_hdr;
6602 ASSERT(HDR_HAS_L1HDR(hdr));
6604 ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
6612 arc_cksum_free(hdr);
6614 if (hdr->b_l1hdr.b_pabd != NULL) {
6616 arc_unshare_buf(hdr, buf);
6619 arc_hdr_free_abd(hdr, B_FALSE);
6623 if (HDR_HAS_RABD(hdr))
6624 arc_hdr_free_abd(hdr, B_TRUE);
6626 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
6627 ASSERT(!HDR_HAS_RABD(hdr));
6628 ASSERT(!HDR_SHARED_DATA(hdr));
6633 if (HDR_IO_IN_PROGRESS(hdr)) {
6636 arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
6637 add_reference(hdr, hdr); /* For IO_IN_PROGRESS. */
6646 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64;
6648 hdr->b_l1hdr.b_byteswap =
6652 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS;
6655 arc_hdr_set_flags(hdr, ARC_FLAG_PROTECTED);
6656 hdr->b_crypt_hdr.b_ot = BP_GET_TYPE(bp);
6657 hdr->b_crypt_hdr.b_dsobj = zio->io_bookmark.zb_objset;
6658 zio_crypt_decode_params_bp(bp, hdr->b_crypt_hdr.b_salt,
6659 hdr->b_crypt_hdr.b_iv);
6660 zio_crypt_decode_mac_bp(bp, hdr->b_crypt_hdr.b_mac);
6662 arc_hdr_clear_flags(hdr, ARC_FLAG_PROTECTED);
6670 arc_hdr_set_flags(hdr, ARC_FLAG_NOAUTH);
6686 ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp));
6689 HDR_SET_PSIZE(hdr, psize);
6690 arc_hdr_set_compress(hdr, compress);
6691 hdr->b_complevel = zio->io_prop.zp_complevel;
6697 * Fill the hdr with data. If the buffer is encrypted we have no choice
6698 * but to copy the data into b_radb. If the hdr is compressed, the data
6702 * We might be able to share the buf's data with the hdr here. However,
6713 arc_hdr_alloc_abd(hdr, ARC_HDR_ALLOC_RDATA |
6715 abd_copy(hdr->b_crypt_hdr.b_rabd, zio->io_abd, psize);
6716 } else if (!(HDR_UNCACHED(hdr) ||
6718 !arc_can_share(hdr, buf)) {
6722 * hdr's compression setting rather than the io_bp's.
6726 arc_hdr_alloc_abd(hdr, ARC_HDR_ALLOC_RDATA |
6728 abd_copy(hdr->b_crypt_hdr.b_rabd, zio->io_abd, psize);
6729 } else if (arc_hdr_get_compress(hdr) != ZIO_COMPRESS_OFF &&
6732 arc_hdr_alloc_abd(hdr, ARC_HDR_USE_RESERVE);
6733 abd_copy(hdr->b_l1hdr.b_pabd, zio->io_abd, psize);
6735 ASSERT3U(zio->io_orig_size, ==, arc_hdr_size(hdr));
6736 arc_hdr_alloc_abd(hdr, ARC_HDR_USE_RESERVE);
6737 abd_copy_from_buf(hdr->b_l1hdr.b_pabd, buf->b_data,
6743 ASSERT3P(hdr->b_l1hdr.b_buf, ==, buf);
6746 arc_share_buf(hdr, buf);
6750 arc_hdr_verify(hdr, bp);
6768 arc_buf_hdr_t *hdr = buf->b_hdr;
6770 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
6773 arc_hdr_verify(hdr, zio->io_bp);
6776 buf_discard_identity(hdr);
6778 hdr->b_dva = *BP_IDENTITY(zio->io_bp);
6779 hdr->b_birth = BP_GET_BIRTH(zio->io_bp);
6782 ASSERT(HDR_EMPTY(hdr));
6791 if (!HDR_EMPTY(hdr)) {
6799 exists = buf_hash_insert(hdr, &hash_lock);
6808 panic("bad overwrite, hdr=%p exists=%p",
6809 (void *)hdr, (void *)exists);
6815 exists = buf_hash_insert(hdr, &hash_lock);
6821 panic("bad nopwrite, hdr=%p exists=%p",
6822 (void *)hdr, (void *)exists);
6825 ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
6826 ASSERT(ARC_BUF_LAST(hdr->b_l1hdr.b_buf));
6827 ASSERT(hdr->b_l1hdr.b_state == arc_anon);
6832 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
6833 VERIFY3S(remove_reference(hdr, hdr), >, 0);
6835 if (exists == NULL && hdr->b_l1hdr.b_state == arc_anon)
6836 arc_access(hdr, 0, B_FALSE);
6839 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS);
6840 VERIFY3S(remove_reference(hdr, hdr), >, 0);
6857 arc_buf_hdr_t *hdr = buf->b_hdr;
6864 ASSERT(!HDR_IO_ERROR(hdr));
6865 ASSERT(!HDR_IO_IN_PROGRESS(hdr));
6866 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
6867 ASSERT3P(hdr->b_l1hdr.b_buf, !=, NULL);
6869 arc_hdr_set_flags(hdr, ARC_FLAG_UNCACHED);
6871 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE);
6876 localprop.zp_compress = HDR_GET_COMPRESS(hdr);
6877 localprop.zp_complevel = hdr->b_complevel;
6879 (hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS) ?
6881 memcpy(localprop.zp_salt, hdr->b_crypt_hdr.b_salt,
6883 memcpy(localprop.zp_iv, hdr->b_crypt_hdr.b_iv,
6885 memcpy(localprop.zp_mac, hdr->b_crypt_hdr.b_mac,
6894 ASSERT3U(HDR_GET_LSIZE(hdr), !=, arc_buf_size(buf));
6895 localprop.zp_compress = HDR_GET_COMPRESS(hdr);
6896 localprop.zp_complevel = hdr->b_complevel;
6907 * The hdr's b_pabd is now stale, free it now. A new data block
6910 if (hdr->b_l1hdr.b_pabd != NULL) {
6913 * the hdr then we need to break that relationship here.
6914 * The hdr will remain with a NULL data pointer and the
6918 arc_unshare_buf(hdr, buf);
6921 arc_hdr_free_abd(hdr, B_FALSE);
6926 if (HDR_HAS_RABD(hdr))
6927 arc_hdr_free_abd(hdr, B_TRUE);
6930 arc_hdr_set_compress(hdr, ZIO_COMPRESS_OFF);
6933 ASSERT3P(hdr->b_l1hdr.b_pabd, ==, NULL);
6936 abd_get_from_buf(buf->b_data, HDR_GET_LSIZE(hdr)),
6937 HDR_GET_LSIZE(hdr), arc_buf_size(buf), &localprop, arc_write_ready,
7323 arc_buf_hdr_t *hdr = obj;
7330 ASSERT(!HDR_EMPTY(hdr));
7345 return ((unsigned int)buf_hash(hdr->b_spa, &hdr->b_dva, hdr->b_birth) %
8184 * || hdr| ^ /^ /^ / / |
8248 l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *hdr)
8257 if (hdr->b_spa != spa_guid || HDR_HAS_L2HDR(hdr) ||
8258 HDR_IO_IN_PROGRESS(hdr) || !HDR_L2CACHE(hdr))
8433 arc_buf_hdr_t *head, *hdr, *hdr_prev;
8454 for (hdr = list_prev(buflist, head); hdr; hdr = hdr_prev) {
8455 hdr_prev = list_prev(buflist, hdr);
8457 hash_lock = HDR_LOCK(hdr);
8478 list_insert_after(buflist, hdr, head);
8498 ASSERT(HDR_HAS_L1HDR(hdr));
8508 list_remove(buflist, hdr);
8509 arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR);
8511 uint64_t psize = HDR_GET_PSIZE(hdr);
8512 l2arc_hdr_arcstats_decrement(hdr);
8519 arc_hdr_size(hdr), hdr);
8526 arc_hdr_clear_flags(hdr, ARC_FLAG_L2_WRITING);
8613 arc_buf_hdr_t *hdr = cb->l2rcb_hdr;
8625 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
8626 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
8630 * we must check the bp here and not the hdr, since the
8631 * hdr does not have its encryption parameters updated
8635 abd_t *eabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr,
8643 salt, iv, mac, HDR_GET_PSIZE(hdr), eabd,
8644 hdr->b_l1hdr.b_pabd, &no_crypt);
8646 arc_free_data_abd(hdr, eabd, arc_hdr_size(hdr), hdr);
8656 arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
8657 arc_hdr_size(hdr), hdr);
8658 hdr->b_l1hdr.b_pabd = eabd;
8661 arc_free_data_abd(hdr, eabd, arc_hdr_size(hdr), hdr);
8670 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
8671 !HDR_COMPRESSION_ENABLED(hdr)) {
8672 abd_t *cabd = arc_get_data_abd(hdr, arc_hdr_size(hdr), hdr,
8675 ret = zio_decompress_data(HDR_GET_COMPRESS(hdr),
8676 hdr->b_l1hdr.b_pabd, cabd, HDR_GET_PSIZE(hdr),
8677 HDR_GET_LSIZE(hdr), &hdr->b_complevel);
8679 arc_free_data_abd(hdr, cabd, arc_hdr_size(hdr), hdr);
8683 arc_free_data_abd(hdr, hdr->b_l1hdr.b_pabd,
8684 arc_hdr_size(hdr), hdr);
8685 hdr->b_l1hdr.b_pabd = cabd;
8687 zio->io_size = HDR_GET_LSIZE(hdr);
8706 arc_buf_hdr_t *hdr;
8718 hdr = cb->l2rcb_hdr;
8719 ASSERT3P(hdr, !=, NULL);
8721 hash_lock = HDR_LOCK(hdr);
8723 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
8730 ASSERT3U(arc_hdr_size(hdr), <, zio->io_size);
8733 abd_copy(hdr->b_crypt_hdr.b_rabd,
8734 cb->l2rcb_abd, arc_hdr_size(hdr));
8736 abd_copy(hdr->b_l1hdr.b_pabd,
8737 cb->l2rcb_abd, arc_hdr_size(hdr));
8753 zio->io_size = zio->io_orig_size = arc_hdr_size(hdr);
8756 ASSERT(HDR_HAS_RABD(hdr));
8758 hdr->b_crypt_hdr.b_rabd;
8760 ASSERT3P(hdr->b_l1hdr.b_pabd, !=, NULL);
8761 zio->io_abd = zio->io_orig_abd = hdr->b_l1hdr.b_pabd;
8770 ASSERT(zio->io_abd == hdr->b_l1hdr.b_pabd ||
8771 (HDR_HAS_RABD(hdr) && zio->io_abd == hdr->b_crypt_hdr.b_rabd));
8774 zio->io_prop.zp_complevel = hdr->b_complevel;
8776 valid_cksum = arc_cksum_is_equal(hdr, zio);
8787 !HDR_L2_EVICTED(hdr)) {
8789 zio->io_private = hdr;
8812 hdr->b_crypt_hdr.b_rabd : hdr->b_l1hdr.b_pabd;
8818 hdr, zio->io_priority, cb->l2rcb_flags,
8826 for (struct arc_callback *acb = hdr->b_l1hdr.b_acb;
8920 arc_buf_hdr_t *hdr, *hdr_prev;
9037 for (hdr = list_tail(buflist); hdr; hdr = hdr_prev) {
9038 hdr_prev = list_prev(buflist, hdr);
9040 ASSERT(!HDR_EMPTY(hdr));
9041 hash_lock = HDR_LOCK(hdr);
9062 ASSERT(HDR_HAS_L2HDR(hdr));
9065 ASSERT(!HDR_L2_WRITING(hdr));
9066 ASSERT(!HDR_L2_WRITE_HEAD(hdr));
9068 if (!all && (hdr->b_l2hdr.b_daddr >= dev->l2ad_evict ||
9069 hdr->b_l2hdr.b_daddr < dev->l2ad_hand)) {
9078 if (!HDR_HAS_L1HDR(hdr)) {
9079 ASSERT(!HDR_L2_READING(hdr));
9085 arc_change_state(arc_anon, hdr);
9086 arc_hdr_destroy(hdr);
9088 ASSERT(hdr->b_l1hdr.b_state != arc_l2c_only);
9095 if (HDR_L2_READING(hdr)) {
9097 arc_hdr_set_flags(hdr, ARC_FLAG_L2_EVICTED);
9100 arc_hdr_l2hdr_destroy(hdr);
9140 l2arc_apply_transforms(spa_t *spa, arc_buf_hdr_t *hdr, uint64_t asize,
9144 abd_t *cabd = NULL, *eabd = NULL, *to_write = hdr->b_l1hdr.b_pabd;
9145 enum zio_compress compress = HDR_GET_COMPRESS(hdr);
9146 uint64_t psize = HDR_GET_PSIZE(hdr);
9147 uint64_t size = arc_hdr_size(hdr);
9148 boolean_t ismd = HDR_ISTYPE_METADATA(hdr);
9149 boolean_t bswap = (hdr->b_l1hdr.b_byteswap != DMU_BSWAP_NUMFUNCS);
9154 ASSERT((HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF &&
9155 !HDR_COMPRESSION_ENABLED(hdr)) ||
9156 HDR_ENCRYPTED(hdr) || HDR_SHARED_DATA(hdr) || psize != asize);
9164 if (HDR_HAS_RABD(hdr)) {
9167 abd_copy(to_write, hdr->b_crypt_hdr.b_rabd, psize);
9172 if ((compress == ZIO_COMPRESS_OFF || HDR_COMPRESSION_ENABLED(hdr)) &&
9173 !HDR_ENCRYPTED(hdr)) {
9176 abd_copy(to_write, hdr->b_l1hdr.b_pabd, size);
9182 if (compress != ZIO_COMPRESS_OFF && !HDR_COMPRESSION_ENABLED(hdr)) {
9185 size, MIN(size, psize), hdr->b_complevel);
9200 if (HDR_ENCRYPTED(hdr)) {
9209 ret = spa_keystore_lookup_key(spa, hdr->b_crypt_hdr.b_dsobj,
9215 hdr->b_crypt_hdr.b_ot, bswap, hdr->b_crypt_hdr.b_salt,
9216 hdr->b_crypt_hdr.b_iv, mac, psize, to_write, eabd,
9228 ASSERT0(memcmp(mac, hdr->b_crypt_hdr.b_mac, ZIO_DATA_MAC_LEN));
9238 ASSERT3P(to_write, !=, hdr->b_l1hdr.b_pabd);
9280 arc_buf_hdr_t *hdr, *head, *marker;
9327 hdr = multilist_sublist_head(mls);
9329 hdr = multilist_sublist_tail(mls);
9331 while (hdr != NULL) {
9335 hash_lock = HDR_LOCK(hdr);
9340 hdr = multilist_sublist_next(mls, hdr);
9342 hdr = multilist_sublist_prev(mls, hdr);
9346 passed_sz += HDR_GET_LSIZE(hdr);
9355 if (!l2arc_write_eligible(guid, hdr)) {
9360 ASSERT(HDR_HAS_L1HDR(hdr));
9361 ASSERT3U(HDR_GET_PSIZE(hdr), >, 0);
9362 ASSERT3U(arc_hdr_size(hdr), >, 0);
9363 ASSERT(hdr->b_l1hdr.b_pabd != NULL ||
9364 HDR_HAS_RABD(hdr));
9365 uint64_t psize = HDR_GET_PSIZE(hdr);
9387 multilist_sublist_insert_after(mls, hdr,
9390 multilist_sublist_insert_before(mls, hdr,
9399 * hdr's data, but if we're sharing data between the
9400 * hdr and one of its bufs, L2ARC needs its own copy of
9409 if (HDR_HAS_RABD(hdr) && psize == asize) {
9410 to_write = hdr->b_crypt_hdr.b_rabd;
9411 } else if ((HDR_COMPRESSION_ENABLED(hdr) ||
9412 HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_OFF) &&
9413 !HDR_ENCRYPTED(hdr) && !HDR_SHARED_DATA(hdr) &&
9415 to_write = hdr->b_l1hdr.b_pabd;
9418 arc_buf_contents_t type = arc_buf_type(hdr);
9420 ret = l2arc_apply_transforms(spa, hdr, asize,
9423 arc_hdr_clear_flags(hdr,
9432 hdr->b_l2hdr.b_dev = dev;
9433 hdr->b_l2hdr.b_daddr = dev->l2ad_hand;
9434 hdr->b_l2hdr.b_hits = 0;
9435 hdr->b_l2hdr.b_arcs_state =
9436 hdr->b_l1hdr.b_state->arcs_state;
9438 HDR_SET_L2SIZE(hdr, asize);
9439 arc_hdr_set_flags(hdr, ARC_FLAG_HAS_L2HDR |
9443 arc_hdr_size(hdr), hdr);
9444 l2arc_hdr_arcstats_increment(hdr);
9456 list_insert_head(&dev->l2ad_buflist, hdr);
9459 boolean_t commit = l2arc_log_blk_insert(dev, hdr);
9476 ZIO_CHECKSUM_OFF, NULL, hdr,
9497 hdr = multilist_sublist_next(mls, marker);
9499 hdr = multilist_sublist_prev(mls, marker);
10333 * it to `hdr'. On success, this function returns 0, otherwise the appropriate
10379 * Attempt to rebuild a device containing no actual dev hdr
10521 * Restores the payload of a log block to ARC. This creates empty ARC hdr
10522 * entries which only contain an l2arc hdr, essentially restoring the
10581 * Restores a single ARC buf hdr from a log entry. The ARC buffer is put
10587 arc_buf_hdr_t *hdr, *exists;
10598 hdr = arc_buf_alloc_l2only(L2BLK_GET_LSIZE((le)->le_prop), type,
10610 l2arc_hdr_arcstats_increment(hdr);
10614 list_insert_tail(&dev->l2ad_buflist, hdr);
10615 (void) zfs_refcount_add_many(&dev->l2ad_alloc, arc_hdr_size(hdr), hdr);
10618 exists = buf_hash_insert(hdr, &hash_lock);
10621 arc_hdr_destroy(hdr);
10908 * Inserts ARC buffer header `hdr' into the current L2ARC log block on
10914 l2arc_log_blk_insert(l2arc_dev_t *dev, const arc_buf_hdr_t *hdr)
10925 ASSERT(HDR_HAS_L2HDR(hdr));
10929 le->le_dva = hdr->b_dva;
10930 le->le_birth = hdr->b_birth;
10931 le->le_daddr = hdr->b_l2hdr.b_daddr;
10934 L2BLK_SET_LSIZE((le)->le_prop, HDR_GET_LSIZE(hdr));
10935 L2BLK_SET_PSIZE((le)->le_prop, HDR_GET_PSIZE(hdr));
10936 L2BLK_SET_COMPRESS((le)->le_prop, HDR_GET_COMPRESS(hdr));
10937 le->le_complevel = hdr->b_complevel;
10938 L2BLK_SET_TYPE((le)->le_prop, hdr->b_type);
10939 L2BLK_SET_PROTECTED((le)->le_prop, !!(HDR_PROTECTED(hdr)));
10940 L2BLK_SET_PREFETCH((le)->le_prop, !!(HDR_PREFETCH(hdr)));
10941 L2BLK_SET_STATE((le)->le_prop, hdr->b_l2hdr.b_arcs_state);
10944 HDR_GET_PSIZE(hdr));