Lines Matching refs:hdr
1176 #define HDR_IN_HASH_TABLE(hdr) ((hdr)->b_flags & ARC_FLAG_IN_HASH_TABLE) argument
1177 #define HDR_IO_IN_PROGRESS(hdr) ((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) argument
1178 #define HDR_IO_ERROR(hdr) ((hdr)->b_flags & ARC_FLAG_IO_ERROR) argument
1179 #define HDR_PREFETCH(hdr) ((hdr)->b_flags & ARC_FLAG_PREFETCH) argument
1180 #define HDR_COMPRESSION_ENABLED(hdr) \ argument
1181 ((hdr)->b_flags & ARC_FLAG_COMPRESSED_ARC)
1183 #define HDR_L2CACHE(hdr) ((hdr)->b_flags & ARC_FLAG_L2CACHE) argument
1184 #define HDR_L2_READING(hdr) \ argument
1185 (((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) && \
1186 ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR))
1187 #define HDR_L2_WRITING(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITING) argument
1188 #define HDR_L2_EVICTED(hdr) ((hdr)->b_flags & ARC_FLAG_L2_EVICTED) argument
1189 #define HDR_L2_WRITE_HEAD(hdr) ((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD) argument
1190 #define HDR_SHARED_DATA(hdr) ((hdr)->b_flags & ARC_FLAG_SHARED_DATA) argument
1192 #define HDR_ISTYPE_METADATA(hdr) \ argument
1193 ((hdr)->b_flags & ARC_FLAG_BUFC_METADATA)
1194 #define HDR_ISTYPE_DATA(hdr) (!HDR_ISTYPE_METADATA(hdr)) argument
1196 #define HDR_HAS_L1HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L1HDR) argument
1197 #define HDR_HAS_L2HDR(hdr) ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR) argument
1202 #define HDR_GET_COMPRESS(hdr) ((enum zio_compress)BF32_GET((hdr)->b_flags, \ argument
1204 #define HDR_SET_COMPRESS(hdr, cmp) BF32_SET((hdr)->b_flags, \ argument
1242 #define HDR_LOCK(hdr) \ argument
1243 (BUF_HASH_LOCK(BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth)))
1394 static void arc_hdr_free_pdata(arc_buf_hdr_t *hdr);
1402 static inline void arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
1403 static inline void arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags);
1409 l2arc_trim(const arc_buf_hdr_t *hdr) in l2arc_trim() argument
1411 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev; in l2arc_trim()
1413 ASSERT(HDR_HAS_L2HDR(hdr)); in l2arc_trim()
1416 if (HDR_GET_PSIZE(hdr) != 0) { in l2arc_trim()
1417 trim_map_free(dev->l2ad_vdev, hdr->b_l2hdr.b_daddr, in l2arc_trim()
1418 HDR_GET_PSIZE(hdr), 0); in l2arc_trim()
1439 #define HDR_EMPTY(hdr) \ argument
1440 ((hdr)->b_dva.dva_word[0] == 0 && \
1441 (hdr)->b_dva.dva_word[1] == 0)
1443 #define HDR_EQUAL(spa, dva, birth, hdr) \ argument
1444 ((hdr)->b_dva.dva_word[0] == (dva)->dva_word[0]) && \
1445 ((hdr)->b_dva.dva_word[1] == (dva)->dva_word[1]) && \
1446 ((hdr)->b_birth == birth) && ((hdr)->b_spa == spa)
1449 buf_discard_identity(arc_buf_hdr_t *hdr) in buf_discard_identity() argument
1451 hdr->b_dva.dva_word[0] = 0; in buf_discard_identity()
1452 hdr->b_dva.dva_word[1] = 0; in buf_discard_identity()
1453 hdr->b_birth = 0; in buf_discard_identity()
1463 arc_buf_hdr_t *hdr; in buf_hash_find() local
1466 for (hdr = buf_hash_table.ht_table[idx]; hdr != NULL; in buf_hash_find()
1467 hdr = hdr->b_hash_next) { in buf_hash_find()
1468 if (HDR_EQUAL(spa, dva, birth, hdr)) { in buf_hash_find()
1470 return (hdr); in buf_hash_find()
1486 buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp) in buf_hash_insert() argument
1488 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth); in buf_hash_insert()
1493 ASSERT(!DVA_IS_EMPTY(&hdr->b_dva)); in buf_hash_insert()
1494 ASSERT(hdr->b_birth != 0); in buf_hash_insert()
1495 ASSERT(!HDR_IN_HASH_TABLE(hdr)); in buf_hash_insert()
1506 if (HDR_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr)) in buf_hash_insert()
1510 hdr->b_hash_next = buf_hash_table.ht_table[idx]; in buf_hash_insert()
1511 buf_hash_table.ht_table[idx] = hdr; in buf_hash_insert()
1512 arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE); in buf_hash_insert()
1530 buf_hash_remove(arc_buf_hdr_t *hdr) in buf_hash_remove() argument
1533 uint64_t idx = BUF_HASH_INDEX(hdr->b_spa, &hdr->b_dva, hdr->b_birth); in buf_hash_remove()
1536 ASSERT(HDR_IN_HASH_TABLE(hdr)); in buf_hash_remove()
1539 while ((fhdr = *hdrp) != hdr) { in buf_hash_remove()
1543 *hdrp = hdr->b_hash_next; in buf_hash_remove()
1544 hdr->b_hash_next = NULL; in buf_hash_remove()
1545 arc_hdr_clear_flags(hdr, ARC_FLAG_IN_HASH_TABLE); in buf_hash_remove()
1584 arc_buf_hdr_t *hdr = vbuf; in hdr_full_cons() local
1586 bzero(hdr, HDR_FULL_SIZE); in hdr_full_cons()
1587 cv_init(&hdr->b_l1hdr.b_cv, NULL, CV_DEFAULT, NULL); in hdr_full_cons()
1588 refcount_create(&hdr->b_l1hdr.b_refcnt); in hdr_full_cons()
1589 mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL); in hdr_full_cons()
1590 multilist_link_init(&hdr->b_l1hdr.b_arc_node); in hdr_full_cons()
1600 arc_buf_hdr_t *hdr = vbuf; in hdr_l2only_cons() local
1602 bzero(hdr, HDR_L2ONLY_SIZE); in hdr_l2only_cons()
1629 arc_buf_hdr_t *hdr = vbuf; in hdr_full_dest() local
1631 ASSERT(HDR_EMPTY(hdr)); in hdr_full_dest()
1632 cv_destroy(&hdr->b_l1hdr.b_cv); in hdr_full_dest()
1633 refcount_destroy(&hdr->b_l1hdr.b_refcnt); in hdr_full_dest()
1634 mutex_destroy(&hdr->b_l1hdr.b_freeze_lock); in hdr_full_dest()
1635 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); in hdr_full_dest()
1643 arc_buf_hdr_t *hdr = vbuf; in hdr_l2only_dest() local
1645 ASSERT(HDR_EMPTY(hdr)); in hdr_l2only_dest()
1730 arc_cksum_free(arc_buf_hdr_t *hdr) in arc_cksum_free() argument
1732 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_cksum_free()
1733 mutex_enter(&hdr->b_l1hdr.b_freeze_lock); in arc_cksum_free()
1734 if (hdr->b_l1hdr.b_freeze_cksum != NULL) { in arc_cksum_free()
1735 kmem_free(hdr->b_l1hdr.b_freeze_cksum, sizeof (zio_cksum_t)); in arc_cksum_free()
1736 hdr->b_l1hdr.b_freeze_cksum = NULL; in arc_cksum_free()
1738 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); in arc_cksum_free()
1744 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_cksum_verify() local
1750 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_cksum_verify()
1752 mutex_enter(&hdr->b_l1hdr.b_freeze_lock); in arc_cksum_verify()
1753 if (hdr->b_l1hdr.b_freeze_cksum == NULL || HDR_IO_ERROR(hdr)) { in arc_cksum_verify()
1754 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); in arc_cksum_verify()
1757 fletcher_2_native(buf->b_data, HDR_GET_LSIZE(hdr), NULL, &zc); in arc_cksum_verify()
1758 if (!ZIO_CHECKSUM_EQUAL(*hdr->b_l1hdr.b_freeze_cksum, zc)) in arc_cksum_verify()
1760 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); in arc_cksum_verify()
1764 arc_cksum_is_equal(arc_buf_hdr_t *hdr, zio_t *zio) in arc_cksum_is_equal() argument
1770 VERIFY3U(BP_GET_PSIZE(zio->io_bp), ==, HDR_GET_PSIZE(hdr)); in arc_cksum_is_equal()
1784 if (!HDR_COMPRESSION_ENABLED(hdr) && compress != ZIO_COMPRESS_OFF) { in arc_cksum_is_equal()
1785 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF); in arc_cksum_is_equal()
1786 uint64_t lsize = HDR_GET_LSIZE(hdr); in arc_cksum_is_equal()
1789 void *cbuf = zio_buf_alloc(HDR_GET_PSIZE(hdr)); in arc_cksum_is_equal()
1791 ASSERT3U(csize, <=, HDR_GET_PSIZE(hdr)); in arc_cksum_is_equal()
1792 if (csize < HDR_GET_PSIZE(hdr)) { in arc_cksum_is_equal()
1805 bzero((char *)cbuf + csize, HDR_GET_PSIZE(hdr) - csize); in arc_cksum_is_equal()
1806 csize = HDR_GET_PSIZE(hdr); in arc_cksum_is_equal()
1808 zio_push_transform(zio, cbuf, csize, HDR_GET_PSIZE(hdr), NULL); in arc_cksum_is_equal()
1834 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_cksum_compute() local
1839 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_cksum_compute()
1841 if (hdr->b_l1hdr.b_freeze_cksum != NULL) { in arc_cksum_compute()
1842 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); in arc_cksum_compute()
1845 hdr->b_l1hdr.b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), in arc_cksum_compute()
1847 fletcher_2_native(buf->b_data, HDR_GET_LSIZE(hdr), NULL, in arc_cksum_compute()
1848 hdr->b_l1hdr.b_freeze_cksum); in arc_cksum_compute()
1849 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); in arc_cksum_compute()
1901 arc_buf_type(arc_buf_hdr_t *hdr) in arc_buf_type() argument
1904 if (HDR_ISTYPE_METADATA(hdr)) { in arc_buf_type()
1909 VERIFY3U(hdr->b_type, ==, type); in arc_buf_type()
1932 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_buf_thaw() local
1935 if (hdr->b_l1hdr.b_state != arc_anon) in arc_buf_thaw()
1937 if (HDR_IO_IN_PROGRESS(hdr)) in arc_buf_thaw()
1942 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_buf_thaw()
1943 arc_cksum_free(hdr); in arc_buf_thaw()
1945 mutex_enter(&hdr->b_l1hdr.b_freeze_lock); in arc_buf_thaw()
1948 if (hdr->b_l1hdr.b_thawed != NULL) in arc_buf_thaw()
1949 kmem_free(hdr->b_l1hdr.b_thawed, 1); in arc_buf_thaw()
1950 hdr->b_l1hdr.b_thawed = kmem_alloc(1, KM_SLEEP); in arc_buf_thaw()
1954 mutex_exit(&hdr->b_l1hdr.b_freeze_lock); in arc_buf_thaw()
1964 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_buf_freeze() local
1970 hash_lock = HDR_LOCK(hdr); in arc_buf_freeze()
1973 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_buf_freeze()
1974 ASSERT(hdr->b_l1hdr.b_freeze_cksum != NULL || in arc_buf_freeze()
1975 hdr->b_l1hdr.b_state == arc_anon); in arc_buf_freeze()
1990 arc_hdr_set_flags(arc_buf_hdr_t *hdr, arc_flags_t flags) in arc_hdr_set_flags() argument
1992 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); in arc_hdr_set_flags()
1993 hdr->b_flags |= flags; in arc_hdr_set_flags()
1997 arc_hdr_clear_flags(arc_buf_hdr_t *hdr, arc_flags_t flags) in arc_hdr_clear_flags() argument
1999 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); in arc_hdr_clear_flags()
2000 hdr->b_flags &= ~flags; in arc_hdr_clear_flags()
2011 arc_hdr_set_compress(arc_buf_hdr_t *hdr, enum zio_compress cmp) in arc_hdr_set_compress() argument
2013 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); in arc_hdr_set_compress()
2022 if (!zfs_compressed_arc_enabled || HDR_GET_PSIZE(hdr) == 0) { in arc_hdr_set_compress()
2023 arc_hdr_clear_flags(hdr, ARC_FLAG_COMPRESSED_ARC); in arc_hdr_set_compress()
2024 HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_OFF); in arc_hdr_set_compress()
2025 ASSERT(!HDR_COMPRESSION_ENABLED(hdr)); in arc_hdr_set_compress()
2026 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF); in arc_hdr_set_compress()
2028 arc_hdr_set_flags(hdr, ARC_FLAG_COMPRESSED_ARC); in arc_hdr_set_compress()
2029 HDR_SET_COMPRESS(hdr, cmp); in arc_hdr_set_compress()
2030 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, cmp); in arc_hdr_set_compress()
2031 ASSERT(HDR_COMPRESSION_ENABLED(hdr)); in arc_hdr_set_compress()
2038 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_decompress() local
2039 dmu_object_byteswap_t bswap = hdr->b_l1hdr.b_byteswap; in arc_decompress()
2043 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF); in arc_decompress()
2044 } else if (HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_OFF) { in arc_decompress()
2050 IMPLY(HDR_COMPRESSION_ENABLED(hdr), HDR_GET_PSIZE(hdr) == 0 || in arc_decompress()
2051 HDR_GET_PSIZE(hdr) == HDR_GET_LSIZE(hdr)); in arc_decompress()
2052 ASSERT(!HDR_SHARED_DATA(hdr)); in arc_decompress()
2053 bcopy(hdr->b_l1hdr.b_pdata, buf->b_data, HDR_GET_LSIZE(hdr)); in arc_decompress()
2055 ASSERT(!HDR_SHARED_DATA(hdr)); in arc_decompress()
2056 ASSERT3U(HDR_GET_LSIZE(hdr), !=, HDR_GET_PSIZE(hdr)); in arc_decompress()
2057 error = zio_decompress_data(HDR_GET_COMPRESS(hdr), in arc_decompress()
2058 hdr->b_l1hdr.b_pdata, buf->b_data, HDR_GET_PSIZE(hdr), in arc_decompress()
2059 HDR_GET_LSIZE(hdr)); in arc_decompress()
2062 hdr, HDR_GET_COMPRESS(hdr), HDR_GET_PSIZE(hdr), in arc_decompress()
2063 HDR_GET_LSIZE(hdr)); in arc_decompress()
2068 ASSERT(!HDR_SHARED_DATA(hdr)); in arc_decompress()
2070 dmu_ot_byteswap[bswap].ob_func(buf->b_data, HDR_GET_LSIZE(hdr)); in arc_decompress()
2080 arc_hdr_size(arc_buf_hdr_t *hdr) in arc_hdr_size() argument
2084 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF && in arc_hdr_size()
2085 HDR_GET_PSIZE(hdr) > 0) { in arc_hdr_size()
2086 size = HDR_GET_PSIZE(hdr); in arc_hdr_size()
2088 ASSERT3U(HDR_GET_LSIZE(hdr), !=, 0); in arc_hdr_size()
2089 size = HDR_GET_LSIZE(hdr); in arc_hdr_size()
2100 arc_evictable_space_increment(arc_buf_hdr_t *hdr, arc_state_t *state) in arc_evictable_space_increment() argument
2102 arc_buf_contents_t type = arc_buf_type(hdr); in arc_evictable_space_increment()
2103 uint64_t lsize = HDR_GET_LSIZE(hdr); in arc_evictable_space_increment()
2105 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_evictable_space_increment()
2108 ASSERT0(hdr->b_l1hdr.b_bufcnt); in arc_evictable_space_increment()
2109 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); in arc_evictable_space_increment()
2110 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL); in arc_evictable_space_increment()
2111 (void) refcount_add_many(&state->arcs_esize[type], lsize, hdr); in arc_evictable_space_increment()
2116 if (hdr->b_l1hdr.b_pdata != NULL) { in arc_evictable_space_increment()
2118 arc_hdr_size(hdr), hdr); in arc_evictable_space_increment()
2120 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; in arc_evictable_space_increment()
2136 arc_evitable_space_decrement(arc_buf_hdr_t *hdr, arc_state_t *state) in arc_evitable_space_decrement() argument
2138 arc_buf_contents_t type = arc_buf_type(hdr); in arc_evitable_space_decrement()
2139 uint64_t lsize = HDR_GET_LSIZE(hdr); in arc_evitable_space_decrement()
2141 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_evitable_space_decrement()
2144 ASSERT0(hdr->b_l1hdr.b_bufcnt); in arc_evitable_space_decrement()
2145 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); in arc_evitable_space_decrement()
2146 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL); in arc_evitable_space_decrement()
2148 lsize, hdr); in arc_evitable_space_decrement()
2153 if (hdr->b_l1hdr.b_pdata != NULL) { in arc_evitable_space_decrement()
2155 arc_hdr_size(hdr), hdr); in arc_evitable_space_decrement()
2157 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; in arc_evitable_space_decrement()
2175 add_reference(arc_buf_hdr_t *hdr, void *tag) in add_reference() argument
2177 ASSERT(HDR_HAS_L1HDR(hdr)); in add_reference()
2178 if (!MUTEX_HELD(HDR_LOCK(hdr))) { in add_reference()
2179 ASSERT(hdr->b_l1hdr.b_state == arc_anon); in add_reference()
2180 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in add_reference()
2181 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); in add_reference()
2184 arc_state_t *state = hdr->b_l1hdr.b_state; in add_reference()
2186 if ((refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) && in add_reference()
2190 multilist_remove(&state->arcs_list[arc_buf_type(hdr)], in add_reference()
2191 hdr); in add_reference()
2192 arc_evitable_space_decrement(hdr, state); in add_reference()
2195 arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH); in add_reference()
2205 remove_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag) in remove_reference() argument
2208 arc_state_t *state = hdr->b_l1hdr.b_state; in remove_reference()
2210 ASSERT(HDR_HAS_L1HDR(hdr)); in remove_reference()
2218 if (((cnt = refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) && in remove_reference()
2220 multilist_insert(&state->arcs_list[arc_buf_type(hdr)], hdr); in remove_reference()
2221 ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0); in remove_reference()
2222 arc_evictable_space_increment(hdr, state); in remove_reference()
2232 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr, in arc_change_state() argument
2239 arc_buf_contents_t buftype = arc_buf_type(hdr); in arc_change_state()
2248 if (HDR_HAS_L1HDR(hdr)) { in arc_change_state()
2249 old_state = hdr->b_l1hdr.b_state; in arc_change_state()
2250 refcnt = refcount_count(&hdr->b_l1hdr.b_refcnt); in arc_change_state()
2251 bufcnt = hdr->b_l1hdr.b_bufcnt; in arc_change_state()
2252 update_old = (bufcnt > 0 || hdr->b_l1hdr.b_pdata != NULL); in arc_change_state()
2272 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_change_state()
2273 multilist_remove(&old_state->arcs_list[buftype], hdr); in arc_change_state()
2277 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); in arc_change_state()
2280 arc_evitable_space_decrement(hdr, old_state); in arc_change_state()
2290 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_change_state()
2291 multilist_insert(&new_state->arcs_list[buftype], hdr); in arc_change_state()
2295 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); in arc_change_state()
2298 arc_evictable_space_increment(hdr, new_state); in arc_change_state()
2302 ASSERT(!HDR_EMPTY(hdr)); in arc_change_state()
2303 if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr)) in arc_change_state()
2304 buf_hash_remove(hdr); in arc_change_state()
2309 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_change_state()
2321 HDR_GET_LSIZE(hdr), hdr); in arc_change_state()
2322 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL); in arc_change_state()
2331 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; in arc_change_state()
2349 HDR_GET_LSIZE(hdr), buf); in arc_change_state()
2353 if (hdr->b_l1hdr.b_pdata != NULL) { in arc_change_state()
2355 arc_hdr_size(hdr), hdr); in arc_change_state()
2363 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_change_state()
2376 HDR_GET_LSIZE(hdr), hdr); in arc_change_state()
2377 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL); in arc_change_state()
2386 for (arc_buf_t *buf = hdr->b_l1hdr.b_buf; buf != NULL; in arc_change_state()
2404 &old_state->arcs_size, HDR_GET_LSIZE(hdr), in arc_change_state()
2408 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL); in arc_change_state()
2410 &old_state->arcs_size, arc_hdr_size(hdr), hdr); in arc_change_state()
2414 if (HDR_HAS_L1HDR(hdr)) in arc_change_state()
2415 hdr->b_l1hdr.b_state = new_state; in arc_change_state()
2493 arc_buf_alloc_impl(arc_buf_hdr_t *hdr, void *tag) in arc_buf_alloc_impl() argument
2497 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_buf_alloc_impl()
2498 ASSERT3U(HDR_GET_LSIZE(hdr), >, 0); in arc_buf_alloc_impl()
2499 VERIFY(hdr->b_type == ARC_BUFC_DATA || in arc_buf_alloc_impl()
2500 hdr->b_type == ARC_BUFC_METADATA); in arc_buf_alloc_impl()
2502 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_buf_alloc_impl()
2503 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); in arc_buf_alloc_impl()
2504 ASSERT0(hdr->b_l1hdr.b_bufcnt); in arc_buf_alloc_impl()
2507 buf->b_hdr = hdr; in arc_buf_alloc_impl()
2511 add_reference(hdr, tag); in arc_buf_alloc_impl()
2517 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); in arc_buf_alloc_impl()
2527 if (hdr->b_l1hdr.b_byteswap == DMU_BSWAP_NUMFUNCS && in arc_buf_alloc_impl()
2528 HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_OFF && !HDR_L2_WRITING(hdr)) { in arc_buf_alloc_impl()
2529 buf->b_data = hdr->b_l1hdr.b_pdata; in arc_buf_alloc_impl()
2530 arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA); in arc_buf_alloc_impl()
2532 buf->b_data = arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf); in arc_buf_alloc_impl()
2533 ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr)); in arc_buf_alloc_impl()
2534 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA); in arc_buf_alloc_impl()
2538 hdr->b_l1hdr.b_buf = buf; in arc_buf_alloc_impl()
2539 hdr->b_l1hdr.b_bufcnt += 1; in arc_buf_alloc_impl()
2551 arc_buf_hdr_t *hdr = from->b_hdr; in arc_buf_clone() local
2552 uint64_t size = HDR_GET_LSIZE(hdr); in arc_buf_clone()
2554 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_buf_clone()
2555 ASSERT(hdr->b_l1hdr.b_state != arc_anon); in arc_buf_clone()
2558 buf->b_hdr = hdr; in arc_buf_clone()
2560 buf->b_next = hdr->b_l1hdr.b_buf; in arc_buf_clone()
2561 hdr->b_l1hdr.b_buf = buf; in arc_buf_clone()
2562 buf->b_data = arc_get_data_buf(hdr, HDR_GET_LSIZE(hdr), buf); in arc_buf_clone()
2564 hdr->b_l1hdr.b_bufcnt += 1; in arc_buf_clone()
2566 ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr)); in arc_buf_clone()
2595 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_return_buf() local
2598 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_return_buf()
2599 (void) refcount_add(&hdr->b_l1hdr.b_refcnt, tag); in arc_return_buf()
2600 (void) refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag); in arc_return_buf()
2602 atomic_add_64(&arc_loaned_bytes, -HDR_GET_LSIZE(hdr)); in arc_return_buf()
2609 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_loan_inuse_buf() local
2612 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_loan_inuse_buf()
2613 (void) refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag); in arc_loan_inuse_buf()
2614 (void) refcount_remove(&hdr->b_l1hdr.b_refcnt, tag); in arc_loan_inuse_buf()
2616 atomic_add_64(&arc_loaned_bytes, HDR_GET_LSIZE(hdr)); in arc_loan_inuse_buf()
2633 arc_hdr_free_on_write(arc_buf_hdr_t *hdr) in arc_hdr_free_on_write() argument
2635 arc_state_t *state = hdr->b_l1hdr.b_state; in arc_hdr_free_on_write()
2636 arc_buf_contents_t type = arc_buf_type(hdr); in arc_hdr_free_on_write()
2637 uint64_t size = arc_hdr_size(hdr); in arc_hdr_free_on_write()
2640 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) { in arc_hdr_free_on_write()
2641 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_hdr_free_on_write()
2645 size, hdr); in arc_hdr_free_on_write()
2647 (void) refcount_remove_many(&state->arcs_size, size, hdr); in arc_hdr_free_on_write()
2655 l2arc_free_data_on_write(hdr->b_l1hdr.b_pdata, size, type); in arc_hdr_free_on_write()
2664 arc_share_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf) in arc_share_buf() argument
2666 arc_state_t *state = hdr->b_l1hdr.b_state; in arc_share_buf()
2668 ASSERT(!HDR_SHARED_DATA(hdr)); in arc_share_buf()
2670 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL); in arc_share_buf()
2671 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); in arc_share_buf()
2678 refcount_transfer_ownership(&state->arcs_size, buf, hdr); in arc_share_buf()
2679 hdr->b_l1hdr.b_pdata = buf->b_data; in arc_share_buf()
2680 arc_hdr_set_flags(hdr, ARC_FLAG_SHARED_DATA); in arc_share_buf()
2687 ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr)); in arc_share_buf()
2688 ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr)); in arc_share_buf()
2689 ARCSTAT_INCR(arcstat_overhead_size, -HDR_GET_LSIZE(hdr)); in arc_share_buf()
2693 arc_unshare_buf(arc_buf_hdr_t *hdr, arc_buf_t *buf) in arc_unshare_buf() argument
2695 arc_state_t *state = hdr->b_l1hdr.b_state; in arc_unshare_buf()
2697 ASSERT(HDR_SHARED_DATA(hdr)); in arc_unshare_buf()
2699 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL); in arc_unshare_buf()
2700 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); in arc_unshare_buf()
2706 refcount_transfer_ownership(&state->arcs_size, hdr, buf); in arc_unshare_buf()
2707 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA); in arc_unshare_buf()
2708 hdr->b_l1hdr.b_pdata = NULL; in arc_unshare_buf()
2714 ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr)); in arc_unshare_buf()
2715 ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr)); in arc_unshare_buf()
2716 ARCSTAT_INCR(arcstat_overhead_size, HDR_GET_LSIZE(hdr)); in arc_unshare_buf()
2727 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_buf_destroy_impl() local
2728 uint64_t size = HDR_GET_LSIZE(hdr); in arc_buf_destroy_impl()
2742 ASSERT(MUTEX_HELD(HDR_LOCK(hdr)) || HDR_EMPTY(hdr)); in arc_buf_destroy_impl()
2751 ASSERT(HDR_SHARED_DATA(hdr)); in arc_buf_destroy_impl()
2752 arc_hdr_clear_flags(hdr, ARC_FLAG_SHARED_DATA); in arc_buf_destroy_impl()
2754 arc_free_data_buf(hdr, buf->b_data, size, buf); in arc_buf_destroy_impl()
2759 ASSERT(hdr->b_l1hdr.b_bufcnt > 0); in arc_buf_destroy_impl()
2760 hdr->b_l1hdr.b_bufcnt -= 1; in arc_buf_destroy_impl()
2769 bufp = &hdr->b_l1hdr.b_buf; in arc_buf_destroy_impl()
2799 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL); in arc_buf_destroy_impl()
2800 arc_hdr_free_pdata(hdr); in arc_buf_destroy_impl()
2808 arc_share_buf(hdr, lastbuf); in arc_buf_destroy_impl()
2809 } else if (HDR_SHARED_DATA(hdr)) { in arc_buf_destroy_impl()
2813 if (hdr->b_l1hdr.b_bufcnt == 0) in arc_buf_destroy_impl()
2814 arc_cksum_free(hdr); in arc_buf_destroy_impl()
2822 arc_hdr_alloc_pdata(arc_buf_hdr_t *hdr) in arc_hdr_alloc_pdata() argument
2824 ASSERT3U(HDR_GET_LSIZE(hdr), >, 0); in arc_hdr_alloc_pdata()
2825 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_hdr_alloc_pdata()
2826 ASSERT(!HDR_SHARED_DATA(hdr)); in arc_hdr_alloc_pdata()
2828 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL); in arc_hdr_alloc_pdata()
2829 hdr->b_l1hdr.b_pdata = arc_get_data_buf(hdr, arc_hdr_size(hdr), hdr); in arc_hdr_alloc_pdata()
2830 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS; in arc_hdr_alloc_pdata()
2831 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL); in arc_hdr_alloc_pdata()
2833 ARCSTAT_INCR(arcstat_compressed_size, arc_hdr_size(hdr)); in arc_hdr_alloc_pdata()
2834 ARCSTAT_INCR(arcstat_uncompressed_size, HDR_GET_LSIZE(hdr)); in arc_hdr_alloc_pdata()
2838 arc_hdr_free_pdata(arc_buf_hdr_t *hdr) in arc_hdr_free_pdata() argument
2840 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_hdr_free_pdata()
2841 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL); in arc_hdr_free_pdata()
2849 if (HDR_L2_WRITING(hdr)) { in arc_hdr_free_pdata()
2850 arc_hdr_free_on_write(hdr); in arc_hdr_free_pdata()
2853 arc_free_data_buf(hdr, hdr->b_l1hdr.b_pdata, in arc_hdr_free_pdata()
2854 arc_hdr_size(hdr), hdr); in arc_hdr_free_pdata()
2856 hdr->b_l1hdr.b_pdata = NULL; in arc_hdr_free_pdata()
2857 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS; in arc_hdr_free_pdata()
2859 ARCSTAT_INCR(arcstat_compressed_size, -arc_hdr_size(hdr)); in arc_hdr_free_pdata()
2860 ARCSTAT_INCR(arcstat_uncompressed_size, -HDR_GET_LSIZE(hdr)); in arc_hdr_free_pdata()
2867 arc_buf_hdr_t *hdr; in arc_hdr_alloc() local
2872 hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE); in arc_hdr_alloc()
2873 ASSERT(HDR_EMPTY(hdr)); in arc_hdr_alloc()
2874 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); in arc_hdr_alloc()
2875 ASSERT3P(hdr->b_l1hdr.b_thawed, ==, NULL); in arc_hdr_alloc()
2876 HDR_SET_PSIZE(hdr, psize); in arc_hdr_alloc()
2877 HDR_SET_LSIZE(hdr, lsize); in arc_hdr_alloc()
2878 hdr->b_spa = spa; in arc_hdr_alloc()
2879 hdr->b_type = type; in arc_hdr_alloc()
2880 hdr->b_flags = 0; in arc_hdr_alloc()
2881 arc_hdr_set_flags(hdr, arc_bufc_to_flags(type) | ARC_FLAG_HAS_L1HDR); in arc_hdr_alloc()
2882 arc_hdr_set_compress(hdr, compress); in arc_hdr_alloc()
2884 hdr->b_l1hdr.b_state = arc_anon; in arc_hdr_alloc()
2885 hdr->b_l1hdr.b_arc_access = 0; in arc_hdr_alloc()
2886 hdr->b_l1hdr.b_bufcnt = 0; in arc_hdr_alloc()
2887 hdr->b_l1hdr.b_buf = NULL; in arc_hdr_alloc()
2894 arc_hdr_alloc_pdata(hdr); in arc_hdr_alloc()
2895 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_hdr_alloc()
2897 return (hdr); in arc_hdr_alloc()
2908 arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new) in arc_hdr_realloc() argument
2910 ASSERT(HDR_HAS_L2HDR(hdr)); in arc_hdr_realloc()
2913 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev; in arc_hdr_realloc()
2920 ASSERT(MUTEX_HELD(HDR_LOCK(hdr))); in arc_hdr_realloc()
2921 buf_hash_remove(hdr); in arc_hdr_realloc()
2923 bcopy(hdr, nhdr, HDR_L2ONLY_SIZE); in arc_hdr_realloc()
2937 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); in arc_hdr_realloc()
2938 ASSERT0(hdr->b_l1hdr.b_bufcnt); in arc_hdr_realloc()
2939 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); in arc_hdr_realloc()
2948 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); in arc_hdr_realloc()
2956 VERIFY(!HDR_L2_WRITING(hdr)); in arc_hdr_realloc()
2957 VERIFY3P(hdr->b_l1hdr.b_pdata, ==, NULL); in arc_hdr_realloc()
2960 if (hdr->b_l1hdr.b_thawed != NULL) { in arc_hdr_realloc()
2961 kmem_free(hdr->b_l1hdr.b_thawed, 1); in arc_hdr_realloc()
2962 hdr->b_l1hdr.b_thawed = NULL; in arc_hdr_realloc()
2974 ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node)); in arc_hdr_realloc()
2984 list_insert_after(&dev->l2ad_buflist, hdr, nhdr); in arc_hdr_realloc()
2985 list_remove(&dev->l2ad_buflist, hdr); in arc_hdr_realloc()
2997 (void) refcount_remove_many(&dev->l2ad_alloc, arc_hdr_size(hdr), hdr); in arc_hdr_realloc()
3000 buf_discard_identity(hdr); in arc_hdr_realloc()
3001 kmem_cache_free(old, hdr); in arc_hdr_realloc()
3013 arc_buf_hdr_t *hdr = arc_hdr_alloc(spa_load_guid(spa), size, size, in arc_alloc_buf() local
3015 ASSERT(!MUTEX_HELD(HDR_LOCK(hdr))); in arc_alloc_buf()
3016 arc_buf_t *buf = arc_buf_alloc_impl(hdr, tag); in arc_alloc_buf()
3022 arc_hdr_l2hdr_destroy(arc_buf_hdr_t *hdr) in arc_hdr_l2hdr_destroy() argument
3024 l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr; in arc_hdr_l2hdr_destroy()
3026 uint64_t asize = arc_hdr_size(hdr); in arc_hdr_l2hdr_destroy()
3029 ASSERT(HDR_HAS_L2HDR(hdr)); in arc_hdr_l2hdr_destroy()
3031 list_remove(&dev->l2ad_buflist, hdr); in arc_hdr_l2hdr_destroy()
3034 ARCSTAT_INCR(arcstat_l2_size, -HDR_GET_LSIZE(hdr)); in arc_hdr_l2hdr_destroy()
3038 (void) refcount_remove_many(&dev->l2ad_alloc, asize, hdr); in arc_hdr_l2hdr_destroy()
3039 arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR); in arc_hdr_l2hdr_destroy()
3043 arc_hdr_destroy(arc_buf_hdr_t *hdr) in arc_hdr_destroy() argument
3045 if (HDR_HAS_L1HDR(hdr)) { in arc_hdr_destroy()
3046 ASSERT(hdr->b_l1hdr.b_buf == NULL || in arc_hdr_destroy()
3047 hdr->b_l1hdr.b_bufcnt > 0); in arc_hdr_destroy()
3048 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_hdr_destroy()
3049 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); in arc_hdr_destroy()
3051 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_hdr_destroy()
3052 ASSERT(!HDR_IN_HASH_TABLE(hdr)); in arc_hdr_destroy()
3054 if (!HDR_EMPTY(hdr)) in arc_hdr_destroy()
3055 buf_discard_identity(hdr); in arc_hdr_destroy()
3057 if (HDR_HAS_L2HDR(hdr)) { in arc_hdr_destroy()
3058 l2arc_dev_t *dev = hdr->b_l2hdr.b_dev; in arc_hdr_destroy()
3073 if (HDR_HAS_L2HDR(hdr)) { in arc_hdr_destroy()
3074 l2arc_trim(hdr); in arc_hdr_destroy()
3075 arc_hdr_l2hdr_destroy(hdr); in arc_hdr_destroy()
3082 if (HDR_HAS_L1HDR(hdr)) { in arc_hdr_destroy()
3083 arc_cksum_free(hdr); in arc_hdr_destroy()
3085 while (hdr->b_l1hdr.b_buf != NULL) in arc_hdr_destroy()
3086 arc_buf_destroy_impl(hdr->b_l1hdr.b_buf, B_TRUE); in arc_hdr_destroy()
3089 if (hdr->b_l1hdr.b_thawed != NULL) { in arc_hdr_destroy()
3090 kmem_free(hdr->b_l1hdr.b_thawed, 1); in arc_hdr_destroy()
3091 hdr->b_l1hdr.b_thawed = NULL; in arc_hdr_destroy()
3095 if (hdr->b_l1hdr.b_pdata != NULL) { in arc_hdr_destroy()
3096 arc_hdr_free_pdata(hdr); in arc_hdr_destroy()
3100 ASSERT3P(hdr->b_hash_next, ==, NULL); in arc_hdr_destroy()
3101 if (HDR_HAS_L1HDR(hdr)) { in arc_hdr_destroy()
3102 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); in arc_hdr_destroy()
3103 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL); in arc_hdr_destroy()
3104 kmem_cache_free(hdr_full_cache, hdr); in arc_hdr_destroy()
3106 kmem_cache_free(hdr_l2only_cache, hdr); in arc_hdr_destroy()
3113 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_buf_destroy() local
3114 kmutex_t *hash_lock = HDR_LOCK(hdr); in arc_buf_destroy()
3116 if (hdr->b_l1hdr.b_state == arc_anon) { in arc_buf_destroy()
3117 ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1); in arc_buf_destroy()
3118 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_buf_destroy()
3119 VERIFY0(remove_reference(hdr, NULL, tag)); in arc_buf_destroy()
3120 arc_hdr_destroy(hdr); in arc_buf_destroy()
3125 ASSERT3P(hdr, ==, buf->b_hdr); in arc_buf_destroy()
3126 ASSERT(hdr->b_l1hdr.b_bufcnt > 0); in arc_buf_destroy()
3127 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); in arc_buf_destroy()
3128 ASSERT3P(hdr->b_l1hdr.b_state, !=, arc_anon); in arc_buf_destroy()
3131 (void) remove_reference(hdr, hash_lock, tag); in arc_buf_destroy()
3155 arc_evict_hdr(arc_buf_hdr_t *hdr, kmutex_t *hash_lock) in arc_evict_hdr() argument
3161 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_evict_hdr()
3163 state = hdr->b_l1hdr.b_state; in arc_evict_hdr()
3165 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_evict_hdr()
3166 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); in arc_evict_hdr()
3175 if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) { in arc_evict_hdr()
3181 bytes_evicted += HDR_GET_LSIZE(hdr); in arc_evict_hdr()
3183 DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr); in arc_evict_hdr()
3185 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL); in arc_evict_hdr()
3186 if (HDR_HAS_L2HDR(hdr)) { in arc_evict_hdr()
3187 ASSERT(hdr->b_l1hdr.b_pdata == NULL); in arc_evict_hdr()
3192 arc_change_state(arc_l2c_only, hdr, hash_lock); in arc_evict_hdr()
3197 hdr = arc_hdr_realloc(hdr, hdr_full_cache, in arc_evict_hdr()
3200 ASSERT(hdr->b_l1hdr.b_pdata == NULL); in arc_evict_hdr()
3201 arc_change_state(arc_anon, hdr, hash_lock); in arc_evict_hdr()
3202 arc_hdr_destroy(hdr); in arc_evict_hdr()
3211 if (HDR_IO_IN_PROGRESS(hdr) || in arc_evict_hdr()
3212 ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) && in arc_evict_hdr()
3213 ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access < in arc_evict_hdr()
3219 ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt)); in arc_evict_hdr()
3220 while (hdr->b_l1hdr.b_buf) { in arc_evict_hdr()
3221 arc_buf_t *buf = hdr->b_l1hdr.b_buf; in arc_evict_hdr()
3227 bytes_evicted += HDR_GET_LSIZE(hdr); in arc_evict_hdr()
3232 if (HDR_HAS_L2HDR(hdr)) { in arc_evict_hdr()
3233 ARCSTAT_INCR(arcstat_evict_l2_cached, HDR_GET_LSIZE(hdr)); in arc_evict_hdr()
3235 if (l2arc_write_eligible(hdr->b_spa, hdr)) { in arc_evict_hdr()
3237 HDR_GET_LSIZE(hdr)); in arc_evict_hdr()
3240 HDR_GET_LSIZE(hdr)); in arc_evict_hdr()
3244 if (hdr->b_l1hdr.b_bufcnt == 0) { in arc_evict_hdr()
3245 arc_cksum_free(hdr); in arc_evict_hdr()
3247 bytes_evicted += arc_hdr_size(hdr); in arc_evict_hdr()
3255 arc_hdr_free_pdata(hdr); in arc_evict_hdr()
3257 arc_change_state(evicted_state, hdr, hash_lock); in arc_evict_hdr()
3258 ASSERT(HDR_IN_HASH_TABLE(hdr)); in arc_evict_hdr()
3259 arc_hdr_set_flags(hdr, ARC_FLAG_IN_HASH_TABLE); in arc_evict_hdr()
3260 DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr); in arc_evict_hdr()
3272 arc_buf_hdr_t *hdr; in arc_evict_state_impl() local
3281 for (hdr = multilist_sublist_prev(mls, marker); hdr != NULL; in arc_evict_state_impl()
3282 hdr = multilist_sublist_prev(mls, marker)) { in arc_evict_state_impl()
3309 if (hdr->b_spa == 0) in arc_evict_state_impl()
3313 if (spa != 0 && hdr->b_spa != spa) { in arc_evict_state_impl()
3318 hash_lock = HDR_LOCK(hdr); in arc_evict_state_impl()
3332 uint64_t evicted = arc_evict_hdr(hdr, hash_lock); in arc_evict_state_impl()
4374 arc_get_data_buf(arc_buf_hdr_t *hdr, uint64_t size, void *tag) in arc_get_data_buf() argument
4377 arc_state_t *state = hdr->b_l1hdr.b_state; in arc_get_data_buf()
4378 arc_buf_contents_t type = arc_buf_type(hdr); in arc_get_data_buf()
4418 VERIFY3U(hdr->b_type, ==, type); in arc_get_data_buf()
4445 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) { in arc_get_data_buf()
4446 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_get_data_buf()
4455 if (arc_size < arc_c && hdr->b_l1hdr.b_state == arc_anon && in arc_get_data_buf()
4468 arc_free_data_buf(arc_buf_hdr_t *hdr, void *data, uint64_t size, void *tag) in arc_free_data_buf() argument
4470 arc_state_t *state = hdr->b_l1hdr.b_state; in arc_free_data_buf()
4471 arc_buf_contents_t type = arc_buf_type(hdr); in arc_free_data_buf()
4474 if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) { in arc_free_data_buf()
4475 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_free_data_buf()
4483 VERIFY3U(hdr->b_type, ==, type); in arc_free_data_buf()
4499 arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock) in arc_access() argument
4504 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_access()
4506 if (hdr->b_l1hdr.b_state == arc_anon) { in arc_access()
4513 ASSERT0(hdr->b_l1hdr.b_arc_access); in arc_access()
4514 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); in arc_access()
4515 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr); in arc_access()
4516 arc_change_state(arc_mru, hdr, hash_lock); in arc_access()
4518 } else if (hdr->b_l1hdr.b_state == arc_mru) { in arc_access()
4529 if (HDR_PREFETCH(hdr)) { in arc_access()
4530 if (refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) { in arc_access()
4533 &hdr->b_l1hdr.b_arc_node)); in arc_access()
4535 arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH); in arc_access()
4538 hdr->b_l1hdr.b_arc_access = now; in arc_access()
4547 if (now > hdr->b_l1hdr.b_arc_access + ARC_MINTIME) { in arc_access()
4553 hdr->b_l1hdr.b_arc_access = now; in arc_access()
4554 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); in arc_access()
4555 arc_change_state(arc_mfu, hdr, hash_lock); in arc_access()
4558 } else if (hdr->b_l1hdr.b_state == arc_mru_ghost) { in arc_access()
4566 if (HDR_PREFETCH(hdr)) { in arc_access()
4568 if (refcount_count(&hdr->b_l1hdr.b_refcnt) > 0) in arc_access()
4569 arc_hdr_clear_flags(hdr, ARC_FLAG_PREFETCH); in arc_access()
4570 DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr); in arc_access()
4573 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); in arc_access()
4576 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); in arc_access()
4577 arc_change_state(new_state, hdr, hash_lock); in arc_access()
4580 } else if (hdr->b_l1hdr.b_state == arc_mfu) { in arc_access()
4590 if ((HDR_PREFETCH(hdr)) != 0) { in arc_access()
4591 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_access()
4593 ASSERT(multilist_link_active(&hdr->b_l1hdr.b_arc_node)); in arc_access()
4596 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); in arc_access()
4597 } else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) { in arc_access()
4605 if (HDR_PREFETCH(hdr)) { in arc_access()
4610 ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt)); in arc_access()
4614 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); in arc_access()
4615 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); in arc_access()
4616 arc_change_state(new_state, hdr, hash_lock); in arc_access()
4619 } else if (hdr->b_l1hdr.b_state == arc_l2c_only) { in arc_access()
4624 hdr->b_l1hdr.b_arc_access = ddi_get_lbolt(); in arc_access()
4625 DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr); in arc_access()
4626 arc_change_state(arc_mfu, hdr, hash_lock); in arc_access()
4657 arc_hdr_verify(arc_buf_hdr_t *hdr, blkptr_t *bp) in arc_hdr_verify() argument
4660 ASSERT3U(HDR_GET_PSIZE(hdr), ==, 0); in arc_hdr_verify()
4661 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, ZIO_COMPRESS_OFF); in arc_hdr_verify()
4663 if (HDR_COMPRESSION_ENABLED(hdr)) { in arc_hdr_verify()
4664 ASSERT3U(HDR_GET_COMPRESS(hdr), ==, in arc_hdr_verify()
4667 ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(bp)); in arc_hdr_verify()
4668 ASSERT3U(HDR_GET_PSIZE(hdr), ==, BP_GET_PSIZE(bp)); in arc_hdr_verify()
4675 arc_buf_hdr_t *hdr = zio->io_private; in arc_read_done() local
4689 if (HDR_IN_HASH_TABLE(hdr)) { in arc_read_done()
4690 ASSERT3U(hdr->b_birth, ==, BP_PHYSICAL_BIRTH(zio->io_bp)); in arc_read_done()
4691 ASSERT3U(hdr->b_dva.dva_word[0], ==, in arc_read_done()
4693 ASSERT3U(hdr->b_dva.dva_word[1], ==, in arc_read_done()
4696 arc_buf_hdr_t *found = buf_hash_find(hdr->b_spa, zio->io_bp, in arc_read_done()
4699 ASSERT((found == hdr && in arc_read_done()
4700 DVA_EQUAL(&hdr->b_dva, BP_IDENTITY(zio->io_bp))) || in arc_read_done()
4701 (found == hdr && HDR_L2_READING(hdr))); in arc_read_done()
4709 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_UINT64; in arc_read_done()
4711 hdr->b_l1hdr.b_byteswap = in arc_read_done()
4715 hdr->b_l1hdr.b_byteswap = DMU_BSWAP_NUMFUNCS; in arc_read_done()
4719 arc_hdr_clear_flags(hdr, ARC_FLAG_L2_EVICTED); in arc_read_done()
4720 if (l2arc_noprefetch && HDR_PREFETCH(hdr)) in arc_read_done()
4721 arc_hdr_clear_flags(hdr, ARC_FLAG_L2CACHE); in arc_read_done()
4723 callback_list = hdr->b_l1hdr.b_acb; in arc_read_done()
4727 hdr->b_l1hdr.b_state == arc_anon) { in arc_read_done()
4734 arc_access(hdr, hash_lock); in arc_read_done()
4750 acb->acb_buf = arc_buf_alloc_impl(hdr, in arc_read_done()
4758 add_reference(hdr, acb->acb_private); in arc_read_done()
4763 hdr->b_l1hdr.b_acb = NULL; in arc_read_done()
4764 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); in arc_read_done()
4770 ASSERT(HDR_PREFETCH(hdr)); in arc_read_done()
4771 ASSERT0(hdr->b_l1hdr.b_bufcnt); in arc_read_done()
4772 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL); in arc_read_done()
4775 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt) || in arc_read_done()
4779 arc_hdr_verify(hdr, zio->io_bp); in arc_read_done()
4781 arc_hdr_set_flags(hdr, ARC_FLAG_IO_ERROR); in arc_read_done()
4782 if (hdr->b_l1hdr.b_state != arc_anon) in arc_read_done()
4783 arc_change_state(arc_anon, hdr, hash_lock); in arc_read_done()
4784 if (HDR_IN_HASH_TABLE(hdr)) in arc_read_done()
4785 buf_hash_remove(hdr); in arc_read_done()
4786 freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt); in arc_read_done()
4794 cv_broadcast(&hdr->b_l1hdr.b_cv); in arc_read_done()
4805 ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon); in arc_read_done()
4806 freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt); in arc_read_done()
4824 arc_hdr_destroy(hdr); in arc_read_done()
4850 arc_buf_hdr_t *hdr = NULL; in arc_read() local
4864 hdr = buf_hash_find(guid, bp, &hash_lock); in arc_read()
4867 if (hdr != NULL && HDR_HAS_L1HDR(hdr) && hdr->b_l1hdr.b_pdata != NULL) { in arc_read()
4871 if (HDR_IO_IN_PROGRESS(hdr)) { in arc_read()
4873 if ((hdr->b_flags & ARC_FLAG_PRIO_ASYNC_READ) && in arc_read()
4896 arc_buf_hdr_t *, hdr); in arc_read()
4899 if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) { in arc_read()
4900 arc_hdr_clear_flags(hdr, in arc_read()
4905 cv_wait(&hdr->b_l1hdr.b_cv, hash_lock); in arc_read()
4923 acb->acb_next = hdr->b_l1hdr.b_acb; in arc_read()
4924 hdr->b_l1hdr.b_acb = acb; in arc_read()
4932 ASSERT(hdr->b_l1hdr.b_state == arc_mru || in arc_read()
4933 hdr->b_l1hdr.b_state == arc_mfu); in arc_read()
4936 if (hdr->b_flags & ARC_FLAG_PREDICTIVE_PREFETCH) { in arc_read()
4944 arc_buf_hdr_t *, hdr); in arc_read()
4947 arc_hdr_clear_flags(hdr, in arc_read()
4957 buf = hdr->b_l1hdr.b_buf; in arc_read()
4959 ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt)); in arc_read()
4960 ASSERT3P(hdr->b_l1hdr.b_freeze_cksum, ==, NULL); in arc_read()
4961 buf = arc_buf_alloc_impl(hdr, private); in arc_read()
4964 add_reference(hdr, private); in arc_read()
4970 refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) { in arc_read()
4971 arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH); in arc_read()
4973 DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr); in arc_read()
4974 arc_access(hdr, hash_lock); in arc_read()
4976 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE); in arc_read()
4979 ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr), in arc_read()
4980 demand, prefetch, !HDR_ISTYPE_METADATA(hdr), in arc_read()
4994 if (hdr == NULL) { in arc_read()
4998 hdr = arc_hdr_alloc(spa_load_guid(spa), psize, lsize, in arc_read()
5002 hdr->b_dva = *BP_IDENTITY(bp); in arc_read()
5003 hdr->b_birth = BP_PHYSICAL_BIRTH(bp); in arc_read()
5004 exists = buf_hash_insert(hdr, &hash_lock); in arc_read()
5009 buf_discard_identity(hdr); in arc_read()
5010 arc_hdr_destroy(hdr); in arc_read()
5019 if (!HDR_HAS_L1HDR(hdr)) { in arc_read()
5020 hdr = arc_hdr_realloc(hdr, hdr_l2only_cache, in arc_read()
5023 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL); in arc_read()
5024 ASSERT(GHOST_STATE(hdr->b_l1hdr.b_state)); in arc_read()
5025 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_read()
5026 ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_read()
5027 ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL); in arc_read()
5039 arc_access(hdr, hash_lock); in arc_read()
5040 arc_hdr_alloc_pdata(hdr); in arc_read()
5042 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL); in arc_read()
5043 size = arc_hdr_size(hdr); in arc_read()
5051 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF) { in arc_read()
5056 arc_hdr_set_flags(hdr, ARC_FLAG_PREFETCH); in arc_read()
5058 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE); in arc_read()
5060 arc_hdr_set_flags(hdr, ARC_FLAG_INDIRECT); in arc_read()
5062 arc_hdr_set_flags(hdr, ARC_FLAG_PREDICTIVE_PREFETCH); in arc_read()
5063 ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state)); in arc_read()
5069 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL); in arc_read()
5070 hdr->b_l1hdr.b_acb = acb; in arc_read()
5071 arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); in arc_read()
5073 if (HDR_HAS_L2HDR(hdr) && in arc_read()
5074 (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) { in arc_read()
5075 devw = hdr->b_l2hdr.b_dev->l2ad_writing; in arc_read()
5076 addr = hdr->b_l2hdr.b_daddr; in arc_read()
5086 arc_hdr_set_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ); in arc_read()
5088 arc_hdr_clear_flags(hdr, ARC_FLAG_PRIO_ASYNC_READ); in arc_read()
5097 ASSERT3U(HDR_GET_LSIZE(hdr), ==, lsize); in arc_read()
5099 DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp, in arc_read()
5102 ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr), in arc_read()
5103 demand, prefetch, !HDR_ISTYPE_METADATA(hdr), in arc_read()
5129 if (HDR_HAS_L2HDR(hdr) && in arc_read()
5130 !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) && in arc_read()
5131 !(l2arc_noprefetch && HDR_PREFETCH(hdr))) { in arc_read()
5135 DTRACE_PROBE1(l2arc__hit, arc_buf_hdr_t *, hdr); in arc_read()
5140 cb->l2rcb_hdr = hdr; in arc_read()
5149 b_data = hdr->b_l1hdr.b_pdata; in arc_read()
5162 ASSERT3U(HDR_GET_COMPRESS(hdr), !=, in arc_read()
5188 arc_buf_hdr_t *, hdr); in arc_read()
5190 if (HDR_L2_WRITING(hdr)) in arc_read()
5199 arc_buf_hdr_t *, hdr); in arc_read()
5204 rzio = zio_read(pio, spa, bp, hdr->b_l1hdr.b_pdata, size, in arc_read()
5205 arc_read_done, hdr, priority, zio_flags, zb); in arc_read()
5222 arc_buf_hdr_t *hdr; in arc_freed() local
5228 hdr = buf_hash_find(guid, bp, &hash_lock); in arc_freed()
5229 if (hdr == NULL) in arc_freed()
5254 if (!HDR_HAS_L1HDR(hdr) || (!HDR_IO_IN_PROGRESS(hdr) && in arc_freed()
5255 refcount_is_zero(&hdr->b_l1hdr.b_refcnt))) { in arc_freed()
5256 arc_change_state(arc_anon, hdr, hash_lock); in arc_freed()
5257 arc_hdr_destroy(hdr); in arc_freed()
5274 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_release() local
5284 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_release()
5291 if (hdr->b_l1hdr.b_state == arc_anon) { in arc_release()
5293 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_release()
5294 ASSERT(!HDR_IN_HASH_TABLE(hdr)); in arc_release()
5295 ASSERT(!HDR_HAS_L2HDR(hdr)); in arc_release()
5296 ASSERT(HDR_EMPTY(hdr)); in arc_release()
5297 ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1); in arc_release()
5298 ASSERT3S(refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1); in arc_release()
5299 ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node)); in arc_release()
5301 hdr->b_l1hdr.b_arc_access = 0; in arc_release()
5307 buf_discard_identity(hdr); in arc_release()
5313 kmutex_t *hash_lock = HDR_LOCK(hdr); in arc_release()
5321 arc_state_t *state = hdr->b_l1hdr.b_state; in arc_release()
5322 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); in arc_release()
5326 ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) > 0); in arc_release()
5328 if (HDR_HAS_L2HDR(hdr)) { in arc_release()
5329 mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx); in arc_release()
5339 if (HDR_HAS_L2HDR(hdr)) { in arc_release()
5340 l2arc_trim(hdr); in arc_release()
5341 arc_hdr_l2hdr_destroy(hdr); in arc_release()
5344 mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx); in arc_release()
5350 if (hdr->b_l1hdr.b_bufcnt > 1) { in arc_release()
5353 uint64_t spa = hdr->b_spa; in arc_release()
5354 uint64_t psize = HDR_GET_PSIZE(hdr); in arc_release()
5355 uint64_t lsize = HDR_GET_LSIZE(hdr); in arc_release()
5356 enum zio_compress compress = HDR_GET_COMPRESS(hdr); in arc_release()
5357 arc_buf_contents_t type = arc_buf_type(hdr); in arc_release()
5358 VERIFY3U(hdr->b_type, ==, type); in arc_release()
5360 ASSERT(hdr->b_l1hdr.b_buf != buf || buf->b_next != NULL); in arc_release()
5361 (void) remove_reference(hdr, hash_lock, tag); in arc_release()
5364 ASSERT(HDR_SHARED_DATA(hdr)); in arc_release()
5365 ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf); in arc_release()
5375 bufp = &hdr->b_l1hdr.b_buf; in arc_release()
5402 ASSERT3P(hdr->b_l1hdr.b_buf, !=, buf); in arc_release()
5412 arc_unshare_buf(hdr, buf); in arc_release()
5413 arc_share_buf(hdr, lastbuf); in arc_release()
5415 } else if (HDR_SHARED_DATA(hdr)) { in arc_release()
5418 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL); in arc_release()
5422 HDR_GET_LSIZE(hdr), buf); in arc_release()
5424 if (refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) { in arc_release()
5427 HDR_GET_LSIZE(hdr), buf); in arc_release()
5430 hdr->b_l1hdr.b_bufcnt -= 1; in arc_release()
5459 ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) == 1); in arc_release()
5461 ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node)); in arc_release()
5462 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_release()
5463 arc_change_state(arc_anon, hdr, hash_lock); in arc_release()
5464 hdr->b_l1hdr.b_arc_access = 0; in arc_release()
5467 buf_discard_identity(hdr); in arc_release()
5502 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_write_ready() local
5505 ASSERT(HDR_HAS_L1HDR(hdr)); in arc_write_ready()
5507 ASSERT(hdr->b_l1hdr.b_bufcnt > 0); in arc_write_ready()
5515 arc_cksum_free(hdr); in arc_write_ready()
5519 if (hdr->b_l1hdr.b_pdata != NULL) { in arc_write_ready()
5521 ASSERT(HDR_SHARED_DATA(hdr)); in arc_write_ready()
5523 arc_unshare_buf(hdr, buf); in arc_write_ready()
5525 arc_hdr_free_pdata(hdr); in arc_write_ready()
5529 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL); in arc_write_ready()
5530 ASSERT(!HDR_SHARED_DATA(hdr)); in arc_write_ready()
5535 if (HDR_IO_IN_PROGRESS(hdr)) in arc_write_ready()
5539 arc_hdr_set_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); in arc_write_ready()
5545 ASSERT3U(HDR_GET_LSIZE(hdr), ==, BP_GET_LSIZE(zio->io_bp)); in arc_write_ready()
5548 HDR_SET_PSIZE(hdr, psize); in arc_write_ready()
5549 arc_hdr_set_compress(hdr, compress); in arc_write_ready()
5559 if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF) { in arc_write_ready()
5562 arc_hdr_alloc_pdata(hdr); in arc_write_ready()
5563 bcopy(zio->io_data, hdr->b_l1hdr.b_pdata, psize); in arc_write_ready()
5566 ASSERT3U(zio->io_orig_size, ==, HDR_GET_LSIZE(hdr)); in arc_write_ready()
5567 ASSERT3U(hdr->b_l1hdr.b_byteswap, ==, DMU_BSWAP_NUMFUNCS); in arc_write_ready()
5568 ASSERT(!HDR_SHARED_DATA(hdr)); in arc_write_ready()
5570 ASSERT3U(hdr->b_l1hdr.b_bufcnt, ==, 1); in arc_write_ready()
5571 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL); in arc_write_ready()
5577 arc_share_buf(hdr, buf); in arc_write_ready()
5578 VERIFY0(bcmp(zio->io_orig_data, hdr->b_l1hdr.b_pdata, in arc_write_ready()
5579 HDR_GET_LSIZE(hdr))); in arc_write_ready()
5581 arc_hdr_verify(hdr, zio->io_bp); in arc_write_ready()
5610 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_write_done() local
5612 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL); in arc_write_done()
5615 arc_hdr_verify(hdr, zio->io_bp); in arc_write_done()
5618 buf_discard_identity(hdr); in arc_write_done()
5620 hdr->b_dva = *BP_IDENTITY(zio->io_bp); in arc_write_done()
5621 hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp); in arc_write_done()
5624 ASSERT(HDR_EMPTY(hdr)); in arc_write_done()
5633 if (!HDR_EMPTY(hdr)) { in arc_write_done()
5641 exists = buf_hash_insert(hdr, &hash_lock); in arc_write_done()
5651 (void *)hdr, (void *)exists); in arc_write_done()
5657 exists = buf_hash_insert(hdr, &hash_lock); in arc_write_done()
5664 (void *)hdr, (void *)exists); in arc_write_done()
5667 ASSERT(hdr->b_l1hdr.b_bufcnt == 1); in arc_write_done()
5668 ASSERT(hdr->b_l1hdr.b_state == arc_anon); in arc_write_done()
5673 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); in arc_write_done()
5675 if (exists == NULL && hdr->b_l1hdr.b_state == arc_anon) in arc_write_done()
5676 arc_access(hdr, hash_lock); in arc_write_done()
5679 arc_hdr_clear_flags(hdr, ARC_FLAG_IO_IN_PROGRESS); in arc_write_done()
5682 ASSERT(!refcount_is_zero(&hdr->b_l1hdr.b_refcnt)); in arc_write_done()
5695 arc_buf_hdr_t *hdr = buf->b_hdr; in arc_write() local
5701 ASSERT(!HDR_IO_ERROR(hdr)); in arc_write()
5702 ASSERT(!HDR_IO_IN_PROGRESS(hdr)); in arc_write()
5703 ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL); in arc_write()
5704 ASSERT3U(hdr->b_l1hdr.b_bufcnt, >, 0); in arc_write()
5706 arc_hdr_set_flags(hdr, ARC_FLAG_L2CACHE); in arc_write()
5719 if (hdr->b_l1hdr.b_pdata != NULL) { in arc_write()
5728 arc_unshare_buf(hdr, buf); in arc_write()
5730 arc_hdr_free_pdata(hdr); in arc_write()
5733 arc_hdr_set_compress(hdr, ZIO_COMPRESS_OFF); in arc_write()
5736 ASSERT3P(hdr->b_l1hdr.b_pdata, ==, NULL); in arc_write()
5738 zio = zio_write(pio, spa, txg, bp, buf->b_data, HDR_GET_LSIZE(hdr), zp, in arc_write()
5903 arc_buf_hdr_t *hdr = obj; in arc_state_multilist_index_func() local
5910 ASSERT(!HDR_EMPTY(hdr)); in arc_state_multilist_index_func()
5924 return (buf_hash(hdr->b_spa, &hdr->b_dva, hdr->b_birth) % in arc_state_multilist_index_func()
6469 l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *hdr) in l2arc_write_eligible() argument
6478 if (hdr->b_spa != spa_guid) { in l2arc_write_eligible()
6482 if (HDR_HAS_L2HDR(hdr)) { in l2arc_write_eligible()
6486 if (HDR_IO_IN_PROGRESS(hdr)) { in l2arc_write_eligible()
6490 if (!HDR_L2CACHE(hdr)) { in l2arc_write_eligible()
6643 arc_buf_hdr_t *head, *hdr, *hdr_prev; in l2arc_write_done() local
6666 for (hdr = list_prev(buflist, head); hdr; hdr = hdr_prev) { in l2arc_write_done()
6667 hdr_prev = list_prev(buflist, hdr); in l2arc_write_done()
6669 hash_lock = HDR_LOCK(hdr); in l2arc_write_done()
6690 list_insert_after(buflist, hdr, head); in l2arc_write_done()
6710 ASSERT(HDR_HAS_L1HDR(hdr)); in l2arc_write_done()
6716 list_remove(buflist, hdr); in l2arc_write_done()
6717 l2arc_trim(hdr); in l2arc_write_done()
6718 arc_hdr_clear_flags(hdr, ARC_FLAG_HAS_L2HDR); in l2arc_write_done()
6720 ARCSTAT_INCR(arcstat_l2_asize, -arc_hdr_size(hdr)); in l2arc_write_done()
6721 ARCSTAT_INCR(arcstat_l2_size, -HDR_GET_LSIZE(hdr)); in l2arc_write_done()
6723 bytes_dropped += arc_hdr_size(hdr); in l2arc_write_done()
6725 arc_hdr_size(hdr), hdr); in l2arc_write_done()
6732 arc_hdr_clear_flags(hdr, ARC_FLAG_L2_WRITING); in l2arc_write_done()
6758 arc_buf_hdr_t *hdr; in l2arc_read_done() local
6769 hdr = cb->l2rcb_hdr; in l2arc_read_done()
6770 ASSERT3P(hdr, !=, NULL); in l2arc_read_done()
6772 hash_lock = HDR_LOCK(hdr); in l2arc_read_done()
6774 ASSERT3P(hash_lock, ==, HDR_LOCK(hdr)); in l2arc_read_done()
6781 ASSERT3U(arc_hdr_size(hdr), <, zio->io_size); in l2arc_read_done()
6783 bcopy(cb->l2rcb_data, hdr->b_l1hdr.b_pdata, in l2arc_read_done()
6784 arc_hdr_size(hdr)); in l2arc_read_done()
6799 zio->io_size = zio->io_orig_size = arc_hdr_size(hdr); in l2arc_read_done()
6800 zio->io_data = zio->io_orig_data = hdr->b_l1hdr.b_pdata; in l2arc_read_done()
6808 ASSERT3P(zio->io_data, ==, hdr->b_l1hdr.b_pdata); in l2arc_read_done()
6812 valid_cksum = arc_cksum_is_equal(hdr, zio); in l2arc_read_done()
6813 if (valid_cksum && zio->io_error == 0 && !HDR_L2_EVICTED(hdr)) { in l2arc_read_done()
6815 zio->io_private = hdr; in l2arc_read_done()
6842 hdr->b_l1hdr.b_pdata, zio->io_size, arc_read_done, in l2arc_read_done()
6843 hdr, zio->io_priority, cb->l2rcb_flags, in l2arc_read_done()
6904 arc_buf_hdr_t *hdr, *hdr_prev; in l2arc_evict() local
6932 for (hdr = list_tail(buflist); hdr; hdr = hdr_prev) { in l2arc_evict()
6933 hdr_prev = list_prev(buflist, hdr); in l2arc_evict()
6935 hash_lock = HDR_LOCK(hdr); in l2arc_evict()
6953 if (HDR_L2_WRITE_HEAD(hdr)) { in l2arc_evict()
6958 list_remove(buflist, hdr); in l2arc_evict()
6963 if (!all && HDR_HAS_L2HDR(hdr) && in l2arc_evict()
6964 (hdr->b_l2hdr.b_daddr >= taddr || in l2arc_evict()
6965 hdr->b_l2hdr.b_daddr < dev->l2ad_hand)) { in l2arc_evict()
6974 ASSERT(HDR_HAS_L2HDR(hdr)); in l2arc_evict()
6975 if (!HDR_HAS_L1HDR(hdr)) { in l2arc_evict()
6976 ASSERT(!HDR_L2_READING(hdr)); in l2arc_evict()
6982 arc_change_state(arc_anon, hdr, hash_lock); in l2arc_evict()
6983 arc_hdr_destroy(hdr); in l2arc_evict()
6985 ASSERT(hdr->b_l1hdr.b_state != arc_l2c_only); in l2arc_evict()
6992 if (HDR_L2_READING(hdr)) { in l2arc_evict()
6994 arc_hdr_set_flags(hdr, ARC_FLAG_L2_EVICTED); in l2arc_evict()
6998 ASSERT(!HDR_L2_WRITING(hdr)); in l2arc_evict()
7000 arc_hdr_l2hdr_destroy(hdr); in l2arc_evict()
7021 arc_buf_hdr_t *hdr, *hdr_prev, *head; in l2arc_write_buffers() local
7054 hdr = multilist_sublist_head(mls); in l2arc_write_buffers()
7056 hdr = multilist_sublist_tail(mls); in l2arc_write_buffers()
7057 if (hdr == NULL) in l2arc_write_buffers()
7064 for (; hdr; hdr = hdr_prev) { in l2arc_write_buffers()
7068 hdr_prev = multilist_sublist_next(mls, hdr); in l2arc_write_buffers()
7070 hdr_prev = multilist_sublist_prev(mls, hdr); in l2arc_write_buffers()
7072 HDR_GET_LSIZE(hdr)); in l2arc_write_buffers()
7074 hash_lock = HDR_LOCK(hdr); in l2arc_write_buffers()
7083 passed_sz += HDR_GET_LSIZE(hdr); in l2arc_write_buffers()
7093 if (!l2arc_write_eligible(guid, hdr)) { in l2arc_write_buffers()
7104 ASSERT(HDR_HAS_L1HDR(hdr)); in l2arc_write_buffers()
7106 ASSERT3U(HDR_GET_PSIZE(hdr), >, 0); in l2arc_write_buffers()
7107 ASSERT3P(hdr->b_l1hdr.b_pdata, !=, NULL); in l2arc_write_buffers()
7108 ASSERT3U(arc_hdr_size(hdr), >, 0); in l2arc_write_buffers()
7109 uint64_t size = arc_hdr_size(hdr); in l2arc_write_buffers()
7139 hdr->b_l2hdr.b_dev = dev; in l2arc_write_buffers()
7140 hdr->b_l2hdr.b_daddr = dev->l2ad_hand; in l2arc_write_buffers()
7141 arc_hdr_set_flags(hdr, in l2arc_write_buffers()
7145 list_insert_head(&dev->l2ad_buflist, hdr); in l2arc_write_buffers()
7148 (void) refcount_add_many(&dev->l2ad_alloc, size, hdr); in l2arc_write_buffers()
7160 if (!HDR_SHARED_DATA(hdr) && size == asize) { in l2arc_write_buffers()
7161 to_write = hdr->b_l1hdr.b_pdata; in l2arc_write_buffers()
7163 arc_buf_contents_t type = arc_buf_type(hdr); in l2arc_write_buffers()
7171 bcopy(hdr->b_l1hdr.b_pdata, to_write, size); in l2arc_write_buffers()
7177 hdr->b_l2hdr.b_daddr, asize, to_write, in l2arc_write_buffers()
7178 ZIO_CHECKSUM_OFF, NULL, hdr, in l2arc_write_buffers()
7182 write_sz += HDR_GET_LSIZE(hdr); in l2arc_write_buffers()