Lines Matching defs:db
292 dmu_buf_impl_t *db = vdb;
293 memset(db, 0, sizeof (dmu_buf_impl_t));
295 mutex_init(&db->db_mtx, NULL, MUTEX_NOLOCKDEP, NULL);
296 rw_init(&db->db_rwlock, NULL, RW_NOLOCKDEP, NULL);
297 cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL);
298 multilist_link_init(&db->db_cache_link);
299 zfs_refcount_create(&db->db_holds);
308 dmu_buf_impl_t *db = vdb;
309 mutex_destroy(&db->db_mtx);
310 rw_destroy(&db->db_rwlock);
311 cv_destroy(&db->db_changed);
312 ASSERT(!multilist_link_active(&db->db_cache_link));
313 zfs_refcount_destroy(&db->db_holds);
331 #define DTRACE_SET_STATE(db, why) \
332 DTRACE_PROBE2(dbuf__state_change, dmu_buf_impl_t *, db, \
336 ((dbuf)->db.db_object == (obj) && \
348 dmu_buf_impl_t *db;
354 for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) {
355 if (DBUF_EQUAL(db, os, obj, level, blkid)) {
356 mutex_enter(&db->db_mtx);
357 if (db->db_state != DB_EVICTING) {
359 return (db);
361 mutex_exit(&db->db_mtx);
374 dmu_buf_impl_t *db = NULL;
379 db = dn->dn_bonus;
380 mutex_enter(&db->db_mtx);
385 return (db);
395 dbuf_hash_insert(dmu_buf_impl_t *db)
398 objset_t *os = db->db_objset;
399 uint64_t obj = db->db.db_object;
400 int level = db->db_level;
405 blkid = db->db_blkid;
406 ASSERT3U(dbuf_hash(os, obj, level, blkid), ==, db->db_hash);
407 idx = db->db_hash & h->hash_table_mask;
430 mutex_enter(&db->db_mtx);
431 db->db_hash_next = h->hash_table[idx];
432 h->hash_table[idx] = db;
445 dbuf_include_in_metadata_cache(dmu_buf_impl_t *db)
447 DB_DNODE_ENTER(db);
448 dmu_object_type_t type = DB_DNODE(db)->dn_type;
449 DB_DNODE_EXIT(db);
477 dbuf_hash_remove(dmu_buf_impl_t *db)
483 ASSERT3U(dbuf_hash(db->db_objset, db->db.db_object, db->db_level,
484 db->db_blkid), ==, db->db_hash);
485 idx = db->db_hash & h->hash_table_mask;
491 ASSERT(zfs_refcount_is_zero(&db->db_holds));
492 ASSERT(db->db_state == DB_EVICTING);
493 ASSERT(!MUTEX_HELD(&db->db_mtx));
497 while ((dbf = *dbp) != db) {
501 *dbp = db->db_hash_next;
502 db->db_hash_next = NULL;
516 dbuf_verify_user(dmu_buf_impl_t *db, dbvu_verify_type_t verify_type)
521 if (db->db_user == NULL)
525 ASSERT(db->db_level == 0);
528 ASSERT(db->db.db_data != NULL);
529 ASSERT3U(db->db_state, ==, DB_CACHED);
531 holds = zfs_refcount_count(&db->db_holds);
542 ASSERT3U(holds, >=, db->db_dirtycnt);
544 if (db->db_user_immediate_evict == TRUE)
545 ASSERT3U(holds, >=, db->db_dirtycnt);
553 dbuf_evict_user(dmu_buf_impl_t *db)
555 dmu_buf_user_t *dbu = db->db_user;
557 ASSERT(MUTEX_HELD(&db->db_mtx));
562 dbuf_verify_user(db, DBVU_EVICTING);
563 db->db_user = NULL;
570 if (db->db_caching_status != DB_NO_CACHE) {
580 &dbuf_caches[db->db_caching_status].size, size, dbu);
581 if (db->db_caching_status == DB_DBUF_CACHE)
582 DBUF_STAT_DECR(cache_levels_bytes[db->db_level], size);
607 dbuf_is_metadata(dmu_buf_impl_t *db)
612 if (db->db_level > 0 || db->db_blkid == DMU_SPILL_BLKID) {
617 DB_DNODE_ENTER(db);
618 is_metadata = DMU_OT_IS_METADATA(DB_DNODE(db)->dn_type);
619 DB_DNODE_EXIT(db);
630 dbuf_is_l2cacheable(dmu_buf_impl_t *db, blkptr_t *bp)
632 if (db->db_objset->os_secondary_cache == ZFS_CACHE_ALL ||
633 (db->db_objset->os_secondary_cache ==
634 ZFS_CACHE_METADATA && dbuf_is_metadata(db))) {
644 blkptr_t *db_bp = bp == NULL ? db->db_blkptr : bp;
649 vdev_t *rvd = db->db_objset->os_spa->spa_root_vdev;
705 dmu_buf_impl_t *db = obj;
720 return ((unsigned int)dbuf_hash(db->db_objset, db->db.db_object,
721 db->db_level, db->db_blkid) %
782 dmu_buf_impl_t *db = multilist_sublist_tail(mls);
783 while (db != NULL && mutex_tryenter(&db->db_mtx) == 0) {
784 db = multilist_sublist_prev(mls, db);
787 DTRACE_PROBE2(dbuf__evict__one, dmu_buf_impl_t *, db,
790 if (db != NULL) {
791 multilist_sublist_remove(mls, db);
793 uint64_t size = db->db.db_size;
794 uint64_t usize = dmu_buf_user_size(&db->db);
796 &dbuf_caches[DB_DBUF_CACHE].size, size, db);
798 &dbuf_caches[DB_DBUF_CACHE].size, usize, db->db_user);
799 DBUF_STAT_BUMPDOWN(cache_levels[db->db_level]);
801 DBUF_STAT_DECR(cache_levels_bytes[db->db_level], size + usize);
802 ASSERT3U(db->db_caching_status, ==, DB_DBUF_CACHE);
803 db->db_caching_status = DB_NO_CACHE;
804 dbuf_destroy(db);
1094 dbuf_verify(dmu_buf_impl_t *db)
1100 ASSERT(MUTEX_HELD(&db->db_mtx));
1105 ASSERT(db->db_objset != NULL);
1106 DB_DNODE_ENTER(db);
1107 dn = DB_DNODE(db);
1109 ASSERT(db->db_parent == NULL);
1110 ASSERT(db->db_blkptr == NULL);
1112 ASSERT3U(db->db.db_object, ==, dn->dn_object);
1113 ASSERT3P(db->db_objset, ==, dn->dn_objset);
1114 ASSERT3U(db->db_level, <, dn->dn_nlevels);
1115 ASSERT(db->db_blkid == DMU_BONUS_BLKID ||
1116 db->db_blkid == DMU_SPILL_BLKID ||
1119 if (db->db_blkid == DMU_BONUS_BLKID) {
1121 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
1122 ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID);
1123 } else if (db->db_blkid == DMU_SPILL_BLKID) {
1125 ASSERT0(db->db.db_offset);
1127 ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size);
1130 if ((dr = list_head(&db->db_dirty_records)) != NULL) {
1131 ASSERT(dr->dr_dbuf == db);
1133 for (dr = list_next(&db->db_dirty_records, dr); dr != NULL;
1134 dr = list_next(&db->db_dirty_records, dr)) {
1135 ASSERT(dr->dr_dbuf == db);
1146 if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) {
1147 dr = db->db_data_pending;
1152 ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf);
1155 /* verify db->db_blkptr */
1156 if (db->db_blkptr) {
1157 if (db->db_parent == dn->dn_dbuf) {
1158 /* db is pointed to by the dnode */
1159 /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */
1160 if (DMU_OBJECT_IS_SPECIAL(db->db.db_object))
1161 ASSERT(db->db_parent == NULL);
1163 ASSERT(db->db_parent != NULL);
1164 if (db->db_blkid != DMU_SPILL_BLKID)
1165 ASSERT3P(db->db_blkptr, ==,
1166 &dn->dn_phys->dn_blkptr[db->db_blkid]);
1168 /* db is pointed to by an indirect block */
1169 int epb __maybe_unused = db->db_parent->db.db_size >>
1171 ASSERT3U(db->db_parent->db_level, ==, db->db_level+1);
1172 ASSERT3U(db->db_parent->db.db_object, ==,
1173 db->db.db_object);
1179 if (RW_LOCK_HELD(&db->db_parent->db_rwlock)) {
1180 ASSERT3P(db->db_blkptr, ==,
1181 ((blkptr_t *)db->db_parent->db.db_data +
1182 db->db_blkid % epb));
1186 if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) &&
1187 (db->db_buf == NULL || db->db_buf->b_data) &&
1188 db->db.db_data && db->db_blkid != DMU_BONUS_BLKID &&
1189 db->db_state != DB_FILL && (dn == NULL || !dn->dn_free_txg)) {
1201 if (db->db_dirtycnt == 0) {
1202 if (db->db_level == 0) {
1203 uint64_t *buf = db->db.db_data;
1206 for (i = 0; i < db->db.db_size >> 3; i++) {
1210 blkptr_t *bps = db->db.db_data;
1211 ASSERT3U(1 << DB_DNODE(db)->dn_indblkshift, ==,
1212 db->db.db_size);
1221 i < db->db.db_size / sizeof (blkptr_t);
1240 DB_DNODE_EXIT(db);
1245 dbuf_clear_data(dmu_buf_impl_t *db)
1247 ASSERT(MUTEX_HELD(&db->db_mtx));
1248 dbuf_evict_user(db);
1249 ASSERT3P(db->db_buf, ==, NULL);
1250 db->db.db_data = NULL;
1251 if (db->db_state != DB_NOFILL) {
1252 db->db_state = DB_UNCACHED;
1253 DTRACE_SET_STATE(db, "clear data");
1258 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf)
1260 ASSERT(MUTEX_HELD(&db->db_mtx));
1263 db->db_buf = buf;
1265 db->db.db_data = buf->b_data;
1269 dbuf_alloc_arcbuf(dmu_buf_impl_t *db)
1271 spa_t *spa = db->db_objset->os_spa;
1273 return (arc_alloc_buf(spa, db, DBUF_GET_BUFC_TYPE(db), db->db.db_size));
1280 dbuf_loan_arcbuf(dmu_buf_impl_t *db)
1284 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1285 mutex_enter(&db->db_mtx);
1286 if (arc_released(db->db_buf) || zfs_refcount_count(&db->db_holds) > 1) {
1287 int blksz = db->db.db_size;
1288 spa_t *spa = db->db_objset->os_spa;
1290 mutex_exit(&db->db_mtx);
1292 memcpy(abuf->b_data, db->db.db_data, blksz);
1294 abuf = db->db_buf;
1295 arc_loan_inuse_buf(abuf, db);
1296 db->db_buf = NULL;
1297 dbuf_clear_data(db);
1298 mutex_exit(&db->db_mtx);
1354 dmu_buf_lock_parent(dmu_buf_impl_t *db, krw_t rw, const void *tag)
1357 if (db->db_parent != NULL) {
1358 rw_enter(&db->db_parent->db_rwlock, rw);
1360 } else if (dmu_objset_ds(db->db_objset) != NULL) {
1361 rrw_enter(&dmu_objset_ds(db->db_objset)->ds_bp_rwlock, rw,
1379 dmu_buf_unlock_parent(dmu_buf_impl_t *db, db_lock_type_t type, const void *tag)
1382 rw_exit(&db->db_parent->db_rwlock);
1384 rrw_exit(&dmu_objset_ds(db->db_objset)->ds_bp_rwlock, tag);
1392 dmu_buf_impl_t *db = vdb;
1394 mutex_enter(&db->db_mtx);
1395 ASSERT3U(db->db_state, ==, DB_READ);
1400 ASSERT(zfs_refcount_count(&db->db_holds) > 0);
1401 ASSERT(db->db_buf == NULL);
1402 ASSERT(db->db.db_data == NULL);
1406 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1407 ASSERT3P(db->db_buf, ==, NULL);
1408 db->db_state = DB_UNCACHED;
1409 DTRACE_SET_STATE(db, "i/o error");
1410 } else if (db->db_level == 0 && db->db_freed_in_flight) {
1413 arc_release(buf, db);
1414 memset(buf->b_data, 0, db->db.db_size);
1416 db->db_freed_in_flight = FALSE;
1417 dbuf_set_data(db, buf);
1418 db->db_state = DB_CACHED;
1419 DTRACE_SET_STATE(db, "freed in flight");
1423 dbuf_set_data(db, buf);
1424 db->db_state = DB_CACHED;
1425 DTRACE_SET_STATE(db, "successful read");
1427 cv_broadcast(&db->db_changed);
1428 dbuf_rele_and_unlock(db, NULL, B_FALSE);
1437 dbuf_read_bonus(dmu_buf_impl_t *db, dnode_t *dn)
1443 ASSERT(MUTEX_HELD(&db->db_mtx));
1444 ASSERT(DB_DNODE_HELD(db));
1445 ASSERT3U(bonuslen, <=, db->db.db_size);
1446 db->db.db_data = kmem_alloc(max_bonuslen, KM_SLEEP);
1449 memset(db->db.db_data, 0, max_bonuslen);
1451 memcpy(db->db.db_data, DN_BONUS(dn->dn_phys), bonuslen);
1452 db->db_state = DB_CACHED;
1453 DTRACE_SET_STATE(db, "bonus buffer filled");
1458 dbuf_handle_indirect_hole(dmu_buf_impl_t *db, dnode_t *dn, blkptr_t *dbbp)
1460 blkptr_t *bps = db->db.db_data;
1482 dbuf_read_hole(dmu_buf_impl_t *db, dnode_t *dn, blkptr_t *bp)
1484 ASSERT(MUTEX_HELD(&db->db_mtx));
1493 if (!is_hole && db->db_level == 0)
1494 is_hole = dnode_block_freed(dn, db->db_blkid) || BP_IS_HOLE(bp);
1497 dbuf_set_data(db, dbuf_alloc_arcbuf(db));
1498 memset(db->db.db_data, 0, db->db.db_size);
1500 if (bp != NULL && db->db_level > 0 && BP_IS_HOLE(bp) &&
1502 dbuf_handle_indirect_hole(db, dn, bp);
1504 db->db_state = DB_CACHED;
1505 DTRACE_SET_STATE(db, "hole read satisfied");
1524 dbuf_read_verify_dnode_crypt(dmu_buf_impl_t *db, dnode_t *dn, uint32_t flags)
1526 objset_t *os = db->db_objset;
1570 if (err == EACCES && ((db->db_blkid != DMU_BONUS_BLKID &&
1572 (db->db_blkid == DMU_BONUS_BLKID &&
1586 dbuf_read_impl(dmu_buf_impl_t *db, dnode_t *dn, zio_t *zio, uint32_t flags,
1593 ASSERT(!zfs_refcount_is_zero(&db->db_holds));
1594 ASSERT(MUTEX_HELD(&db->db_mtx));
1595 ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL);
1596 ASSERT(db->db_buf == NULL);
1597 ASSERT(db->db_parent == NULL ||
1598 RW_LOCK_HELD(&db->db_parent->db_rwlock));
1600 if (db->db_blkid == DMU_BONUS_BLKID) {
1601 err = dbuf_read_bonus(db, dn);
1605 err = dbuf_read_hole(db, dn, bp);
1618 db->db_objset->os_dsl_dataset,
1624 SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset),
1625 db->db.db_object, db->db_level, db->db_blkid);
1631 if (db->db_objset->os_encrypted && !BP_USES_CRYPT(bp)) {
1632 spa_log_error(db->db_objset->os_spa, &zb,
1638 db->db_state = DB_READ;
1639 DTRACE_SET_STATE(db, "read issued");
1640 mutex_exit(&db->db_mtx);
1642 if (!DBUF_IS_CACHEABLE(db))
1644 else if (dbuf_is_l2cacheable(db, bp))
1647 dbuf_add_ref(db, NULL);
1663 dmu_buf_unlock_parent(db, dblt, tag);
1664 return (arc_read(zio, db->db_objset->os_spa, ©,
1665 dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, zio_flags,
1669 mutex_exit(&db->db_mtx);
1670 dmu_buf_unlock_parent(db, dblt, tag);
1688 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg)
1690 dbuf_dirty_record_t *dr = list_head(&db->db_dirty_records);
1692 ASSERT(MUTEX_HELD(&db->db_mtx));
1693 ASSERT(db->db.db_data != NULL);
1694 ASSERT(db->db_level == 0);
1695 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT);
1699 ((db->db_blkid == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf)))
1710 if (db->db_blkid == DMU_BONUS_BLKID) {
1711 dnode_t *dn = DB_DNODE(db);
1715 memcpy(dr->dt.dl.dr_data, db->db.db_data, bonuslen);
1716 } else if (zfs_refcount_count(&db->db_holds) > db->db_dirtycnt) {
1717 dnode_t *dn = DB_DNODE(db);
1718 int size = arc_buf_size(db->db_buf);
1719 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
1720 spa_t *spa = db->db_objset->os_spa;
1722 arc_get_compression(db->db_buf);
1723 uint8_t complevel = arc_get_complevel(db->db_buf);
1725 if (arc_is_encrypted(db->db_buf)) {
1731 arc_get_raw_params(db->db_buf, &byteorder, salt,
1733 dr->dt.dl.dr_data = arc_alloc_raw_buf(spa, db,
1735 mac, dn->dn_type, size, arc_buf_lsize(db->db_buf),
1739 dr->dt.dl.dr_data = arc_alloc_compressed_buf(spa, db,
1740 size, arc_buf_lsize(db->db_buf), compress_type,
1743 dr->dt.dl.dr_data = arc_alloc_buf(spa, db, type, size);
1745 memcpy(dr->dt.dl.dr_data->b_data, db->db.db_data, size);
1747 db->db_buf = NULL;
1748 dbuf_clear_data(db);
1753 dbuf_read(dmu_buf_impl_t *db, zio_t *pio, uint32_t flags)
1759 ASSERT(!zfs_refcount_is_zero(&db->db_holds));
1761 DB_DNODE_ENTER(db);
1762 dn = DB_DNODE(db);
1768 err = dbuf_read_verify_dnode_crypt(db, dn, flags);
1772 prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
1775 mutex_enter(&db->db_mtx);
1777 db->db_partial_read = B_TRUE;
1779 db->db_partial_read = B_FALSE;
1780 miss = (db->db_state != DB_CACHED);
1782 if (db->db_state == DB_READ || db->db_state == DB_FILL) {
1792 mutex_exit(&db->db_mtx);
1793 DB_DNODE_EXIT(db);
1797 ASSERT(db->db_state == DB_READ ||
1799 DTRACE_PROBE2(blocked__read, dmu_buf_impl_t *, db,
1801 cv_wait(&db->db_changed, &db->db_mtx);
1802 } while (db->db_state == DB_READ || db->db_state == DB_FILL);
1803 if (db->db_state == DB_UNCACHED) {
1805 mutex_exit(&db->db_mtx);
1806 DB_DNODE_EXIT(db);
1811 if (db->db_state == DB_CACHED) {
1819 if ((flags & DB_RF_NO_DECRYPT) == 0 && db->db_buf != NULL &&
1820 (arc_is_encrypted(db->db_buf) ||
1821 arc_is_unauthenticated(db->db_buf) ||
1822 arc_get_compression(db->db_buf) != ZIO_COMPRESS_OFF)) {
1826 SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset),
1827 db->db.db_object, db->db_level, db->db_blkid);
1828 dbuf_fix_old_data(db, spa_syncing_txg(spa));
1829 err = arc_untransform(db->db_buf, spa, &zb, B_FALSE);
1830 dbuf_set_data(db, db->db_buf);
1832 mutex_exit(&db->db_mtx);
1834 ASSERT(db->db_state == DB_UNCACHED ||
1835 db->db_state == DB_NOFILL);
1836 db_lock_type_t dblt = dmu_buf_lock_parent(db, RW_READER, FTAG);
1844 err = dmu_buf_get_bp_from_dbuf(db, &bp);
1847 if (pio == NULL && (db->db_state == DB_NOFILL ||
1856 dbuf_read_impl(db, dn, pio, flags, dblt, bp, FTAG);
1858 mutex_exit(&db->db_mtx);
1859 dmu_buf_unlock_parent(db, dblt, FTAG);
1862 miss = (db->db_state != DB_CACHED);
1866 dmu_zfetch(&dn->dn_zfetch, db->db_blkid, 1, B_TRUE, miss,
1869 DB_DNODE_EXIT(db);
1899 dbuf_noread(dmu_buf_impl_t *db)
1901 ASSERT(!zfs_refcount_is_zero(&db->db_holds));
1902 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
1903 mutex_enter(&db->db_mtx);
1904 while (db->db_state == DB_READ || db->db_state == DB_FILL)
1905 cv_wait(&db->db_changed, &db->db_mtx);
1906 if (db->db_state == DB_UNCACHED) {
1907 ASSERT(db->db_buf == NULL);
1908 ASSERT(db->db.db_data == NULL);
1909 dbuf_set_data(db, dbuf_alloc_arcbuf(db));
1910 db->db_state = DB_FILL;
1911 DTRACE_SET_STATE(db, "assigning filled buffer");
1912 } else if (db->db_state == DB_NOFILL) {
1913 dbuf_clear_data(db);
1915 ASSERT3U(db->db_state, ==, DB_CACHED);
1917 mutex_exit(&db->db_mtx);
1923 dmu_buf_impl_t *db = dr->dr_dbuf;
1927 ASSERT(MUTEX_HELD(&db->db_mtx));
1935 ASSERT(db->db_level == 0);
1937 if (db->db_blkid == DMU_BONUS_BLKID ||
1941 ASSERT(db->db_data_pending != dr);
1945 zio_free(db->db_objset->os_spa, txg, bp);
1949 dr->dt.dl.dr_data = db->db_buf;
1969 arc_release(dr->dt.dl.dr_data, db);
1982 dmu_buf_impl_t *db, *db_next;
1999 db = avl_find(&dn->dn_dbufs, db_search, &where);
2000 ASSERT3P(db, ==, NULL);
2002 db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER);
2004 for (; db != NULL; db = db_next) {
2005 db_next = AVL_NEXT(&dn->dn_dbufs, db);
2006 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
2008 if (db->db_level != 0 || db->db_blkid > end_blkid) {
2011 ASSERT3U(db->db_blkid, >=, start_blkid);
2014 mutex_enter(&db->db_mtx);
2015 if (dbuf_undirty(db, tx)) {
2020 if (db->db_state == DB_UNCACHED ||
2021 db->db_state == DB_NOFILL ||
2022 db->db_state == DB_EVICTING) {
2023 ASSERT(db->db.db_data == NULL);
2024 mutex_exit(&db->db_mtx);
2027 if (db->db_state == DB_READ || db->db_state == DB_FILL) {
2029 db->db_freed_in_flight = TRUE;
2030 mutex_exit(&db->db_mtx);
2033 if (zfs_refcount_count(&db->db_holds) == 0) {
2034 ASSERT(db->db_buf);
2035 dbuf_destroy(db);
2040 dr = list_head(&db->db_dirty_records);
2048 if (db->db_blkid != DMU_SPILL_BLKID &&
2049 db->db_blkid > dn->dn_maxblkid)
2050 dn->dn_maxblkid = db->db_blkid;
2059 dbuf_fix_old_data(db, txg);
2063 if (db->db_state == DB_CACHED) {
2064 ASSERT(db->db.db_data != NULL);
2065 arc_release(db->db_buf, db);
2066 rw_enter(&db->db_rwlock, RW_WRITER);
2067 memset(db->db.db_data, 0, db->db.db_size);
2068 rw_exit(&db->db_rwlock);
2069 arc_buf_freeze(db->db_buf);
2072 mutex_exit(&db->db_mtx);
2080 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx)
2084 int osize = db->db.db_size;
2085 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
2088 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
2090 DB_DNODE_ENTER(db);
2091 dn = DB_DNODE(db);
2097 dmu_buf_will_dirty(&db->db, tx);
2099 VERIFY3P(db->db_buf, !=, NULL);
2102 buf = arc_alloc_buf(dn->dn_objset->os_spa, db, type, size);
2105 old_buf = db->db_buf;
2111 mutex_enter(&db->db_mtx);
2112 dbuf_set_data(db, buf);
2113 arc_buf_destroy(old_buf, db);
2114 db->db.db_size = size;
2116 dr = list_head(&db->db_dirty_records);
2119 if (db->db_level == 0)
2124 mutex_exit(&db->db_mtx);
2127 DB_DNODE_EXIT(db);
2131 dbuf_release_bp(dmu_buf_impl_t *db)
2133 objset_t *os __maybe_unused = db->db_objset;
2138 ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf));
2140 (void) arc_release(db->db_buf, db);
2150 dmu_buf_impl_t *db = dr->dr_dbuf;
2152 ASSERT(MUTEX_HELD(&db->db_mtx));
2154 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) {
2160 if (db->db.db_object != DMU_META_DNODE_OBJECT &&
2161 db->db_state != DB_NOFILL) {
2163 ASSERT(arc_released(db->db_buf));
2164 arc_buf_thaw(db->db_buf);
2239 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
2248 ASSERT(!zfs_refcount_is_zero(&db->db_holds));
2249 DMU_TX_DIRTY_BUF(tx, db);
2251 DB_DNODE_ENTER(db);
2252 dn = DB_DNODE(db);
2279 mutex_enter(&db->db_mtx);
2285 ASSERT(db->db_level != 0 ||
2286 db->db_state == DB_CACHED || db->db_state == DB_FILL ||
2287 db->db_state == DB_NOFILL);
2290 dnode_set_dirtyctx(dn, tx, db);
2295 if (db->db_blkid == DMU_SPILL_BLKID)
2301 dr_head = list_head(&db->db_dirty_records);
2303 db->db.db_object == DMU_META_DNODE_OBJECT);
2304 dr_next = dbuf_find_dirty_lte(db, tx->tx_txg);
2306 DB_DNODE_EXIT(db);
2309 mutex_exit(&db->db_mtx);
2320 ASSERT3U(dn->dn_nlevels, >, db->db_level);
2339 ASSERT(db->db.db_size != 0);
2341 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);
2343 if (db->db_blkid != DMU_BONUS_BLKID && db->db_state != DB_NOFILL) {
2344 dmu_objset_willuse_space(os, db->db.db_size, tx);
2357 if (db->db_level == 0) {
2358 void *data_old = db->db_buf;
2360 if (db->db_state != DB_NOFILL) {
2361 if (db->db_blkid == DMU_BONUS_BLKID) {
2362 dbuf_fix_old_data(db, tx->tx_txg);
2363 data_old = db->db.db_data;
2364 } else if (db->db.db_object != DMU_META_DNODE_OBJECT) {
2374 arc_release(db->db_buf, db);
2375 dbuf_fix_old_data(db, tx->tx_txg);
2376 data_old = db->db_buf;
2387 if (db->db_blkid != DMU_BONUS_BLKID && db->db_state != DB_NOFILL) {
2388 dr->dr_accounted = db->db.db_size;
2390 dr->dr_dbuf = db;
2392 list_insert_before(&db->db_dirty_records, dr_next, dr);
2399 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
2400 db->db_blkid != DMU_SPILL_BLKID) {
2404 db->db_blkid, 1);
2407 db->db_freed_in_flight = FALSE;
2413 dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg);
2414 db->db_dirtycnt += 1;
2415 ASSERT3U(db->db_dirtycnt, <=, 3);
2417 mutex_exit(&db->db_mtx);
2419 if (db->db_blkid == DMU_BONUS_BLKID ||
2420 db->db_blkid == DMU_SPILL_BLKID) {
2426 DB_DNODE_EXIT(db);
2441 if (db->db_blkptr != NULL) {
2442 db_lock_type_t dblt = dmu_buf_lock_parent(db, RW_READER, FTAG);
2443 ddt_prefetch(os->os_spa, db->db_blkptr);
2444 dmu_buf_unlock_parent(db, dblt, FTAG);
2451 ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) ||
2452 dn->dn_phys->dn_nlevels > db->db_level ||
2453 dn->dn_next_nlevels[txgoff] > db->db_level ||
2454 dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level ||
2455 dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level);
2458 if (db->db_level == 0) {
2459 ASSERT(!db->db_objset->os_raw_receive ||
2460 dn->dn_maxblkid >= db->db_blkid);
2461 dnode_new_blkid(dn, db->db_blkid, tx,
2463 ASSERT(dn->dn_maxblkid >= db->db_blkid);
2466 if (db->db_level+1 < dn->dn_nlevels) {
2467 dmu_buf_impl_t *parent = db->db_parent;
2471 if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) {
2473 parent = dbuf_hold_level(dn, db->db_level + 1,
2474 db->db_blkid >> epbs, FTAG);
2480 ASSERT3U(db->db_level + 1, ==, parent->db_level);
2485 mutex_enter(&db->db_mtx);
2490 if (list_head(&db->db_dirty_records) == dr ||
2499 mutex_exit(&db->db_mtx);
2501 ASSERT(db->db_level + 1 == dn->dn_nlevels);
2502 ASSERT(db->db_blkid < dn->dn_nblkptr);
2503 ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf);
2513 DB_DNODE_EXIT(db);
2520 dmu_buf_impl_t *db = dr->dr_dbuf;
2522 if (dr->dt.dl.dr_data != db->db.db_data) {
2529 db->db_data_pending = NULL;
2530 ASSERT(list_next(&db->db_dirty_records, dr) == NULL);
2531 list_remove(&db->db_dirty_records, dr);
2537 ASSERT3U(db->db_dirtycnt, >, 0);
2538 db->db_dirtycnt -= 1;
2546 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
2560 ASSERT(db->db_objset ==
2561 dmu_objset_pool(db->db_objset)->dp_meta_objset ||
2562 txg != spa_syncing_txg(dmu_objset_spa(db->db_objset)));
2563 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
2564 ASSERT0(db->db_level);
2565 ASSERT(MUTEX_HELD(&db->db_mtx));
2570 dbuf_dirty_record_t *dr = dbuf_find_dirty_eq(db, txg);
2573 ASSERT(dr->dr_dbuf == db);
2585 brt_pending_remove(dmu_objset_spa(db->db_objset),
2592 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size);
2594 ASSERT(db->db.db_size != 0);
2599 list_remove(&db->db_dirty_records, dr);
2611 } else if (db->db_blkid == DMU_SPILL_BLKID ||
2612 db->db_level + 1 == dn->dn_nlevels) {
2613 ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf);
2619 if (db->db_state != DB_NOFILL && !brtwrite) {
2622 if (dr->dt.dl.dr_data != db->db_buf) {
2623 ASSERT(db->db_buf != NULL);
2625 arc_buf_destroy(dr->dt.dl.dr_data, db);
2631 ASSERT(db->db_dirtycnt > 0);
2632 db->db_dirtycnt -= 1;
2634 if (zfs_refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) {
2635 ASSERT(db->db_state == DB_NOFILL || brtwrite || diowrite ||
2636 arc_released(db->db_buf));
2637 dbuf_destroy(db);
2647 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2651 ASSERT(!zfs_refcount_is_zero(&db->db_holds));
2657 mutex_enter(&db->db_mtx);
2658 if (db->db_state == DB_CACHED || db->db_state == DB_NOFILL) {
2664 dbuf_dirty_record_t *dr = dbuf_find_dirty_eq(db, tx->tx_txg);
2666 if (db->db_level == 0 &&
2679 mutex_exit(&db->db_mtx);
2684 mutex_exit(&db->db_mtx);
2686 DB_DNODE_ENTER(db);
2687 if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock))
2689 DB_DNODE_EXIT(db);
2697 (void) dbuf_read(db, NULL, flags);
2699 mutex_enter(&db->db_mtx);
2700 VERIFY(!dbuf_undirty(db, tx));
2701 mutex_exit(&db->db_mtx);
2703 (void) dbuf_dirty(db, tx);
2716 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2719 mutex_enter(&db->db_mtx);
2720 dr = dbuf_find_dirty_eq(db, tx->tx_txg);
2721 mutex_exit(&db->db_mtx);
2732 dmu_buf_get_bp_from_dbuf(dmu_buf_impl_t *db, blkptr_t **bp)
2734 ASSERT(MUTEX_HELD(&db->db_mtx));
2737 if (db->db_level != 0) {
2738 *bp = db->db_blkptr;
2742 *bp = db->db_blkptr;
2743 dbuf_dirty_record_t *dr = list_head(&db->db_dirty_records);
2744 if (dr && db->db_state == DB_NOFILL) {
2750 } else if (dr && db->db_state == DB_UNCACHED) {
2764 dmu_buf_untransform_direct(dmu_buf_impl_t *db, spa_t *spa)
2767 DB_DNODE_ENTER(db);
2768 dnode_t *dn = DB_DNODE(db);
2770 ASSERT3S(db->db_state, ==, DB_CACHED);
2771 ASSERT(MUTEX_HELD(&db->db_mtx));
2777 err = dbuf_read_verify_dnode_crypt(db, dn, 0);
2786 if (err == 0 && db->db_buf != NULL &&
2787 (arc_is_encrypted(db->db_buf) ||
2788 arc_is_unauthenticated(db->db_buf) ||
2789 arc_get_compression(db->db_buf) != ZIO_COMPRESS_OFF)) {
2792 SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset),
2793 db->db.db_object, db->db_level, db->db_blkid);
2794 dbuf_fix_old_data(db, spa_syncing_txg(spa));
2795 err = arc_untransform(db->db_buf, spa, &zb, B_FALSE);
2796 dbuf_set_data(db, db->db_buf);
2798 DB_DNODE_EXIT(db);
2810 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2811 ASSERT0(db->db_level);
2813 ASSERT0(db->db_level);
2814 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
2815 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT);
2817 mutex_enter(&db->db_mtx);
2818 DBUF_VERIFY(db);
2839 * hold on the db, so it should never be evicted after calling
2842 VERIFY3B(dbuf_undirty(db, tx), ==, B_FALSE);
2843 ASSERT0P(dbuf_find_dirty_eq(db, tx->tx_txg));
2845 if (db->db_buf != NULL) {
2851 dbuf_dirty_record_t *dr = list_head(&db->db_dirty_records);
2852 if (dr == NULL || dr->dt.dl.dr_data != db->db_buf)
2853 arc_buf_destroy(db->db_buf, db);
2860 db->db_buf = NULL;
2861 dbuf_clear_data(db);
2864 ASSERT3P(db->db_buf, ==, NULL);
2865 ASSERT3P(db->db.db_data, ==, NULL);
2867 db->db_state = DB_NOFILL;
2868 DTRACE_SET_STATE(db,
2871 DBUF_VERIFY(db);
2872 mutex_exit(&db->db_mtx);
2874 dbuf_noread(db);
2875 (void) dbuf_dirty(db, tx);
2881 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2883 mutex_enter(&db->db_mtx);
2884 db->db_state = DB_NOFILL;
2885 DTRACE_SET_STATE(db, "allocating NOFILL buffer");
2886 mutex_exit(&db->db_mtx);
2888 dbuf_noread(db);
2889 (void) dbuf_dirty(db, tx);
2895 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2897 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
2899 ASSERT(db->db_level == 0);
2900 ASSERT(!zfs_refcount_is_zero(&db->db_holds));
2902 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT ||
2905 mutex_enter(&db->db_mtx);
2906 dbuf_dirty_record_t *dr = dbuf_find_dirty_eq(db, tx->tx_txg);
2907 if (db->db_state == DB_NOFILL ||
2908 (db->db_state == DB_UNCACHED && dr && dr->dt.dl.dr_diowrite)) {
2914 mutex_exit(&db->db_mtx);
2924 if (db->db_state == DB_NOFILL) {
2925 VERIFY(!dbuf_undirty(db, tx));
2926 db->db_state = DB_UNCACHED;
2929 mutex_exit(&db->db_mtx);
2931 dbuf_noread(db);
2932 (void) dbuf_dirty(db, tx);
2937 * indicates the caller expects raw encrypted data in the db, and provides
2946 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
2953 ASSERT3U(db->db.db_object, ==, DMU_META_DNODE_OBJECT);
2954 ASSERT0(db->db_level);
2955 ASSERT(db->db_objset->os_raw_receive);
2960 dr = dbuf_find_dirty_eq(db, tx->tx_txg);
2973 dbuf_override_impl(dmu_buf_impl_t *db, const blkptr_t *bp, dmu_tx_t *tx)
2978 ASSERT3U(db->db.db_object, !=, DMU_META_DNODE_OBJECT);
2979 ASSERT0(db->db_level);
2981 dr = list_head(&db->db_dirty_records);
2995 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf;
2996 mutex_enter(&db->db_mtx);
2997 DBUF_VERIFY(db);
2999 if (db->db_state == DB_FILL) {
3000 if (db->db_level == 0 && db->db_freed_in_flight) {
3001 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
3004 memset(db->db.db_data, 0, db->db.db_size);
3005 db->db_freed_in_flight = FALSE;
3006 db->db_state = DB_CACHED;
3007 DTRACE_SET_STATE(db,
3011 VERIFY(!dbuf_undirty(db, tx));
3012 arc_buf_destroy(db->db_buf, db);
3013 db->db_buf = NULL;
3014 dbuf_clear_data(db);
3015 DTRACE_SET_STATE(db, "fill failed");
3017 db->db_state = DB_CACHED;
3018 DTRACE_SET_STATE(db, "fill done");
3020 cv_broadcast(&db->db_changed);
3022 db->db_state = DB_CACHED;
3025 mutex_exit(&db->db_mtx);
3035 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf;
3041 ASSERT(spa_feature_is_active(dmu_objset_spa(db->db_objset),
3045 DB_DNODE_ENTER(db);
3046 type = DB_DNODE(db)->dn_type;
3047 DB_DNODE_EXIT(db);
3049 ASSERT0(db->db_level);
3050 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
3054 dr = list_head(&db->db_dirty_records);
3073 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf;
3075 ASSERT(dsl_dataset_feature_is_active(db->db_objset->os_dsl_dataset,
3078 DB_DNODE_ENTER(db);
3079 type = DB_DNODE(db)->dn_type;
3080 DB_DNODE_EXIT(db);
3082 ASSERT0(db->db_level);
3092 dbuf_override_impl(db, &bp, tx);
3100 dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx)
3102 ASSERT(!zfs_refcount_is_zero(&db->db_holds));
3103 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
3104 ASSERT(db->db_level == 0);
3105 ASSERT3U(dbuf_is_metadata(db), ==, arc_is_metadata(buf));
3107 ASSERT3U(arc_buf_lsize(buf), ==, db->db.db_size);
3110 arc_return_buf(buf, db);
3113 mutex_enter(&db->db_mtx);
3115 while (db->db_state == DB_READ || db->db_state == DB_FILL)
3116 cv_wait(&db->db_changed, &db->db_mtx);
3118 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED ||
3119 db->db_state == DB_NOFILL);
3121 if (db->db_state == DB_CACHED &&
3122 zfs_refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) {
3130 mutex_exit(&db->db_mtx);
3131 (void) dbuf_dirty(db, tx);
3132 memcpy(db->db.db_data, buf->b_data, db->db.db_size);
3133 arc_buf_destroy(buf, db);
3137 if (db->db_state == DB_CACHED) {
3138 dbuf_dirty_record_t *dr = list_head(&db->db_dirty_records);
3140 ASSERT(db->db_buf != NULL);
3142 ASSERT(dr->dt.dl.dr_data == db->db_buf);
3144 if (!arc_released(db->db_buf)) {
3147 arc_release(db->db_buf, db);
3150 arc_buf_destroy(db->db_buf, db);
3151 } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) {
3152 arc_release(db->db_buf, db);
3153 arc_buf_destroy(db->db_buf, db);
3155 db->db_buf = NULL;
3156 } else if (db->db_state == DB_NOFILL) {
3163 VERIFY(!dbuf_undirty(db, tx));
3164 db->db_state = DB_UNCACHED;
3166 ASSERT(db->db_buf == NULL);
3167 dbuf_set_data(db, buf);
3168 db->db_state = DB_FILL;
3169 DTRACE_SET_STATE(db, "filling assigned arcbuf");
3170 mutex_exit(&db->db_mtx);
3171 (void) dbuf_dirty(db, tx);
3172 dmu_buf_fill_done(&db->db, tx, B_FALSE);
3176 dbuf_destroy(dmu_buf_impl_t *db)
3179 dmu_buf_impl_t *parent = db->db_parent;
3182 ASSERT(MUTEX_HELD(&db->db_mtx));
3183 ASSERT(zfs_refcount_is_zero(&db->db_holds));
3185 if (db->db_buf != NULL) {
3186 arc_buf_destroy(db->db_buf, db);
3187 db->db_buf = NULL;
3190 if (db->db_blkid == DMU_BONUS_BLKID) {
3191 int slots = DB_DNODE(db)->dn_num_slots;
3193 if (db->db.db_data != NULL) {
3194 kmem_free(db->db.db_data, bonuslen);
3196 db->db_state = DB_UNCACHED;
3197 DTRACE_SET_STATE(db, "buffer cleared");
3201 dbuf_clear_data(db);
3203 if (multilist_link_active(&db->db_cache_link)) {
3204 ASSERT(db->db_caching_status == DB_DBUF_CACHE ||
3205 db->db_caching_status == DB_DBUF_METADATA_CACHE);
3207 multilist_remove(&dbuf_caches[db->db_caching_status].cache, db);
3209 ASSERT0(dmu_buf_user_size(&db->db));
3211 &dbuf_caches[db->db_caching_status].size,
3212 db->db.db_size, db);
3214 if (db->db_caching_status == DB_DBUF_METADATA_CACHE) {
3217 DBUF_STAT_BUMPDOWN(cache_levels[db->db_level]);
3219 DBUF_STAT_DECR(cache_levels_bytes[db->db_level],
3220 db->db.db_size);
3222 db->db_caching_status = DB_NO_CACHE;
3225 ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL);
3226 ASSERT(db->db_data_pending == NULL);
3227 ASSERT(list_is_empty(&db->db_dirty_records));
3229 db->db_state = DB_EVICTING;
3230 DTRACE_SET_STATE(db, "buffer eviction started");
3231 db->db_blkptr = NULL;
3238 mutex_exit(&db->db_mtx);
3240 DB_DNODE_ENTER(db);
3241 dn = DB_DNODE(db);
3243 if (db->db_blkid != DMU_BONUS_BLKID) {
3248 avl_remove(&dn->dn_dbufs, db);
3250 DB_DNODE_EXIT(db);
3262 dnode_rele_and_unlock(dn, db, B_TRUE);
3264 db->db_dnode_handle = NULL;
3266 db->db_dnode = NULL;
3269 dbuf_hash_remove(db);
3271 DB_DNODE_EXIT(db);
3274 ASSERT(zfs_refcount_is_zero(&db->db_holds));
3276 db->db_parent = NULL;
3278 ASSERT(db->db_buf == NULL);
3279 ASSERT(db->db.db_data == NULL);
3280 ASSERT(db->db_hash_next == NULL);
3281 ASSERT(db->db_blkptr == NULL);
3282 ASSERT(db->db_data_pending == NULL);
3283 ASSERT3U(db->db_caching_status, ==, DB_NO_CACHE);
3284 ASSERT(!multilist_link_active(&db->db_cache_link));
3292 dbuf_rele_and_unlock(parent, db, B_TRUE);
3295 kmem_cache_free(dbuf_kmem_cache, db);
3376 *bpp = ((blkptr_t *)(*parentp)->db.db_data) +
3401 dmu_buf_impl_t *db, *odb;
3406 db = kmem_cache_alloc(dbuf_kmem_cache, KM_SLEEP);
3408 list_create(&db->db_dirty_records, sizeof (dbuf_dirty_record_t),
3411 db->db_objset = os;
3412 db->db.db_object = dn->dn_object;
3413 db->db_level = level;
3414 db->db_blkid = blkid;
3415 db->db_dirtycnt = 0;
3417 db->db_dnode_handle = dn->dn_handle;
3419 db->db_dnode = dn;
3421 db->db_parent = parent;
3422 db->db_blkptr = blkptr;
3423 db->db_hash = hash;
3425 db->db_user = NULL;
3426 db->db_user_immediate_evict = FALSE;
3427 db->db_freed_in_flight = FALSE;
3428 db->db_pending_evict = FALSE;
3432 db->db.db_size = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) -
3434 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen);
3435 db->db.db_offset = DMU_BONUS_BLKID;
3436 db->db_state = DB_UNCACHED;
3437 DTRACE_SET_STATE(db, "bonus buffer created");
3438 db->db_caching_status = DB_NO_CACHE;
3441 return (db);
3443 db->db.db_size = (blkptr != NULL) ?
3445 db->db.db_offset = 0;
3448 db->db_level ? 1 << dn->dn_indblkshift : dn->dn_datablksz;
3449 db->db.db_size = blocksize;
3450 db->db.db_offset = db->db_blkid * blocksize;
3461 db->db_state = DB_EVICTING; /* not worth logging this state change */
3462 if ((odb = dbuf_hash_insert(db)) != NULL) {
3465 kmem_cache_free(dbuf_kmem_cache, db);
3469 avl_add(&dn->dn_dbufs, db);
3471 db->db_state = DB_UNCACHED;
3472 DTRACE_SET_STATE(db, "regular buffer created");
3473 db->db_caching_status = DB_NO_CACHE;
3478 dbuf_add_ref(parent, db);
3482 (void) zfs_refcount_add(&dn->dn_holds, db);
3484 dprintf_dbuf(db, "db=%p\n", db);
3486 return (db);
3635 dmu_buf_impl_t *db = dbuf_hold_level(dpa->dpa_dnode,
3637 if (db == NULL) {
3642 (void) dbuf_read(db, NULL,
3644 dbuf_rele(db, FTAG);
3724 dmu_buf_impl_t *db = dbuf_find(dn->dn_objset, dn->dn_object,
3726 if (db != NULL) {
3727 mutex_exit(&db->db_mtx);
3745 dmu_buf_impl_t *db;
3748 FALSE, TRUE, FTAG, &db) == 0) {
3749 blkptr_t *bpp = db->db_buf->b_data;
3751 dbuf_rele(db, FTAG);
3849 dbuf_hold_copy(dnode_t *dn, dmu_buf_impl_t *db)
3851 dbuf_dirty_record_t *dr = db->db_data_pending;
3863 dbuf_set_data(db, arc_alloc_raw_buf(dn->dn_objset->os_spa, db,
3868 dbuf_set_data(db, arc_alloc_compressed_buf(
3869 dn->dn_objset->os_spa, db, arc_buf_size(data),
3872 dbuf_set_data(db, arc_alloc_buf(dn->dn_objset->os_spa, db,
3873 DBUF_GET_BUFC_TYPE(db), db->db.db_size));
3876 rw_enter(&db->db_rwlock, RW_WRITER);
3877 memcpy(db->db.db_data, data->b_data, arc_buf_size(data));
3878 rw_exit(&db->db_rwlock);
3890 dmu_buf_impl_t *db, *parent = NULL;
3907 db = dbuf_find(dn->dn_objset, dn->dn_object, level, blkid, &hv);
3909 if (db == NULL) {
3929 db = dbuf_create(dn, level, blkid, parent, bp, hv);
3932 if (fail_uncached && db->db_state != DB_CACHED) {
3933 mutex_exit(&db->db_mtx);
3937 if (db->db_buf != NULL) {
3938 arc_buf_access(db->db_buf);
3939 ASSERT3P(db->db.db_data, ==, db->db_buf->b_data);
3942 ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf));
3949 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID &&
3951 db->db_state == DB_CACHED && db->db_data_pending) {
3952 dbuf_dirty_record_t *dr = db->db_data_pending;
3953 if (dr->dt.dl.dr_data == db->db_buf) {
3954 ASSERT3P(db->db_buf, !=, NULL);
3955 dbuf_hold_copy(dn, db);
3959 if (multilist_link_active(&db->db_cache_link)) {
3960 ASSERT(zfs_refcount_is_zero(&db->db_holds));
3961 ASSERT(db->db_caching_status == DB_DBUF_CACHE ||
3962 db->db_caching_status == DB_DBUF_METADATA_CACHE);
3964 multilist_remove(&dbuf_caches[db->db_caching_status].cache, db);
3966 uint64_t size = db->db.db_size;
3967 uint64_t usize = dmu_buf_user_size(&db->db);
3969 &dbuf_caches[db->db_caching_status].size, size, db);
3971 &dbuf_caches[db->db_caching_status].size, usize,
3972 db->db_user);
3974 if (db->db_caching_status == DB_DBUF_METADATA_CACHE) {
3977 DBUF_STAT_BUMPDOWN(cache_levels[db->db_level]);
3979 DBUF_STAT_DECR(cache_levels_bytes[db->db_level],
3982 db->db_caching_status = DB_NO_CACHE;
3984 (void) zfs_refcount_add(&db->db_holds, tag);
3985 DBUF_VERIFY(db);
3986 mutex_exit(&db->db_mtx);
3992 ASSERT3P(DB_DNODE(db), ==, dn);
3993 ASSERT3U(db->db_blkid, ==, blkid);
3994 ASSERT3U(db->db_level, ==, level);
3995 *dbp = db;
4009 dmu_buf_impl_t *db;
4010 int err = dbuf_hold_impl(dn, level, blkid, FALSE, FALSE, tag, &db);
4011 return (err ? NULL : db);
4027 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
4029 if (db->db_blkid != DMU_SPILL_BLKID)
4033 ASSERT3U(blksz, <=, spa_maxblocksize(dmu_objset_spa(db->db_objset)));
4036 dbuf_new_size(db, blksz, tx);
4049 dbuf_add_ref(dmu_buf_impl_t *db, const void *tag)
4051 int64_t holds = zfs_refcount_add(&db->db_holds, tag);
4060 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
4070 if (db == found_db && dbuf_refcount(db) > db->db_dirtycnt) {
4071 (void) zfs_refcount_add(&db->db_holds, tag);
4087 dbuf_rele(dmu_buf_impl_t *db, const void *tag)
4089 mutex_enter(&db->db_mtx);
4090 dbuf_rele_and_unlock(db, tag, B_FALSE);
4094 dmu_buf_rele(dmu_buf_t *db, const void *tag)
4096 dbuf_rele((dmu_buf_impl_t *)db, tag);
4113 dbuf_rele_and_unlock(dmu_buf_impl_t *db, const void *tag, boolean_t evicting)
4118 ASSERT(MUTEX_HELD(&db->db_mtx));
4119 DBUF_VERIFY(db);
4126 holds = zfs_refcount_remove(&db->db_holds, tag);
4133 if (db->db_buf != NULL &&
4134 holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) {
4135 arc_buf_freeze(db->db_buf);
4138 if (holds == db->db_dirtycnt &&
4139 db->db_level == 0 && db->db_user_immediate_evict)
4140 dbuf_evict_user(db);
4143 if (db->db_blkid == DMU_BONUS_BLKID) {
4145 boolean_t evict_dbuf = db->db_pending_evict;
4151 DB_DNODE_ENTER(db);
4153 dn = DB_DNODE(db);
4162 DB_DNODE_EXIT(db);
4165 * Do not reference db after its lock is dropped.
4168 mutex_exit(&db->db_mtx);
4173 dnode_rele(dn, db);
4174 } else if (db->db_buf == NULL) {
4179 ASSERT(db->db_state == DB_UNCACHED ||
4180 db->db_state == DB_NOFILL);
4181 dbuf_destroy(db);
4182 } else if (arc_released(db->db_buf)) {
4186 dbuf_destroy(db);
4187 } else if (!(DBUF_IS_CACHEABLE(db) || db->db_partial_read) ||
4188 db->db_pending_evict) {
4189 dbuf_destroy(db);
4190 } else if (!multilist_link_active(&db->db_cache_link)) {
4191 ASSERT3U(db->db_caching_status, ==, DB_NO_CACHE);
4194 dbuf_include_in_metadata_cache(db) ?
4196 db->db_caching_status = dcs;
4198 multilist_insert(&dbuf_caches[dcs].cache, db);
4199 uint64_t db_size = db->db.db_size;
4200 uint64_t dbu_size = dmu_buf_user_size(&db->db);
4202 &dbuf_caches[dcs].size, db_size, db);
4204 &dbuf_caches[dcs].size, dbu_size, db->db_user);
4205 uint8_t db_level = db->db_level;
4206 mutex_exit(&db->db_mtx);
4224 mutex_exit(&db->db_mtx);
4230 dbuf_refcount(dmu_buf_impl_t *db)
4232 return (zfs_refcount_count(&db->db_holds));
4239 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
4241 mutex_enter(&db->db_mtx);
4242 ASSERT3U(zfs_refcount_count(&db->db_holds), >=, db->db_dirtycnt);
4243 holds = zfs_refcount_count(&db->db_holds) - db->db_dirtycnt;
4244 mutex_exit(&db->db_mtx);
4253 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
4255 mutex_enter(&db->db_mtx);
4256 dbuf_verify_user(db, DBVU_NOT_EVICTING);
4257 if (db->db_user == old_user)
4258 db->db_user = new_user;
4260 old_user = db->db_user;
4261 dbuf_verify_user(db, DBVU_NOT_EVICTING);
4262 mutex_exit(&db->db_mtx);
4276 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
4278 db->db_user_immediate_evict = TRUE;
4291 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
4293 dbuf_verify_user(db, DBVU_NOT_EVICTING);
4294 return (db->db_user);
4300 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
4301 if (db->db_user == NULL)
4303 return (atomic_load_64(&db->db_user->dbu_size));
4309 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
4310 ASSERT3U(db->db_caching_status, ==, DB_NO_CACHE);
4311 ASSERT3P(db->db_user, !=, NULL);
4312 ASSERT3U(atomic_load_64(&db->db_user->dbu_size), <, UINT64_MAX - nadd);
4313 atomic_add_64(&db->db_user->dbu_size, nadd);
4319 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
4320 ASSERT3U(db->db_caching_status, ==, DB_NO_CACHE);
4321 ASSERT3P(db->db_user, !=, NULL);
4322 ASSERT3U(atomic_load_64(&db->db_user->dbu_size), >=, nsub);
4323 atomic_sub_64(&db->db_user->dbu_size, nsub);
4333 dmu_buf_get_blkptr(dmu_buf_t *db)
4335 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db;
4340 dmu_buf_get_objset(dmu_buf_t *db)
4342 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db;
4347 dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db)
4350 ASSERT(MUTEX_HELD(&db->db_mtx));
4352 if (db->db_blkptr != NULL)
4355 if (db->db_blkid == DMU_SPILL_BLKID) {
4356 db->db_blkptr = DN_SPILL_BLKPTR(dn->dn_phys);
4357 BP_ZERO(db->db_blkptr);
4360 if (db->db_level == dn->dn_phys->dn_nlevels-1) {
4366 ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr);
4367 ASSERT(db->db_parent == NULL);
4368 db->db_parent = dn->dn_dbuf;
4369 db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid];
4370 DBUF_VERIFY(db);
4372 dmu_buf_impl_t *parent = db->db_parent;
4377 mutex_exit(&db->db_mtx);
4379 parent = dbuf_hold_level(dn, db->db_level + 1,
4380 db->db_blkid >> epbs, db);
4382 mutex_enter(&db->db_mtx);
4383 db->db_parent = parent;
4385 db->db_blkptr = (blkptr_t *)parent->db.db_data +
4386 (db->db_blkid & ((1ULL << epbs) - 1));
4387 DBUF_VERIFY(db);
4394 dmu_buf_impl_t *db = dr->dr_dbuf;
4397 ASSERT0(db->db_level);
4398 ASSERT(MUTEX_HELD(&db->db_mtx));
4399 ASSERT(db->db_blkid == DMU_BONUS_BLKID);
4410 dbuf_rele_and_unlock(db, (void *)(uintptr_t)tx->tx_txg, B_FALSE);
4424 dmu_buf_impl_t *db = dr->dr_dbuf;
4426 ASSERT(MUTEX_HELD(&db->db_mtx));
4427 ASSERT3U(db->db.db_object, ==, DMU_META_DNODE_OBJECT);
4428 ASSERT3U(db->db_level, ==, 0);
4430 if (!db->db_objset->os_raw_receive && arc_is_encrypted(db->db_buf)) {
4440 SET_BOOKMARK(&zb, dmu_objset_id(db->db_objset),
4441 db->db.db_object, db->db_level, db->db_blkid);
4442 err = arc_untransform(db->db_buf, db->db_objset->os_spa,
4447 (void) arc_release(dr->dt.dl.dr_data, db);
4449 dmu_objset_id(db->db_objset),
4463 dmu_buf_impl_t *db = dr->dr_dbuf;
4468 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr);
4470 mutex_enter(&db->db_mtx);
4472 ASSERT(db->db_level > 0);
4473 DBUF_VERIFY(db);
4476 if (db->db_buf == NULL) {
4477 mutex_exit(&db->db_mtx);
4478 (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED);
4479 mutex_enter(&db->db_mtx);
4481 ASSERT3U(db->db_state, ==, DB_CACHED);
4482 ASSERT(db->db_buf != NULL);
4485 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
4486 dbuf_check_blkptr(dn, db);
4489 db->db_data_pending = dr;
4491 mutex_exit(&db->db_mtx);
4493 dbuf_write(dr, db->db_buf, tx);
4497 dbuf_sync_list(&dr->dt.di.dr_children, db->db_level - 1, tx);
4559 blkptr_t *bp = parent_db->db.db_data;
4674 dmu_buf_impl_t *db = dr->dr_dbuf;
4681 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr);
4683 mutex_enter(&db->db_mtx);
4688 if (db->db_state == DB_UNCACHED) {
4690 ASSERT3P(db->db.db_data, ==, NULL);
4691 } else if (db->db_state == DB_FILL) {
4693 ASSERT(db->db.db_data != dr->dt.dl.dr_data);
4694 } else if (db->db_state == DB_READ) {
4706 list_head(&db->db_dirty_records);
4707 ASSERT3P(db->db_buf, ==, NULL);
4708 ASSERT3P(db->db.db_data, ==, NULL);
4712 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL);
4714 DBUF_VERIFY(db);
4716 if (db->db_blkid == DMU_SPILL_BLKID) {
4732 db->db_blkptr = NULL;
4744 if (db->db_blkid == DMU_BONUS_BLKID) {
4745 ASSERT(dr->dr_dbuf == db);
4758 dbuf_check_blkptr(dn, db);
4770 cv_wait(&db->db_changed, &db->db_mtx);
4780 if (*datap != NULL && *datap == db->db_buf &&
4782 zfs_refcount_count(&db->db_holds) > 1) {
4796 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db);
4807 *datap = arc_alloc_raw_buf(os->os_spa, db,
4813 *datap = arc_alloc_compressed_buf(os->os_spa, db,
4816 *datap = arc_alloc_buf(os->os_spa, db, type, psize);
4818 memcpy((*datap)->b_data, db->db.db_data, psize);
4820 db->db_data_pending = dr;
4822 mutex_exit(&db->db_mtx);
4852 ASSERT3U(dr->dr_dbuf->db.db_object, ==,
4876 dmu_buf_impl_t *db = vdb;
4885 ASSERT3P(db->db_blkptr, !=, NULL);
4886 ASSERT3P(&db->db_data_pending->dr_bp_copy, ==, bp);
4888 DB_DNODE_ENTER(db);
4889 dn = DB_DNODE(db);
4895 ASSERT((db->db_blkid != DMU_SPILL_BLKID &&
4897 (db->db_blkid == DMU_SPILL_BLKID &&
4900 ASSERT(BP_GET_LEVEL(bp) == db->db_level);
4903 mutex_enter(&db->db_mtx);
4906 if (db->db_blkid == DMU_SPILL_BLKID) {
4909 db->db_blkptr == DN_SPILL_BLKPTR(dn->dn_phys));
4913 if (db->db_level == 0) {
4915 if (db->db_blkid > dn->dn_phys->dn_maxblkid &&
4916 db->db_blkid != DMU_SPILL_BLKID) {
4917 ASSERT0(db->db_objset->os_raw_receive);
4918 dn->dn_phys->dn_maxblkid = db->db_blkid;
4924 while (i < db->db.db_size) {
4926 (void *)(((char *)db->db.db_data) + i);
4957 blkptr_t *ibp = db->db.db_data;
4958 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift);
4959 for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) {
4967 DB_DNODE_EXIT(db);
4972 mutex_exit(&db->db_mtx);
4974 db_lock_type_t dblt = dmu_buf_lock_parent(db, RW_WRITER, FTAG);
4975 *db->db_blkptr = *bp;
4976 dmu_buf_unlock_parent(db, dblt, FTAG);
4990 dmu_buf_impl_t *db = vdb;
4994 ASSERT3U(db->db_level, >, 0);
4995 DB_DNODE_ENTER(db);
4996 epbs = DB_DNODE(db)->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
4997 DB_DNODE_EXIT(db);
5001 for (i = 0, bp = db->db.db_data; i < 1ULL << epbs; i++, bp++) {
5016 rw_enter(&db->db_rwlock, RW_WRITER);
5017 memset(db->db.db_data, 0, db->db.db_size);
5018 rw_exit(&db->db_rwlock);
5026 dmu_buf_impl_t *db = vdb;
5028 blkptr_t *bp = db->db_blkptr;
5029 objset_t *os = db->db_objset;
5033 ASSERT(db->db_blkptr == bp);
5047 mutex_enter(&db->db_mtx);
5049 DBUF_VERIFY(db);
5051 dbuf_dirty_record_t *dr = db->db_data_pending;
5054 ASSERT(dr->dr_dbuf == db);
5055 ASSERT(list_next(&db->db_dirty_records, dr) == NULL);
5056 list_remove(&db->db_dirty_records, dr);
5059 if (db->db_blkid == DMU_SPILL_BLKID) {
5061 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) &&
5062 db->db_blkptr == DN_SPILL_BLKPTR(dn->dn_phys));
5066 if (db->db_level == 0) {
5067 ASSERT(db->db_blkid != DMU_BONUS_BLKID);
5072 dr->dt.dl.dr_data != db->db_buf) {
5075 arc_buf_destroy(dr->dt.dl.dr_data, db);
5079 ASSERT3U(db->db.db_size, ==, 1 << dn->dn_phys->dn_indblkshift);
5080 if (!BP_IS_HOLE(db->db_blkptr)) {
5083 ASSERT3U(db->db_blkid, <=,
5084 dn->dn_phys->dn_maxblkid >> (db->db_level * epbs));
5085 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==,
5086 db->db.db_size);
5092 cv_broadcast(&db->db_changed);
5093 ASSERT(db->db_dirtycnt > 0);
5094 db->db_dirtycnt -= 1;
5095 db->db_data_pending = NULL;
5096 dbuf_rele_and_unlock(db, (void *)(uintptr_t)tx->tx_txg, B_FALSE);
5120 dmu_buf_impl_t *db = dr->dr_dbuf;
5122 dbuf_write_ready(zio, NULL, db);
5129 dmu_buf_impl_t *db = dr->dr_dbuf;
5132 mutex_enter(&db->db_mtx);
5136 arc_release(dr->dt.dl.dr_data, db);
5138 mutex_exit(&db->db_mtx);
5140 dbuf_write_done(zio, NULL, db);
5227 dbuf_remap(dnode_t *dn, dmu_buf_impl_t *db, dmu_tx_t *tx)
5229 spa_t *spa = dmu_objset_spa(db->db_objset);
5235 if (db->db_level > 0) {
5236 blkptr_t *bp = db->db.db_data;
5237 for (int i = 0; i < db->db.db_size >> SPA_BLKPTRSHIFT; i++) {
5238 dbuf_remap_impl(dn, &bp[i], &db->db_rwlock, tx);
5240 } else if (db->db.db_object == DMU_META_DNODE_OBJECT) {
5241 dnode_phys_t *dnp = db->db.db_data;
5243 for (int i = 0; i < db->db.db_size >> DNODE_SHIFT;
5263 dmu_buf_impl_t *db = dr->dr_dbuf;
5266 dmu_buf_impl_t *parent = db->db_parent;
5277 if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) {
5284 if (BP_IS_HOLE(db->db_blkptr))
5287 dbuf_release_bp(db);
5288 dbuf_remap(dn, db, tx);
5296 ASSERT(db->db_level == parent->db_level-1);
5305 ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 &&
5306 db->db_blkid != DMU_SPILL_BLKID) ||
5307 (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0));
5308 if (db->db_blkid != DMU_SPILL_BLKID)
5309 ASSERT3P(db->db_blkptr, ==,
5310 &dn->dn_phys->dn_blkptr[db->db_blkid]);
5314 ASSERT(db->db_level == 0 || data == db->db_buf);
5315 ASSERT3U(BP_GET_LOGICAL_BIRTH(db->db_blkptr), <=, txg);
5320 db->db.db_object, db->db_level, db->db_blkid);
5322 if (db->db_blkid == DMU_SPILL_BLKID)
5326 dmu_write_policy(os, dn, db->db_level, wp_flag, &zp);
5334 dr->dr_bp_copy = *db->db_blkptr;
5336 if (db->db_level == 0 &&
5347 contents, db->db.db_size, db->db.db_size, &zp,
5351 mutex_enter(&db->db_mtx);
5356 mutex_exit(&db->db_mtx);
5361 &dr->dr_bp_copy, NULL, db->db.db_size, db->db.db_size, &zp,
5363 dbuf_write_nofill_done, db,
5375 if (db->db_level != 0)
5379 &dr->dr_bp_copy, data, !DBUF_IS_CACHEABLE(db),
5380 dbuf_is_l2cacheable(db, NULL), &zp, dbuf_write_ready,
5381 children_ready_cb, dbuf_write_done, db,