xref: /freebsd-src/sys/contrib/openzfs/module/zfs/dnode.c (revision ce4dcb97ca433b2a2f03fbae957dae0ff16f6f51)
1eda14cbcSMatt Macy /*
2eda14cbcSMatt Macy  * CDDL HEADER START
3eda14cbcSMatt Macy  *
4eda14cbcSMatt Macy  * The contents of this file are subject to the terms of the
5eda14cbcSMatt Macy  * Common Development and Distribution License (the "License").
6eda14cbcSMatt Macy  * You may not use this file except in compliance with the License.
7eda14cbcSMatt Macy  *
8eda14cbcSMatt Macy  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9271171e0SMartin Matuska  * or https://opensource.org/licenses/CDDL-1.0.
10eda14cbcSMatt Macy  * See the License for the specific language governing permissions
11eda14cbcSMatt Macy  * and limitations under the License.
12eda14cbcSMatt Macy  *
13eda14cbcSMatt Macy  * When distributing Covered Code, include this CDDL HEADER in each
14eda14cbcSMatt Macy  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15eda14cbcSMatt Macy  * If applicable, add the following below this CDDL HEADER, with the
16eda14cbcSMatt Macy  * fields enclosed by brackets "[]" replaced with your own identifying
17eda14cbcSMatt Macy  * information: Portions Copyright [yyyy] [name of copyright owner]
18eda14cbcSMatt Macy  *
19eda14cbcSMatt Macy  * CDDL HEADER END
20eda14cbcSMatt Macy  */
21eda14cbcSMatt Macy /*
22eda14cbcSMatt Macy  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
237877fdebSMatt Macy  * Copyright (c) 2012, 2020 by Delphix. All rights reserved.
24eda14cbcSMatt Macy  * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
25eda14cbcSMatt Macy  */
26eda14cbcSMatt Macy 
27eda14cbcSMatt Macy #include <sys/zfs_context.h>
28eda14cbcSMatt Macy #include <sys/dbuf.h>
29eda14cbcSMatt Macy #include <sys/dnode.h>
30eda14cbcSMatt Macy #include <sys/dmu.h>
31eda14cbcSMatt Macy #include <sys/dmu_impl.h>
32eda14cbcSMatt Macy #include <sys/dmu_tx.h>
33eda14cbcSMatt Macy #include <sys/dmu_objset.h>
34eda14cbcSMatt Macy #include <sys/dsl_dir.h>
35eda14cbcSMatt Macy #include <sys/dsl_dataset.h>
36eda14cbcSMatt Macy #include <sys/spa.h>
37eda14cbcSMatt Macy #include <sys/zio.h>
38eda14cbcSMatt Macy #include <sys/dmu_zfetch.h>
39eda14cbcSMatt Macy #include <sys/range_tree.h>
40eda14cbcSMatt Macy #include <sys/trace_zfs.h>
41eda14cbcSMatt Macy #include <sys/zfs_project.h>
42eda14cbcSMatt Macy 
43eda14cbcSMatt Macy dnode_stats_t dnode_stats = {
44eda14cbcSMatt Macy 	{ "dnode_hold_dbuf_hold",		KSTAT_DATA_UINT64 },
45eda14cbcSMatt Macy 	{ "dnode_hold_dbuf_read",		KSTAT_DATA_UINT64 },
46eda14cbcSMatt Macy 	{ "dnode_hold_alloc_hits",		KSTAT_DATA_UINT64 },
47eda14cbcSMatt Macy 	{ "dnode_hold_alloc_misses",		KSTAT_DATA_UINT64 },
48eda14cbcSMatt Macy 	{ "dnode_hold_alloc_interior",		KSTAT_DATA_UINT64 },
49eda14cbcSMatt Macy 	{ "dnode_hold_alloc_lock_retry",	KSTAT_DATA_UINT64 },
50eda14cbcSMatt Macy 	{ "dnode_hold_alloc_lock_misses",	KSTAT_DATA_UINT64 },
51eda14cbcSMatt Macy 	{ "dnode_hold_alloc_type_none",		KSTAT_DATA_UINT64 },
52eda14cbcSMatt Macy 	{ "dnode_hold_free_hits",		KSTAT_DATA_UINT64 },
53eda14cbcSMatt Macy 	{ "dnode_hold_free_misses",		KSTAT_DATA_UINT64 },
54eda14cbcSMatt Macy 	{ "dnode_hold_free_lock_misses",	KSTAT_DATA_UINT64 },
55eda14cbcSMatt Macy 	{ "dnode_hold_free_lock_retry",		KSTAT_DATA_UINT64 },
56eda14cbcSMatt Macy 	{ "dnode_hold_free_overflow",		KSTAT_DATA_UINT64 },
57eda14cbcSMatt Macy 	{ "dnode_hold_free_refcount",		KSTAT_DATA_UINT64 },
58eda14cbcSMatt Macy 	{ "dnode_free_interior_lock_retry",	KSTAT_DATA_UINT64 },
59eda14cbcSMatt Macy 	{ "dnode_allocate",			KSTAT_DATA_UINT64 },
60eda14cbcSMatt Macy 	{ "dnode_reallocate",			KSTAT_DATA_UINT64 },
61eda14cbcSMatt Macy 	{ "dnode_buf_evict",			KSTAT_DATA_UINT64 },
62eda14cbcSMatt Macy 	{ "dnode_alloc_next_chunk",		KSTAT_DATA_UINT64 },
63eda14cbcSMatt Macy 	{ "dnode_alloc_race",			KSTAT_DATA_UINT64 },
64eda14cbcSMatt Macy 	{ "dnode_alloc_next_block",		KSTAT_DATA_UINT64 },
65eda14cbcSMatt Macy 	{ "dnode_move_invalid",			KSTAT_DATA_UINT64 },
66eda14cbcSMatt Macy 	{ "dnode_move_recheck1",		KSTAT_DATA_UINT64 },
67eda14cbcSMatt Macy 	{ "dnode_move_recheck2",		KSTAT_DATA_UINT64 },
68eda14cbcSMatt Macy 	{ "dnode_move_special",			KSTAT_DATA_UINT64 },
69eda14cbcSMatt Macy 	{ "dnode_move_handle",			KSTAT_DATA_UINT64 },
70eda14cbcSMatt Macy 	{ "dnode_move_rwlock",			KSTAT_DATA_UINT64 },
71eda14cbcSMatt Macy 	{ "dnode_move_active",			KSTAT_DATA_UINT64 },
72eda14cbcSMatt Macy };
73eda14cbcSMatt Macy 
74bb2d13b6SMartin Matuska dnode_sums_t dnode_sums;
75bb2d13b6SMartin Matuska 
76eda14cbcSMatt Macy static kstat_t *dnode_ksp;
77eda14cbcSMatt Macy static kmem_cache_t *dnode_cache;
78eda14cbcSMatt Macy 
79eda14cbcSMatt Macy static dnode_phys_t dnode_phys_zero __maybe_unused;
80eda14cbcSMatt Macy 
81eda14cbcSMatt Macy int zfs_default_bs = SPA_MINBLOCKSHIFT;
82eda14cbcSMatt Macy int zfs_default_ibs = DN_MAX_INDBLKSHIFT;
83eda14cbcSMatt Macy 
84eda14cbcSMatt Macy #ifdef	_KERNEL
85eda14cbcSMatt Macy static kmem_cbrc_t dnode_move(void *, void *, size_t, void *);
86eda14cbcSMatt Macy #endif /* _KERNEL */
87eda14cbcSMatt Macy 
88eda14cbcSMatt Macy static int
89eda14cbcSMatt Macy dbuf_compare(const void *x1, const void *x2)
90eda14cbcSMatt Macy {
91eda14cbcSMatt Macy 	const dmu_buf_impl_t *d1 = x1;
92eda14cbcSMatt Macy 	const dmu_buf_impl_t *d2 = x2;
93eda14cbcSMatt Macy 
94eda14cbcSMatt Macy 	int cmp = TREE_CMP(d1->db_level, d2->db_level);
95eda14cbcSMatt Macy 	if (likely(cmp))
96eda14cbcSMatt Macy 		return (cmp);
97eda14cbcSMatt Macy 
98eda14cbcSMatt Macy 	cmp = TREE_CMP(d1->db_blkid, d2->db_blkid);
99eda14cbcSMatt Macy 	if (likely(cmp))
100eda14cbcSMatt Macy 		return (cmp);
101eda14cbcSMatt Macy 
1025fb307d2SMartin Matuska 	if (d1->db_state == DB_MARKER) {
1035fb307d2SMartin Matuska 		ASSERT3S(d2->db_state, !=, DB_MARKER);
1045fb307d2SMartin Matuska 		return (TREE_PCMP(d1->db_parent, d2));
1055fb307d2SMartin Matuska 	} else if (d2->db_state == DB_MARKER) {
1065fb307d2SMartin Matuska 		ASSERT3S(d1->db_state, !=, DB_MARKER);
1075fb307d2SMartin Matuska 		return (TREE_PCMP(d1, d2->db_parent));
1085fb307d2SMartin Matuska 	}
1095fb307d2SMartin Matuska 
110eda14cbcSMatt Macy 	if (d1->db_state == DB_SEARCH) {
111eda14cbcSMatt Macy 		ASSERT3S(d2->db_state, !=, DB_SEARCH);
112eda14cbcSMatt Macy 		return (-1);
113eda14cbcSMatt Macy 	} else if (d2->db_state == DB_SEARCH) {
114eda14cbcSMatt Macy 		ASSERT3S(d1->db_state, !=, DB_SEARCH);
115eda14cbcSMatt Macy 		return (1);
116eda14cbcSMatt Macy 	}
117eda14cbcSMatt Macy 
118eda14cbcSMatt Macy 	return (TREE_PCMP(d1, d2));
119eda14cbcSMatt Macy }
120eda14cbcSMatt Macy 
121eda14cbcSMatt Macy static int
122eda14cbcSMatt Macy dnode_cons(void *arg, void *unused, int kmflag)
123eda14cbcSMatt Macy {
124e92ffd9bSMartin Matuska 	(void) unused, (void) kmflag;
125eda14cbcSMatt Macy 	dnode_t *dn = arg;
126eda14cbcSMatt Macy 
127eda14cbcSMatt Macy 	rw_init(&dn->dn_struct_rwlock, NULL, RW_NOLOCKDEP, NULL);
128eda14cbcSMatt Macy 	mutex_init(&dn->dn_mtx, NULL, MUTEX_DEFAULT, NULL);
129eda14cbcSMatt Macy 	mutex_init(&dn->dn_dbufs_mtx, NULL, MUTEX_DEFAULT, NULL);
130eda14cbcSMatt Macy 	cv_init(&dn->dn_notxholds, NULL, CV_DEFAULT, NULL);
131eda14cbcSMatt Macy 	cv_init(&dn->dn_nodnholds, NULL, CV_DEFAULT, NULL);
132eda14cbcSMatt Macy 
133eda14cbcSMatt Macy 	/*
134eda14cbcSMatt Macy 	 * Every dbuf has a reference, and dropping a tracked reference is
135eda14cbcSMatt Macy 	 * O(number of references), so don't track dn_holds.
136eda14cbcSMatt Macy 	 */
137eda14cbcSMatt Macy 	zfs_refcount_create_untracked(&dn->dn_holds);
138eda14cbcSMatt Macy 	zfs_refcount_create(&dn->dn_tx_holds);
139eda14cbcSMatt Macy 	list_link_init(&dn->dn_link);
140eda14cbcSMatt Macy 
141da5137abSMartin Matuska 	memset(dn->dn_next_type, 0, sizeof (dn->dn_next_type));
142da5137abSMartin Matuska 	memset(dn->dn_next_nblkptr, 0, sizeof (dn->dn_next_nblkptr));
143da5137abSMartin Matuska 	memset(dn->dn_next_nlevels, 0, sizeof (dn->dn_next_nlevels));
144da5137abSMartin Matuska 	memset(dn->dn_next_indblkshift, 0, sizeof (dn->dn_next_indblkshift));
145da5137abSMartin Matuska 	memset(dn->dn_next_bonustype, 0, sizeof (dn->dn_next_bonustype));
146da5137abSMartin Matuska 	memset(dn->dn_rm_spillblk, 0, sizeof (dn->dn_rm_spillblk));
147da5137abSMartin Matuska 	memset(dn->dn_next_bonuslen, 0, sizeof (dn->dn_next_bonuslen));
148da5137abSMartin Matuska 	memset(dn->dn_next_blksz, 0, sizeof (dn->dn_next_blksz));
149da5137abSMartin Matuska 	memset(dn->dn_next_maxblkid, 0, sizeof (dn->dn_next_maxblkid));
150eda14cbcSMatt Macy 
151e92ffd9bSMartin Matuska 	for (int i = 0; i < TXG_SIZE; i++) {
152eda14cbcSMatt Macy 		multilist_link_init(&dn->dn_dirty_link[i]);
153eda14cbcSMatt Macy 		dn->dn_free_ranges[i] = NULL;
154eda14cbcSMatt Macy 		list_create(&dn->dn_dirty_records[i],
155eda14cbcSMatt Macy 		    sizeof (dbuf_dirty_record_t),
156eda14cbcSMatt Macy 		    offsetof(dbuf_dirty_record_t, dr_dirty_node));
157eda14cbcSMatt Macy 	}
158eda14cbcSMatt Macy 
159eda14cbcSMatt Macy 	dn->dn_allocated_txg = 0;
160eda14cbcSMatt Macy 	dn->dn_free_txg = 0;
161eda14cbcSMatt Macy 	dn->dn_assigned_txg = 0;
162eda14cbcSMatt Macy 	dn->dn_dirty_txg = 0;
163eda14cbcSMatt Macy 	dn->dn_dirtyctx = 0;
164eda14cbcSMatt Macy 	dn->dn_dirtyctx_firstset = NULL;
165eda14cbcSMatt Macy 	dn->dn_bonus = NULL;
166eda14cbcSMatt Macy 	dn->dn_have_spill = B_FALSE;
167eda14cbcSMatt Macy 	dn->dn_zio = NULL;
168eda14cbcSMatt Macy 	dn->dn_oldused = 0;
169eda14cbcSMatt Macy 	dn->dn_oldflags = 0;
170eda14cbcSMatt Macy 	dn->dn_olduid = 0;
171eda14cbcSMatt Macy 	dn->dn_oldgid = 0;
172eda14cbcSMatt Macy 	dn->dn_oldprojid = ZFS_DEFAULT_PROJID;
173eda14cbcSMatt Macy 	dn->dn_newuid = 0;
174eda14cbcSMatt Macy 	dn->dn_newgid = 0;
175eda14cbcSMatt Macy 	dn->dn_newprojid = ZFS_DEFAULT_PROJID;
176eda14cbcSMatt Macy 	dn->dn_id_flags = 0;
177eda14cbcSMatt Macy 
178eda14cbcSMatt Macy 	dn->dn_dbufs_count = 0;
179eda14cbcSMatt Macy 	avl_create(&dn->dn_dbufs, dbuf_compare, sizeof (dmu_buf_impl_t),
180eda14cbcSMatt Macy 	    offsetof(dmu_buf_impl_t, db_link));
181eda14cbcSMatt Macy 
182eda14cbcSMatt Macy 	dn->dn_moved = 0;
183eda14cbcSMatt Macy 	return (0);
184eda14cbcSMatt Macy }
185eda14cbcSMatt Macy 
186eda14cbcSMatt Macy static void
187eda14cbcSMatt Macy dnode_dest(void *arg, void *unused)
188eda14cbcSMatt Macy {
189e92ffd9bSMartin Matuska 	(void) unused;
190eda14cbcSMatt Macy 	dnode_t *dn = arg;
191eda14cbcSMatt Macy 
192eda14cbcSMatt Macy 	rw_destroy(&dn->dn_struct_rwlock);
193eda14cbcSMatt Macy 	mutex_destroy(&dn->dn_mtx);
194eda14cbcSMatt Macy 	mutex_destroy(&dn->dn_dbufs_mtx);
195eda14cbcSMatt Macy 	cv_destroy(&dn->dn_notxholds);
196eda14cbcSMatt Macy 	cv_destroy(&dn->dn_nodnholds);
197eda14cbcSMatt Macy 	zfs_refcount_destroy(&dn->dn_holds);
198eda14cbcSMatt Macy 	zfs_refcount_destroy(&dn->dn_tx_holds);
199eda14cbcSMatt Macy 	ASSERT(!list_link_active(&dn->dn_link));
200eda14cbcSMatt Macy 
201e92ffd9bSMartin Matuska 	for (int i = 0; i < TXG_SIZE; i++) {
202eda14cbcSMatt Macy 		ASSERT(!multilist_link_active(&dn->dn_dirty_link[i]));
203eda14cbcSMatt Macy 		ASSERT3P(dn->dn_free_ranges[i], ==, NULL);
204eda14cbcSMatt Macy 		list_destroy(&dn->dn_dirty_records[i]);
205eda14cbcSMatt Macy 		ASSERT0(dn->dn_next_nblkptr[i]);
206eda14cbcSMatt Macy 		ASSERT0(dn->dn_next_nlevels[i]);
207eda14cbcSMatt Macy 		ASSERT0(dn->dn_next_indblkshift[i]);
208eda14cbcSMatt Macy 		ASSERT0(dn->dn_next_bonustype[i]);
209eda14cbcSMatt Macy 		ASSERT0(dn->dn_rm_spillblk[i]);
210eda14cbcSMatt Macy 		ASSERT0(dn->dn_next_bonuslen[i]);
211eda14cbcSMatt Macy 		ASSERT0(dn->dn_next_blksz[i]);
212eda14cbcSMatt Macy 		ASSERT0(dn->dn_next_maxblkid[i]);
213eda14cbcSMatt Macy 	}
214eda14cbcSMatt Macy 
215eda14cbcSMatt Macy 	ASSERT0(dn->dn_allocated_txg);
216eda14cbcSMatt Macy 	ASSERT0(dn->dn_free_txg);
217eda14cbcSMatt Macy 	ASSERT0(dn->dn_assigned_txg);
218eda14cbcSMatt Macy 	ASSERT0(dn->dn_dirty_txg);
219eda14cbcSMatt Macy 	ASSERT0(dn->dn_dirtyctx);
220eda14cbcSMatt Macy 	ASSERT3P(dn->dn_dirtyctx_firstset, ==, NULL);
221eda14cbcSMatt Macy 	ASSERT3P(dn->dn_bonus, ==, NULL);
222eda14cbcSMatt Macy 	ASSERT(!dn->dn_have_spill);
223eda14cbcSMatt Macy 	ASSERT3P(dn->dn_zio, ==, NULL);
224eda14cbcSMatt Macy 	ASSERT0(dn->dn_oldused);
225eda14cbcSMatt Macy 	ASSERT0(dn->dn_oldflags);
226eda14cbcSMatt Macy 	ASSERT0(dn->dn_olduid);
227eda14cbcSMatt Macy 	ASSERT0(dn->dn_oldgid);
228eda14cbcSMatt Macy 	ASSERT0(dn->dn_oldprojid);
229eda14cbcSMatt Macy 	ASSERT0(dn->dn_newuid);
230eda14cbcSMatt Macy 	ASSERT0(dn->dn_newgid);
231eda14cbcSMatt Macy 	ASSERT0(dn->dn_newprojid);
232eda14cbcSMatt Macy 	ASSERT0(dn->dn_id_flags);
233eda14cbcSMatt Macy 
234eda14cbcSMatt Macy 	ASSERT0(dn->dn_dbufs_count);
235eda14cbcSMatt Macy 	avl_destroy(&dn->dn_dbufs);
236eda14cbcSMatt Macy }
237eda14cbcSMatt Macy 
238bb2d13b6SMartin Matuska static int
239bb2d13b6SMartin Matuska dnode_kstats_update(kstat_t *ksp, int rw)
240bb2d13b6SMartin Matuska {
241bb2d13b6SMartin Matuska 	dnode_stats_t *ds = ksp->ks_data;
242bb2d13b6SMartin Matuska 
243bb2d13b6SMartin Matuska 	if (rw == KSTAT_WRITE)
244bb2d13b6SMartin Matuska 		return (EACCES);
245bb2d13b6SMartin Matuska 	ds->dnode_hold_dbuf_hold.value.ui64 =
246bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_hold_dbuf_hold);
247bb2d13b6SMartin Matuska 	ds->dnode_hold_dbuf_read.value.ui64 =
248bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_hold_dbuf_read);
249bb2d13b6SMartin Matuska 	ds->dnode_hold_alloc_hits.value.ui64 =
250bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_hold_alloc_hits);
251bb2d13b6SMartin Matuska 	ds->dnode_hold_alloc_misses.value.ui64 =
252bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_hold_alloc_misses);
253bb2d13b6SMartin Matuska 	ds->dnode_hold_alloc_interior.value.ui64 =
254bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_hold_alloc_interior);
255bb2d13b6SMartin Matuska 	ds->dnode_hold_alloc_lock_retry.value.ui64 =
256bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_hold_alloc_lock_retry);
257bb2d13b6SMartin Matuska 	ds->dnode_hold_alloc_lock_misses.value.ui64 =
258bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_hold_alloc_lock_misses);
259bb2d13b6SMartin Matuska 	ds->dnode_hold_alloc_type_none.value.ui64 =
260bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_hold_alloc_type_none);
261bb2d13b6SMartin Matuska 	ds->dnode_hold_free_hits.value.ui64 =
262bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_hold_free_hits);
263bb2d13b6SMartin Matuska 	ds->dnode_hold_free_misses.value.ui64 =
264bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_hold_free_misses);
265bb2d13b6SMartin Matuska 	ds->dnode_hold_free_lock_misses.value.ui64 =
266bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_hold_free_lock_misses);
267bb2d13b6SMartin Matuska 	ds->dnode_hold_free_lock_retry.value.ui64 =
268bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_hold_free_lock_retry);
269bb2d13b6SMartin Matuska 	ds->dnode_hold_free_refcount.value.ui64 =
270bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_hold_free_refcount);
271bb2d13b6SMartin Matuska 	ds->dnode_hold_free_overflow.value.ui64 =
272bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_hold_free_overflow);
273bb2d13b6SMartin Matuska 	ds->dnode_free_interior_lock_retry.value.ui64 =
274bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_free_interior_lock_retry);
275bb2d13b6SMartin Matuska 	ds->dnode_allocate.value.ui64 =
276bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_allocate);
277bb2d13b6SMartin Matuska 	ds->dnode_reallocate.value.ui64 =
278bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_reallocate);
279bb2d13b6SMartin Matuska 	ds->dnode_buf_evict.value.ui64 =
280bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_buf_evict);
281bb2d13b6SMartin Matuska 	ds->dnode_alloc_next_chunk.value.ui64 =
282bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_alloc_next_chunk);
283bb2d13b6SMartin Matuska 	ds->dnode_alloc_race.value.ui64 =
284bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_alloc_race);
285bb2d13b6SMartin Matuska 	ds->dnode_alloc_next_block.value.ui64 =
286bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_alloc_next_block);
287bb2d13b6SMartin Matuska 	ds->dnode_move_invalid.value.ui64 =
288bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_move_invalid);
289bb2d13b6SMartin Matuska 	ds->dnode_move_recheck1.value.ui64 =
290bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_move_recheck1);
291bb2d13b6SMartin Matuska 	ds->dnode_move_recheck2.value.ui64 =
292bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_move_recheck2);
293bb2d13b6SMartin Matuska 	ds->dnode_move_special.value.ui64 =
294bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_move_special);
295bb2d13b6SMartin Matuska 	ds->dnode_move_handle.value.ui64 =
296bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_move_handle);
297bb2d13b6SMartin Matuska 	ds->dnode_move_rwlock.value.ui64 =
298bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_move_rwlock);
299bb2d13b6SMartin Matuska 	ds->dnode_move_active.value.ui64 =
300bb2d13b6SMartin Matuska 	    wmsum_value(&dnode_sums.dnode_move_active);
301bb2d13b6SMartin Matuska 	return (0);
302bb2d13b6SMartin Matuska }
303bb2d13b6SMartin Matuska 
304eda14cbcSMatt Macy void
305eda14cbcSMatt Macy dnode_init(void)
306eda14cbcSMatt Macy {
307eda14cbcSMatt Macy 	ASSERT(dnode_cache == NULL);
308eda14cbcSMatt Macy 	dnode_cache = kmem_cache_create("dnode_t", sizeof (dnode_t),
309*ce4dcb97SMartin Matuska 	    0, dnode_cons, dnode_dest, NULL, NULL, NULL, KMC_RECLAIMABLE);
310eda14cbcSMatt Macy 	kmem_cache_set_move(dnode_cache, dnode_move);
311eda14cbcSMatt Macy 
312bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_hold_dbuf_hold, 0);
313bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_hold_dbuf_read, 0);
314bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_hold_alloc_hits, 0);
315bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_hold_alloc_misses, 0);
316bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_hold_alloc_interior, 0);
317bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_hold_alloc_lock_retry, 0);
318bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_hold_alloc_lock_misses, 0);
319bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_hold_alloc_type_none, 0);
320bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_hold_free_hits, 0);
321bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_hold_free_misses, 0);
322bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_hold_free_lock_misses, 0);
323bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_hold_free_lock_retry, 0);
324bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_hold_free_refcount, 0);
325bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_hold_free_overflow, 0);
326bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_free_interior_lock_retry, 0);
327bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_allocate, 0);
328bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_reallocate, 0);
329bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_buf_evict, 0);
330bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_alloc_next_chunk, 0);
331bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_alloc_race, 0);
332bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_alloc_next_block, 0);
333bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_move_invalid, 0);
334bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_move_recheck1, 0);
335bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_move_recheck2, 0);
336bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_move_special, 0);
337bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_move_handle, 0);
338bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_move_rwlock, 0);
339bb2d13b6SMartin Matuska 	wmsum_init(&dnode_sums.dnode_move_active, 0);
340bb2d13b6SMartin Matuska 
341eda14cbcSMatt Macy 	dnode_ksp = kstat_create("zfs", 0, "dnodestats", "misc",
342eda14cbcSMatt Macy 	    KSTAT_TYPE_NAMED, sizeof (dnode_stats) / sizeof (kstat_named_t),
343eda14cbcSMatt Macy 	    KSTAT_FLAG_VIRTUAL);
344eda14cbcSMatt Macy 	if (dnode_ksp != NULL) {
345eda14cbcSMatt Macy 		dnode_ksp->ks_data = &dnode_stats;
346bb2d13b6SMartin Matuska 		dnode_ksp->ks_update = dnode_kstats_update;
347eda14cbcSMatt Macy 		kstat_install(dnode_ksp);
348eda14cbcSMatt Macy 	}
349eda14cbcSMatt Macy }
350eda14cbcSMatt Macy 
351eda14cbcSMatt Macy void
352eda14cbcSMatt Macy dnode_fini(void)
353eda14cbcSMatt Macy {
354eda14cbcSMatt Macy 	if (dnode_ksp != NULL) {
355eda14cbcSMatt Macy 		kstat_delete(dnode_ksp);
356eda14cbcSMatt Macy 		dnode_ksp = NULL;
357eda14cbcSMatt Macy 	}
358eda14cbcSMatt Macy 
359bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_hold_dbuf_hold);
360bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_hold_dbuf_read);
361bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_hold_alloc_hits);
362bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_hold_alloc_misses);
363bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_hold_alloc_interior);
364bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_hold_alloc_lock_retry);
365bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_hold_alloc_lock_misses);
366bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_hold_alloc_type_none);
367bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_hold_free_hits);
368bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_hold_free_misses);
369bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_hold_free_lock_misses);
370bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_hold_free_lock_retry);
371bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_hold_free_refcount);
372bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_hold_free_overflow);
373bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_free_interior_lock_retry);
374bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_allocate);
375bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_reallocate);
376bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_buf_evict);
377bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_alloc_next_chunk);
378bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_alloc_race);
379bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_alloc_next_block);
380bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_move_invalid);
381bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_move_recheck1);
382bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_move_recheck2);
383bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_move_special);
384bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_move_handle);
385bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_move_rwlock);
386bb2d13b6SMartin Matuska 	wmsum_fini(&dnode_sums.dnode_move_active);
387bb2d13b6SMartin Matuska 
388eda14cbcSMatt Macy 	kmem_cache_destroy(dnode_cache);
389eda14cbcSMatt Macy 	dnode_cache = NULL;
390eda14cbcSMatt Macy }
391eda14cbcSMatt Macy 
392eda14cbcSMatt Macy 
393eda14cbcSMatt Macy #ifdef ZFS_DEBUG
394eda14cbcSMatt Macy void
395eda14cbcSMatt Macy dnode_verify(dnode_t *dn)
396eda14cbcSMatt Macy {
397eda14cbcSMatt Macy 	int drop_struct_lock = FALSE;
398eda14cbcSMatt Macy 
399eda14cbcSMatt Macy 	ASSERT(dn->dn_phys);
400eda14cbcSMatt Macy 	ASSERT(dn->dn_objset);
401eda14cbcSMatt Macy 	ASSERT(dn->dn_handle->dnh_dnode == dn);
402eda14cbcSMatt Macy 
403eda14cbcSMatt Macy 	ASSERT(DMU_OT_IS_VALID(dn->dn_phys->dn_type));
404eda14cbcSMatt Macy 
405eda14cbcSMatt Macy 	if (!(zfs_flags & ZFS_DEBUG_DNODE_VERIFY))
406eda14cbcSMatt Macy 		return;
407eda14cbcSMatt Macy 
408eda14cbcSMatt Macy 	if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) {
409eda14cbcSMatt Macy 		rw_enter(&dn->dn_struct_rwlock, RW_READER);
410eda14cbcSMatt Macy 		drop_struct_lock = TRUE;
411eda14cbcSMatt Macy 	}
412eda14cbcSMatt Macy 	if (dn->dn_phys->dn_type != DMU_OT_NONE || dn->dn_allocated_txg != 0) {
413eda14cbcSMatt Macy 		int i;
414eda14cbcSMatt Macy 		int max_bonuslen = DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots);
415eda14cbcSMatt Macy 		ASSERT3U(dn->dn_indblkshift, <=, SPA_MAXBLOCKSHIFT);
416eda14cbcSMatt Macy 		if (dn->dn_datablkshift) {
417eda14cbcSMatt Macy 			ASSERT3U(dn->dn_datablkshift, >=, SPA_MINBLOCKSHIFT);
418eda14cbcSMatt Macy 			ASSERT3U(dn->dn_datablkshift, <=, SPA_MAXBLOCKSHIFT);
419eda14cbcSMatt Macy 			ASSERT3U(1<<dn->dn_datablkshift, ==, dn->dn_datablksz);
420eda14cbcSMatt Macy 		}
421eda14cbcSMatt Macy 		ASSERT3U(dn->dn_nlevels, <=, 30);
422eda14cbcSMatt Macy 		ASSERT(DMU_OT_IS_VALID(dn->dn_type));
423eda14cbcSMatt Macy 		ASSERT3U(dn->dn_nblkptr, >=, 1);
424eda14cbcSMatt Macy 		ASSERT3U(dn->dn_nblkptr, <=, DN_MAX_NBLKPTR);
425eda14cbcSMatt Macy 		ASSERT3U(dn->dn_bonuslen, <=, max_bonuslen);
426eda14cbcSMatt Macy 		ASSERT3U(dn->dn_datablksz, ==,
427eda14cbcSMatt Macy 		    dn->dn_datablkszsec << SPA_MINBLOCKSHIFT);
428eda14cbcSMatt Macy 		ASSERT3U(ISP2(dn->dn_datablksz), ==, dn->dn_datablkshift != 0);
429eda14cbcSMatt Macy 		ASSERT3U((dn->dn_nblkptr - 1) * sizeof (blkptr_t) +
430eda14cbcSMatt Macy 		    dn->dn_bonuslen, <=, max_bonuslen);
431eda14cbcSMatt Macy 		for (i = 0; i < TXG_SIZE; i++) {
432eda14cbcSMatt Macy 			ASSERT3U(dn->dn_next_nlevels[i], <=, dn->dn_nlevels);
433eda14cbcSMatt Macy 		}
434eda14cbcSMatt Macy 	}
435eda14cbcSMatt Macy 	if (dn->dn_phys->dn_type != DMU_OT_NONE)
436eda14cbcSMatt Macy 		ASSERT3U(dn->dn_phys->dn_nlevels, <=, dn->dn_nlevels);
437eda14cbcSMatt Macy 	ASSERT(DMU_OBJECT_IS_SPECIAL(dn->dn_object) || dn->dn_dbuf != NULL);
438eda14cbcSMatt Macy 	if (dn->dn_dbuf != NULL) {
439eda14cbcSMatt Macy 		ASSERT3P(dn->dn_phys, ==,
440eda14cbcSMatt Macy 		    (dnode_phys_t *)dn->dn_dbuf->db.db_data +
441eda14cbcSMatt Macy 		    (dn->dn_object % (dn->dn_dbuf->db.db_size >> DNODE_SHIFT)));
442eda14cbcSMatt Macy 	}
443eda14cbcSMatt Macy 	if (drop_struct_lock)
444eda14cbcSMatt Macy 		rw_exit(&dn->dn_struct_rwlock);
445eda14cbcSMatt Macy }
446eda14cbcSMatt Macy #endif
447eda14cbcSMatt Macy 
448eda14cbcSMatt Macy void
449eda14cbcSMatt Macy dnode_byteswap(dnode_phys_t *dnp)
450eda14cbcSMatt Macy {
451eda14cbcSMatt Macy 	uint64_t *buf64 = (void*)&dnp->dn_blkptr;
452eda14cbcSMatt Macy 	int i;
453eda14cbcSMatt Macy 
454eda14cbcSMatt Macy 	if (dnp->dn_type == DMU_OT_NONE) {
455da5137abSMartin Matuska 		memset(dnp, 0, sizeof (dnode_phys_t));
456eda14cbcSMatt Macy 		return;
457eda14cbcSMatt Macy 	}
458eda14cbcSMatt Macy 
459eda14cbcSMatt Macy 	dnp->dn_datablkszsec = BSWAP_16(dnp->dn_datablkszsec);
460eda14cbcSMatt Macy 	dnp->dn_bonuslen = BSWAP_16(dnp->dn_bonuslen);
461eda14cbcSMatt Macy 	dnp->dn_extra_slots = BSWAP_8(dnp->dn_extra_slots);
462eda14cbcSMatt Macy 	dnp->dn_maxblkid = BSWAP_64(dnp->dn_maxblkid);
463eda14cbcSMatt Macy 	dnp->dn_used = BSWAP_64(dnp->dn_used);
464eda14cbcSMatt Macy 
465eda14cbcSMatt Macy 	/*
466eda14cbcSMatt Macy 	 * dn_nblkptr is only one byte, so it's OK to read it in either
467eda14cbcSMatt Macy 	 * byte order.  We can't read dn_bouslen.
468eda14cbcSMatt Macy 	 */
469eda14cbcSMatt Macy 	ASSERT(dnp->dn_indblkshift <= SPA_MAXBLOCKSHIFT);
470eda14cbcSMatt Macy 	ASSERT(dnp->dn_nblkptr <= DN_MAX_NBLKPTR);
471eda14cbcSMatt Macy 	for (i = 0; i < dnp->dn_nblkptr * sizeof (blkptr_t)/8; i++)
472eda14cbcSMatt Macy 		buf64[i] = BSWAP_64(buf64[i]);
473eda14cbcSMatt Macy 
474eda14cbcSMatt Macy 	/*
475eda14cbcSMatt Macy 	 * OK to check dn_bonuslen for zero, because it won't matter if
476eda14cbcSMatt Macy 	 * we have the wrong byte order.  This is necessary because the
477eda14cbcSMatt Macy 	 * dnode dnode is smaller than a regular dnode.
478eda14cbcSMatt Macy 	 */
479eda14cbcSMatt Macy 	if (dnp->dn_bonuslen != 0) {
480eda14cbcSMatt Macy 		dmu_object_byteswap_t byteswap;
481eda14cbcSMatt Macy 		ASSERT(DMU_OT_IS_VALID(dnp->dn_bonustype));
482eda14cbcSMatt Macy 		byteswap = DMU_OT_BYTESWAP(dnp->dn_bonustype);
483a0b956f5SMartin Matuska 		dmu_ot_byteswap[byteswap].ob_func(DN_BONUS(dnp),
484a0b956f5SMartin Matuska 		    DN_MAX_BONUS_LEN(dnp));
485eda14cbcSMatt Macy 	}
486eda14cbcSMatt Macy 
487eda14cbcSMatt Macy 	/* Swap SPILL block if we have one */
488eda14cbcSMatt Macy 	if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR)
489eda14cbcSMatt Macy 		byteswap_uint64_array(DN_SPILL_BLKPTR(dnp), sizeof (blkptr_t));
490eda14cbcSMatt Macy }
491eda14cbcSMatt Macy 
492eda14cbcSMatt Macy void
493eda14cbcSMatt Macy dnode_buf_byteswap(void *vbuf, size_t size)
494eda14cbcSMatt Macy {
495eda14cbcSMatt Macy 	int i = 0;
496eda14cbcSMatt Macy 
497eda14cbcSMatt Macy 	ASSERT3U(sizeof (dnode_phys_t), ==, (1<<DNODE_SHIFT));
498eda14cbcSMatt Macy 	ASSERT((size & (sizeof (dnode_phys_t)-1)) == 0);
499eda14cbcSMatt Macy 
500eda14cbcSMatt Macy 	while (i < size) {
501eda14cbcSMatt Macy 		dnode_phys_t *dnp = (void *)(((char *)vbuf) + i);
502eda14cbcSMatt Macy 		dnode_byteswap(dnp);
503eda14cbcSMatt Macy 
504eda14cbcSMatt Macy 		i += DNODE_MIN_SIZE;
505eda14cbcSMatt Macy 		if (dnp->dn_type != DMU_OT_NONE)
506eda14cbcSMatt Macy 			i += dnp->dn_extra_slots * DNODE_MIN_SIZE;
507eda14cbcSMatt Macy 	}
508eda14cbcSMatt Macy }
509eda14cbcSMatt Macy 
510eda14cbcSMatt Macy void
511eda14cbcSMatt Macy dnode_setbonuslen(dnode_t *dn, int newsize, dmu_tx_t *tx)
512eda14cbcSMatt Macy {
513eda14cbcSMatt Macy 	ASSERT3U(zfs_refcount_count(&dn->dn_holds), >=, 1);
514eda14cbcSMatt Macy 
515eda14cbcSMatt Macy 	dnode_setdirty(dn, tx);
516eda14cbcSMatt Macy 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
517eda14cbcSMatt Macy 	ASSERT3U(newsize, <=, DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) -
518eda14cbcSMatt Macy 	    (dn->dn_nblkptr-1) * sizeof (blkptr_t));
519eda14cbcSMatt Macy 
520eda14cbcSMatt Macy 	if (newsize < dn->dn_bonuslen) {
521eda14cbcSMatt Macy 		/* clear any data after the end of the new size */
522eda14cbcSMatt Macy 		size_t diff = dn->dn_bonuslen - newsize;
523eda14cbcSMatt Macy 		char *data_end = ((char *)dn->dn_bonus->db.db_data) + newsize;
524da5137abSMartin Matuska 		memset(data_end, 0, diff);
525eda14cbcSMatt Macy 	}
526eda14cbcSMatt Macy 
527eda14cbcSMatt Macy 	dn->dn_bonuslen = newsize;
528eda14cbcSMatt Macy 	if (newsize == 0)
529eda14cbcSMatt Macy 		dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = DN_ZERO_BONUSLEN;
530eda14cbcSMatt Macy 	else
531eda14cbcSMatt Macy 		dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = dn->dn_bonuslen;
532eda14cbcSMatt Macy 	rw_exit(&dn->dn_struct_rwlock);
533eda14cbcSMatt Macy }
534eda14cbcSMatt Macy 
535eda14cbcSMatt Macy void
536eda14cbcSMatt Macy dnode_setbonus_type(dnode_t *dn, dmu_object_type_t newtype, dmu_tx_t *tx)
537eda14cbcSMatt Macy {
538eda14cbcSMatt Macy 	ASSERT3U(zfs_refcount_count(&dn->dn_holds), >=, 1);
539eda14cbcSMatt Macy 	dnode_setdirty(dn, tx);
540eda14cbcSMatt Macy 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
541eda14cbcSMatt Macy 	dn->dn_bonustype = newtype;
542eda14cbcSMatt Macy 	dn->dn_next_bonustype[tx->tx_txg & TXG_MASK] = dn->dn_bonustype;
543eda14cbcSMatt Macy 	rw_exit(&dn->dn_struct_rwlock);
544eda14cbcSMatt Macy }
545eda14cbcSMatt Macy 
546eda14cbcSMatt Macy void
547*ce4dcb97SMartin Matuska dnode_set_storage_type(dnode_t *dn, dmu_object_type_t newtype)
548*ce4dcb97SMartin Matuska {
549*ce4dcb97SMartin Matuska 	/*
550*ce4dcb97SMartin Matuska 	 * This is not in the dnode_phys, but it should be, and perhaps one day
551*ce4dcb97SMartin Matuska 	 * will. For now we require it be set after taking a hold.
552*ce4dcb97SMartin Matuska 	 */
553*ce4dcb97SMartin Matuska 	ASSERT3U(zfs_refcount_count(&dn->dn_holds), >=, 1);
554*ce4dcb97SMartin Matuska 	dn->dn_storage_type = newtype;
555*ce4dcb97SMartin Matuska }
556*ce4dcb97SMartin Matuska 
557*ce4dcb97SMartin Matuska void
558eda14cbcSMatt Macy dnode_rm_spill(dnode_t *dn, dmu_tx_t *tx)
559eda14cbcSMatt Macy {
560eda14cbcSMatt Macy 	ASSERT3U(zfs_refcount_count(&dn->dn_holds), >=, 1);
561eda14cbcSMatt Macy 	ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock));
562eda14cbcSMatt Macy 	dnode_setdirty(dn, tx);
563eda14cbcSMatt Macy 	dn->dn_rm_spillblk[tx->tx_txg & TXG_MASK] = DN_KILL_SPILLBLK;
564eda14cbcSMatt Macy 	dn->dn_have_spill = B_FALSE;
565eda14cbcSMatt Macy }
566eda14cbcSMatt Macy 
567eda14cbcSMatt Macy static void
568eda14cbcSMatt Macy dnode_setdblksz(dnode_t *dn, int size)
569eda14cbcSMatt Macy {
570eda14cbcSMatt Macy 	ASSERT0(P2PHASE(size, SPA_MINBLOCKSIZE));
571eda14cbcSMatt Macy 	ASSERT3U(size, <=, SPA_MAXBLOCKSIZE);
572eda14cbcSMatt Macy 	ASSERT3U(size, >=, SPA_MINBLOCKSIZE);
573eda14cbcSMatt Macy 	ASSERT3U(size >> SPA_MINBLOCKSHIFT, <,
574eda14cbcSMatt Macy 	    1<<(sizeof (dn->dn_phys->dn_datablkszsec) * 8));
575eda14cbcSMatt Macy 	dn->dn_datablksz = size;
576eda14cbcSMatt Macy 	dn->dn_datablkszsec = size >> SPA_MINBLOCKSHIFT;
577eda14cbcSMatt Macy 	dn->dn_datablkshift = ISP2(size) ? highbit64(size - 1) : 0;
578eda14cbcSMatt Macy }
579eda14cbcSMatt Macy 
580eda14cbcSMatt Macy static dnode_t *
581eda14cbcSMatt Macy dnode_create(objset_t *os, dnode_phys_t *dnp, dmu_buf_impl_t *db,
582eda14cbcSMatt Macy     uint64_t object, dnode_handle_t *dnh)
583eda14cbcSMatt Macy {
584eda14cbcSMatt Macy 	dnode_t *dn;
585eda14cbcSMatt Macy 
586eda14cbcSMatt Macy 	dn = kmem_cache_alloc(dnode_cache, KM_SLEEP);
587eda14cbcSMatt Macy 	dn->dn_moved = 0;
588eda14cbcSMatt Macy 
589eda14cbcSMatt Macy 	/*
590eda14cbcSMatt Macy 	 * Defer setting dn_objset until the dnode is ready to be a candidate
591eda14cbcSMatt Macy 	 * for the dnode_move() callback.
592eda14cbcSMatt Macy 	 */
593eda14cbcSMatt Macy 	dn->dn_object = object;
594eda14cbcSMatt Macy 	dn->dn_dbuf = db;
595eda14cbcSMatt Macy 	dn->dn_handle = dnh;
596eda14cbcSMatt Macy 	dn->dn_phys = dnp;
597eda14cbcSMatt Macy 
598eda14cbcSMatt Macy 	if (dnp->dn_datablkszsec) {
599eda14cbcSMatt Macy 		dnode_setdblksz(dn, dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT);
600eda14cbcSMatt Macy 	} else {
601eda14cbcSMatt Macy 		dn->dn_datablksz = 0;
602eda14cbcSMatt Macy 		dn->dn_datablkszsec = 0;
603eda14cbcSMatt Macy 		dn->dn_datablkshift = 0;
604eda14cbcSMatt Macy 	}
605eda14cbcSMatt Macy 	dn->dn_indblkshift = dnp->dn_indblkshift;
606eda14cbcSMatt Macy 	dn->dn_nlevels = dnp->dn_nlevels;
607eda14cbcSMatt Macy 	dn->dn_type = dnp->dn_type;
608eda14cbcSMatt Macy 	dn->dn_nblkptr = dnp->dn_nblkptr;
609eda14cbcSMatt Macy 	dn->dn_checksum = dnp->dn_checksum;
610eda14cbcSMatt Macy 	dn->dn_compress = dnp->dn_compress;
611eda14cbcSMatt Macy 	dn->dn_bonustype = dnp->dn_bonustype;
612eda14cbcSMatt Macy 	dn->dn_bonuslen = dnp->dn_bonuslen;
613eda14cbcSMatt Macy 	dn->dn_num_slots = dnp->dn_extra_slots + 1;
614eda14cbcSMatt Macy 	dn->dn_maxblkid = dnp->dn_maxblkid;
615eda14cbcSMatt Macy 	dn->dn_have_spill = ((dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) != 0);
616eda14cbcSMatt Macy 	dn->dn_id_flags = 0;
617eda14cbcSMatt Macy 
618*ce4dcb97SMartin Matuska 	dn->dn_storage_type = DMU_OT_NONE;
619*ce4dcb97SMartin Matuska 
620eda14cbcSMatt Macy 	dmu_zfetch_init(&dn->dn_zfetch, dn);
621eda14cbcSMatt Macy 
622eda14cbcSMatt Macy 	ASSERT(DMU_OT_IS_VALID(dn->dn_phys->dn_type));
623eda14cbcSMatt Macy 	ASSERT(zrl_is_locked(&dnh->dnh_zrlock));
624eda14cbcSMatt Macy 	ASSERT(!DN_SLOT_IS_PTR(dnh->dnh_dnode));
625eda14cbcSMatt Macy 
626eda14cbcSMatt Macy 	mutex_enter(&os->os_lock);
627eda14cbcSMatt Macy 
628eda14cbcSMatt Macy 	/*
629eda14cbcSMatt Macy 	 * Exclude special dnodes from os_dnodes so an empty os_dnodes
630eda14cbcSMatt Macy 	 * signifies that the special dnodes have no references from
631eda14cbcSMatt Macy 	 * their children (the entries in os_dnodes).  This allows
632eda14cbcSMatt Macy 	 * dnode_destroy() to easily determine if the last child has
633eda14cbcSMatt Macy 	 * been removed and then complete eviction of the objset.
634eda14cbcSMatt Macy 	 */
635eda14cbcSMatt Macy 	if (!DMU_OBJECT_IS_SPECIAL(object))
636eda14cbcSMatt Macy 		list_insert_head(&os->os_dnodes, dn);
637eda14cbcSMatt Macy 	membar_producer();
638eda14cbcSMatt Macy 
639eda14cbcSMatt Macy 	/*
640eda14cbcSMatt Macy 	 * Everything else must be valid before assigning dn_objset
641eda14cbcSMatt Macy 	 * makes the dnode eligible for dnode_move().
642eda14cbcSMatt Macy 	 */
643eda14cbcSMatt Macy 	dn->dn_objset = os;
644eda14cbcSMatt Macy 
645eda14cbcSMatt Macy 	dnh->dnh_dnode = dn;
646eda14cbcSMatt Macy 	mutex_exit(&os->os_lock);
647eda14cbcSMatt Macy 
648eda14cbcSMatt Macy 	arc_space_consume(sizeof (dnode_t), ARC_SPACE_DNODE);
649eda14cbcSMatt Macy 
650eda14cbcSMatt Macy 	return (dn);
651eda14cbcSMatt Macy }
652eda14cbcSMatt Macy 
653eda14cbcSMatt Macy /*
654eda14cbcSMatt Macy  * Caller must be holding the dnode handle, which is released upon return.
655eda14cbcSMatt Macy  */
656eda14cbcSMatt Macy static void
657eda14cbcSMatt Macy dnode_destroy(dnode_t *dn)
658eda14cbcSMatt Macy {
659eda14cbcSMatt Macy 	objset_t *os = dn->dn_objset;
660eda14cbcSMatt Macy 	boolean_t complete_os_eviction = B_FALSE;
661eda14cbcSMatt Macy 
662eda14cbcSMatt Macy 	ASSERT((dn->dn_id_flags & DN_ID_NEW_EXIST) == 0);
663eda14cbcSMatt Macy 
664eda14cbcSMatt Macy 	mutex_enter(&os->os_lock);
665eda14cbcSMatt Macy 	POINTER_INVALIDATE(&dn->dn_objset);
666eda14cbcSMatt Macy 	if (!DMU_OBJECT_IS_SPECIAL(dn->dn_object)) {
667eda14cbcSMatt Macy 		list_remove(&os->os_dnodes, dn);
668eda14cbcSMatt Macy 		complete_os_eviction =
669eda14cbcSMatt Macy 		    list_is_empty(&os->os_dnodes) &&
670eda14cbcSMatt Macy 		    list_link_active(&os->os_evicting_node);
671eda14cbcSMatt Macy 	}
672eda14cbcSMatt Macy 	mutex_exit(&os->os_lock);
673eda14cbcSMatt Macy 
674eda14cbcSMatt Macy 	/* the dnode can no longer move, so we can release the handle */
675eda14cbcSMatt Macy 	if (!zrl_is_locked(&dn->dn_handle->dnh_zrlock))
676eda14cbcSMatt Macy 		zrl_remove(&dn->dn_handle->dnh_zrlock);
677eda14cbcSMatt Macy 
678eda14cbcSMatt Macy 	dn->dn_allocated_txg = 0;
679eda14cbcSMatt Macy 	dn->dn_free_txg = 0;
680eda14cbcSMatt Macy 	dn->dn_assigned_txg = 0;
681eda14cbcSMatt Macy 	dn->dn_dirty_txg = 0;
682eda14cbcSMatt Macy 
683eda14cbcSMatt Macy 	dn->dn_dirtyctx = 0;
684eda14cbcSMatt Macy 	dn->dn_dirtyctx_firstset = NULL;
685eda14cbcSMatt Macy 	if (dn->dn_bonus != NULL) {
686eda14cbcSMatt Macy 		mutex_enter(&dn->dn_bonus->db_mtx);
687eda14cbcSMatt Macy 		dbuf_destroy(dn->dn_bonus);
688eda14cbcSMatt Macy 		dn->dn_bonus = NULL;
689eda14cbcSMatt Macy 	}
690eda14cbcSMatt Macy 	dn->dn_zio = NULL;
691eda14cbcSMatt Macy 
692eda14cbcSMatt Macy 	dn->dn_have_spill = B_FALSE;
693eda14cbcSMatt Macy 	dn->dn_oldused = 0;
694eda14cbcSMatt Macy 	dn->dn_oldflags = 0;
695eda14cbcSMatt Macy 	dn->dn_olduid = 0;
696eda14cbcSMatt Macy 	dn->dn_oldgid = 0;
697eda14cbcSMatt Macy 	dn->dn_oldprojid = ZFS_DEFAULT_PROJID;
698eda14cbcSMatt Macy 	dn->dn_newuid = 0;
699eda14cbcSMatt Macy 	dn->dn_newgid = 0;
700eda14cbcSMatt Macy 	dn->dn_newprojid = ZFS_DEFAULT_PROJID;
701eda14cbcSMatt Macy 	dn->dn_id_flags = 0;
702eda14cbcSMatt Macy 
703*ce4dcb97SMartin Matuska 	dn->dn_storage_type = DMU_OT_NONE;
704*ce4dcb97SMartin Matuska 
705eda14cbcSMatt Macy 	dmu_zfetch_fini(&dn->dn_zfetch);
706eda14cbcSMatt Macy 	kmem_cache_free(dnode_cache, dn);
707eda14cbcSMatt Macy 	arc_space_return(sizeof (dnode_t), ARC_SPACE_DNODE);
708eda14cbcSMatt Macy 
709eda14cbcSMatt Macy 	if (complete_os_eviction)
710eda14cbcSMatt Macy 		dmu_objset_evict_done(os);
711eda14cbcSMatt Macy }
712eda14cbcSMatt Macy 
713eda14cbcSMatt Macy void
714eda14cbcSMatt Macy dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs,
715eda14cbcSMatt Macy     dmu_object_type_t bonustype, int bonuslen, int dn_slots, dmu_tx_t *tx)
716eda14cbcSMatt Macy {
717eda14cbcSMatt Macy 	int i;
718eda14cbcSMatt Macy 
719eda14cbcSMatt Macy 	ASSERT3U(dn_slots, >, 0);
720eda14cbcSMatt Macy 	ASSERT3U(dn_slots << DNODE_SHIFT, <=,
721eda14cbcSMatt Macy 	    spa_maxdnodesize(dmu_objset_spa(dn->dn_objset)));
722eda14cbcSMatt Macy 	ASSERT3U(blocksize, <=,
723eda14cbcSMatt Macy 	    spa_maxblocksize(dmu_objset_spa(dn->dn_objset)));
724eda14cbcSMatt Macy 	if (blocksize == 0)
725eda14cbcSMatt Macy 		blocksize = 1 << zfs_default_bs;
726eda14cbcSMatt Macy 	else
727eda14cbcSMatt Macy 		blocksize = P2ROUNDUP(blocksize, SPA_MINBLOCKSIZE);
728eda14cbcSMatt Macy 
729eda14cbcSMatt Macy 	if (ibs == 0)
730eda14cbcSMatt Macy 		ibs = zfs_default_ibs;
731eda14cbcSMatt Macy 
732eda14cbcSMatt Macy 	ibs = MIN(MAX(ibs, DN_MIN_INDBLKSHIFT), DN_MAX_INDBLKSHIFT);
733eda14cbcSMatt Macy 
734eda14cbcSMatt Macy 	dprintf("os=%p obj=%llu txg=%llu blocksize=%d ibs=%d dn_slots=%d\n",
73533b8c039SMartin Matuska 	    dn->dn_objset, (u_longlong_t)dn->dn_object,
73633b8c039SMartin Matuska 	    (u_longlong_t)tx->tx_txg, blocksize, ibs, dn_slots);
737eda14cbcSMatt Macy 	DNODE_STAT_BUMP(dnode_allocate);
738eda14cbcSMatt Macy 
739eda14cbcSMatt Macy 	ASSERT(dn->dn_type == DMU_OT_NONE);
740da5137abSMartin Matuska 	ASSERT0(memcmp(dn->dn_phys, &dnode_phys_zero, sizeof (dnode_phys_t)));
741eda14cbcSMatt Macy 	ASSERT(dn->dn_phys->dn_type == DMU_OT_NONE);
742eda14cbcSMatt Macy 	ASSERT(ot != DMU_OT_NONE);
743eda14cbcSMatt Macy 	ASSERT(DMU_OT_IS_VALID(ot));
744eda14cbcSMatt Macy 	ASSERT((bonustype == DMU_OT_NONE && bonuslen == 0) ||
745eda14cbcSMatt Macy 	    (bonustype == DMU_OT_SA && bonuslen == 0) ||
7462ad756a6SMartin Matuska 	    (bonustype == DMU_OTN_UINT64_METADATA && bonuslen == 0) ||
747eda14cbcSMatt Macy 	    (bonustype != DMU_OT_NONE && bonuslen != 0));
748eda14cbcSMatt Macy 	ASSERT(DMU_OT_IS_VALID(bonustype));
749eda14cbcSMatt Macy 	ASSERT3U(bonuslen, <=, DN_SLOTS_TO_BONUSLEN(dn_slots));
750eda14cbcSMatt Macy 	ASSERT(dn->dn_type == DMU_OT_NONE);
751eda14cbcSMatt Macy 	ASSERT0(dn->dn_maxblkid);
752eda14cbcSMatt Macy 	ASSERT0(dn->dn_allocated_txg);
753eda14cbcSMatt Macy 	ASSERT0(dn->dn_assigned_txg);
754eda14cbcSMatt Macy 	ASSERT(zfs_refcount_is_zero(&dn->dn_tx_holds));
755eda14cbcSMatt Macy 	ASSERT3U(zfs_refcount_count(&dn->dn_holds), <=, 1);
756eda14cbcSMatt Macy 	ASSERT(avl_is_empty(&dn->dn_dbufs));
757eda14cbcSMatt Macy 
758eda14cbcSMatt Macy 	for (i = 0; i < TXG_SIZE; i++) {
759eda14cbcSMatt Macy 		ASSERT0(dn->dn_next_nblkptr[i]);
760eda14cbcSMatt Macy 		ASSERT0(dn->dn_next_nlevels[i]);
761eda14cbcSMatt Macy 		ASSERT0(dn->dn_next_indblkshift[i]);
762eda14cbcSMatt Macy 		ASSERT0(dn->dn_next_bonuslen[i]);
763eda14cbcSMatt Macy 		ASSERT0(dn->dn_next_bonustype[i]);
764eda14cbcSMatt Macy 		ASSERT0(dn->dn_rm_spillblk[i]);
765eda14cbcSMatt Macy 		ASSERT0(dn->dn_next_blksz[i]);
766eda14cbcSMatt Macy 		ASSERT0(dn->dn_next_maxblkid[i]);
767eda14cbcSMatt Macy 		ASSERT(!multilist_link_active(&dn->dn_dirty_link[i]));
768eda14cbcSMatt Macy 		ASSERT3P(list_head(&dn->dn_dirty_records[i]), ==, NULL);
769eda14cbcSMatt Macy 		ASSERT3P(dn->dn_free_ranges[i], ==, NULL);
770eda14cbcSMatt Macy 	}
771eda14cbcSMatt Macy 
772eda14cbcSMatt Macy 	dn->dn_type = ot;
773eda14cbcSMatt Macy 	dnode_setdblksz(dn, blocksize);
774eda14cbcSMatt Macy 	dn->dn_indblkshift = ibs;
775eda14cbcSMatt Macy 	dn->dn_nlevels = 1;
776eda14cbcSMatt Macy 	dn->dn_num_slots = dn_slots;
777eda14cbcSMatt Macy 	if (bonustype == DMU_OT_SA) /* Maximize bonus space for SA */
778eda14cbcSMatt Macy 		dn->dn_nblkptr = 1;
779eda14cbcSMatt Macy 	else {
780eda14cbcSMatt Macy 		dn->dn_nblkptr = MIN(DN_MAX_NBLKPTR,
781eda14cbcSMatt Macy 		    1 + ((DN_SLOTS_TO_BONUSLEN(dn_slots) - bonuslen) >>
782eda14cbcSMatt Macy 		    SPA_BLKPTRSHIFT));
783eda14cbcSMatt Macy 	}
784eda14cbcSMatt Macy 
785eda14cbcSMatt Macy 	dn->dn_bonustype = bonustype;
786eda14cbcSMatt Macy 	dn->dn_bonuslen = bonuslen;
787eda14cbcSMatt Macy 	dn->dn_checksum = ZIO_CHECKSUM_INHERIT;
788eda14cbcSMatt Macy 	dn->dn_compress = ZIO_COMPRESS_INHERIT;
789eda14cbcSMatt Macy 	dn->dn_dirtyctx = 0;
790eda14cbcSMatt Macy 
791eda14cbcSMatt Macy 	dn->dn_free_txg = 0;
792eda14cbcSMatt Macy 	dn->dn_dirtyctx_firstset = NULL;
7937877fdebSMatt Macy 	dn->dn_dirty_txg = 0;
794eda14cbcSMatt Macy 
795eda14cbcSMatt Macy 	dn->dn_allocated_txg = tx->tx_txg;
796eda14cbcSMatt Macy 	dn->dn_id_flags = 0;
797eda14cbcSMatt Macy 
798eda14cbcSMatt Macy 	dnode_setdirty(dn, tx);
799eda14cbcSMatt Macy 	dn->dn_next_indblkshift[tx->tx_txg & TXG_MASK] = ibs;
800eda14cbcSMatt Macy 	dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = dn->dn_bonuslen;
801eda14cbcSMatt Macy 	dn->dn_next_bonustype[tx->tx_txg & TXG_MASK] = dn->dn_bonustype;
802eda14cbcSMatt Macy 	dn->dn_next_blksz[tx->tx_txg & TXG_MASK] = dn->dn_datablksz;
803eda14cbcSMatt Macy }
804eda14cbcSMatt Macy 
805eda14cbcSMatt Macy void
806eda14cbcSMatt Macy dnode_reallocate(dnode_t *dn, dmu_object_type_t ot, int blocksize,
807eda14cbcSMatt Macy     dmu_object_type_t bonustype, int bonuslen, int dn_slots,
808eda14cbcSMatt Macy     boolean_t keep_spill, dmu_tx_t *tx)
809eda14cbcSMatt Macy {
810eda14cbcSMatt Macy 	int nblkptr;
811eda14cbcSMatt Macy 
812eda14cbcSMatt Macy 	ASSERT3U(blocksize, >=, SPA_MINBLOCKSIZE);
813eda14cbcSMatt Macy 	ASSERT3U(blocksize, <=,
814eda14cbcSMatt Macy 	    spa_maxblocksize(dmu_objset_spa(dn->dn_objset)));
815eda14cbcSMatt Macy 	ASSERT0(blocksize % SPA_MINBLOCKSIZE);
816eda14cbcSMatt Macy 	ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT || dmu_tx_private_ok(tx));
817eda14cbcSMatt Macy 	ASSERT(tx->tx_txg != 0);
818eda14cbcSMatt Macy 	ASSERT((bonustype == DMU_OT_NONE && bonuslen == 0) ||
819eda14cbcSMatt Macy 	    (bonustype != DMU_OT_NONE && bonuslen != 0) ||
820eda14cbcSMatt Macy 	    (bonustype == DMU_OT_SA && bonuslen == 0));
821eda14cbcSMatt Macy 	ASSERT(DMU_OT_IS_VALID(bonustype));
822eda14cbcSMatt Macy 	ASSERT3U(bonuslen, <=,
823eda14cbcSMatt Macy 	    DN_BONUS_SIZE(spa_maxdnodesize(dmu_objset_spa(dn->dn_objset))));
824eda14cbcSMatt Macy 	ASSERT3U(bonuslen, <=, DN_BONUS_SIZE(dn_slots << DNODE_SHIFT));
825eda14cbcSMatt Macy 
826eda14cbcSMatt Macy 	dnode_free_interior_slots(dn);
827eda14cbcSMatt Macy 	DNODE_STAT_BUMP(dnode_reallocate);
828eda14cbcSMatt Macy 
829eda14cbcSMatt Macy 	/* clean up any unreferenced dbufs */
830eda14cbcSMatt Macy 	dnode_evict_dbufs(dn);
831eda14cbcSMatt Macy 
832eda14cbcSMatt Macy 	dn->dn_id_flags = 0;
833eda14cbcSMatt Macy 
834eda14cbcSMatt Macy 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
835eda14cbcSMatt Macy 	dnode_setdirty(dn, tx);
836eda14cbcSMatt Macy 	if (dn->dn_datablksz != blocksize) {
837eda14cbcSMatt Macy 		/* change blocksize */
838eda14cbcSMatt Macy 		ASSERT0(dn->dn_maxblkid);
839eda14cbcSMatt Macy 		ASSERT(BP_IS_HOLE(&dn->dn_phys->dn_blkptr[0]) ||
840eda14cbcSMatt Macy 		    dnode_block_freed(dn, 0));
841eda14cbcSMatt Macy 
842eda14cbcSMatt Macy 		dnode_setdblksz(dn, blocksize);
843eda14cbcSMatt Macy 		dn->dn_next_blksz[tx->tx_txg & TXG_MASK] = blocksize;
844eda14cbcSMatt Macy 	}
845eda14cbcSMatt Macy 	if (dn->dn_bonuslen != bonuslen)
846eda14cbcSMatt Macy 		dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = bonuslen;
847eda14cbcSMatt Macy 
848eda14cbcSMatt Macy 	if (bonustype == DMU_OT_SA) /* Maximize bonus space for SA */
849eda14cbcSMatt Macy 		nblkptr = 1;
850eda14cbcSMatt Macy 	else
851eda14cbcSMatt Macy 		nblkptr = MIN(DN_MAX_NBLKPTR,
852eda14cbcSMatt Macy 		    1 + ((DN_SLOTS_TO_BONUSLEN(dn_slots) - bonuslen) >>
853eda14cbcSMatt Macy 		    SPA_BLKPTRSHIFT));
854eda14cbcSMatt Macy 	if (dn->dn_bonustype != bonustype)
855eda14cbcSMatt Macy 		dn->dn_next_bonustype[tx->tx_txg & TXG_MASK] = bonustype;
856eda14cbcSMatt Macy 	if (dn->dn_nblkptr != nblkptr)
857eda14cbcSMatt Macy 		dn->dn_next_nblkptr[tx->tx_txg & TXG_MASK] = nblkptr;
858eda14cbcSMatt Macy 	if (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR && !keep_spill) {
859eda14cbcSMatt Macy 		dbuf_rm_spill(dn, tx);
860eda14cbcSMatt Macy 		dnode_rm_spill(dn, tx);
861eda14cbcSMatt Macy 	}
862eda14cbcSMatt Macy 
863eda14cbcSMatt Macy 	rw_exit(&dn->dn_struct_rwlock);
864eda14cbcSMatt Macy 
865eda14cbcSMatt Macy 	/* change type */
866eda14cbcSMatt Macy 	dn->dn_type = ot;
867eda14cbcSMatt Macy 
868eda14cbcSMatt Macy 	/* change bonus size and type */
869eda14cbcSMatt Macy 	mutex_enter(&dn->dn_mtx);
870eda14cbcSMatt Macy 	dn->dn_bonustype = bonustype;
871eda14cbcSMatt Macy 	dn->dn_bonuslen = bonuslen;
872eda14cbcSMatt Macy 	dn->dn_num_slots = dn_slots;
873eda14cbcSMatt Macy 	dn->dn_nblkptr = nblkptr;
874eda14cbcSMatt Macy 	dn->dn_checksum = ZIO_CHECKSUM_INHERIT;
875eda14cbcSMatt Macy 	dn->dn_compress = ZIO_COMPRESS_INHERIT;
876eda14cbcSMatt Macy 	ASSERT3U(dn->dn_nblkptr, <=, DN_MAX_NBLKPTR);
877eda14cbcSMatt Macy 
878eda14cbcSMatt Macy 	/* fix up the bonus db_size */
879eda14cbcSMatt Macy 	if (dn->dn_bonus) {
880eda14cbcSMatt Macy 		dn->dn_bonus->db.db_size =
881eda14cbcSMatt Macy 		    DN_SLOTS_TO_BONUSLEN(dn->dn_num_slots) -
882eda14cbcSMatt Macy 		    (dn->dn_nblkptr-1) * sizeof (blkptr_t);
883eda14cbcSMatt Macy 		ASSERT(dn->dn_bonuslen <= dn->dn_bonus->db.db_size);
884eda14cbcSMatt Macy 	}
885eda14cbcSMatt Macy 
886eda14cbcSMatt Macy 	dn->dn_allocated_txg = tx->tx_txg;
887eda14cbcSMatt Macy 	mutex_exit(&dn->dn_mtx);
888eda14cbcSMatt Macy }
889eda14cbcSMatt Macy 
890eda14cbcSMatt Macy #ifdef	_KERNEL
891eda14cbcSMatt Macy static void
892eda14cbcSMatt Macy dnode_move_impl(dnode_t *odn, dnode_t *ndn)
893eda14cbcSMatt Macy {
894eda14cbcSMatt Macy 	ASSERT(!RW_LOCK_HELD(&odn->dn_struct_rwlock));
895eda14cbcSMatt Macy 	ASSERT(MUTEX_NOT_HELD(&odn->dn_mtx));
896eda14cbcSMatt Macy 	ASSERT(MUTEX_NOT_HELD(&odn->dn_dbufs_mtx));
897eda14cbcSMatt Macy 
898eda14cbcSMatt Macy 	/* Copy fields. */
899eda14cbcSMatt Macy 	ndn->dn_objset = odn->dn_objset;
900eda14cbcSMatt Macy 	ndn->dn_object = odn->dn_object;
901eda14cbcSMatt Macy 	ndn->dn_dbuf = odn->dn_dbuf;
902eda14cbcSMatt Macy 	ndn->dn_handle = odn->dn_handle;
903eda14cbcSMatt Macy 	ndn->dn_phys = odn->dn_phys;
904eda14cbcSMatt Macy 	ndn->dn_type = odn->dn_type;
905eda14cbcSMatt Macy 	ndn->dn_bonuslen = odn->dn_bonuslen;
906eda14cbcSMatt Macy 	ndn->dn_bonustype = odn->dn_bonustype;
907eda14cbcSMatt Macy 	ndn->dn_nblkptr = odn->dn_nblkptr;
908eda14cbcSMatt Macy 	ndn->dn_checksum = odn->dn_checksum;
909eda14cbcSMatt Macy 	ndn->dn_compress = odn->dn_compress;
910eda14cbcSMatt Macy 	ndn->dn_nlevels = odn->dn_nlevels;
911eda14cbcSMatt Macy 	ndn->dn_indblkshift = odn->dn_indblkshift;
912eda14cbcSMatt Macy 	ndn->dn_datablkshift = odn->dn_datablkshift;
913eda14cbcSMatt Macy 	ndn->dn_datablkszsec = odn->dn_datablkszsec;
914eda14cbcSMatt Macy 	ndn->dn_datablksz = odn->dn_datablksz;
915eda14cbcSMatt Macy 	ndn->dn_maxblkid = odn->dn_maxblkid;
916eda14cbcSMatt Macy 	ndn->dn_num_slots = odn->dn_num_slots;
917da5137abSMartin Matuska 	memcpy(ndn->dn_next_type, odn->dn_next_type,
918eda14cbcSMatt Macy 	    sizeof (odn->dn_next_type));
919da5137abSMartin Matuska 	memcpy(ndn->dn_next_nblkptr, odn->dn_next_nblkptr,
920eda14cbcSMatt Macy 	    sizeof (odn->dn_next_nblkptr));
921da5137abSMartin Matuska 	memcpy(ndn->dn_next_nlevels, odn->dn_next_nlevels,
922eda14cbcSMatt Macy 	    sizeof (odn->dn_next_nlevels));
923da5137abSMartin Matuska 	memcpy(ndn->dn_next_indblkshift, odn->dn_next_indblkshift,
924eda14cbcSMatt Macy 	    sizeof (odn->dn_next_indblkshift));
925da5137abSMartin Matuska 	memcpy(ndn->dn_next_bonustype, odn->dn_next_bonustype,
926eda14cbcSMatt Macy 	    sizeof (odn->dn_next_bonustype));
927da5137abSMartin Matuska 	memcpy(ndn->dn_rm_spillblk, odn->dn_rm_spillblk,
928eda14cbcSMatt Macy 	    sizeof (odn->dn_rm_spillblk));
929da5137abSMartin Matuska 	memcpy(ndn->dn_next_bonuslen, odn->dn_next_bonuslen,
930eda14cbcSMatt Macy 	    sizeof (odn->dn_next_bonuslen));
931da5137abSMartin Matuska 	memcpy(ndn->dn_next_blksz, odn->dn_next_blksz,
932eda14cbcSMatt Macy 	    sizeof (odn->dn_next_blksz));
933da5137abSMartin Matuska 	memcpy(ndn->dn_next_maxblkid, odn->dn_next_maxblkid,
934eda14cbcSMatt Macy 	    sizeof (odn->dn_next_maxblkid));
935da5137abSMartin Matuska 	for (int i = 0; i < TXG_SIZE; i++) {
936eda14cbcSMatt Macy 		list_move_tail(&ndn->dn_dirty_records[i],
937eda14cbcSMatt Macy 		    &odn->dn_dirty_records[i]);
938eda14cbcSMatt Macy 	}
939da5137abSMartin Matuska 	memcpy(ndn->dn_free_ranges, odn->dn_free_ranges,
940eda14cbcSMatt Macy 	    sizeof (odn->dn_free_ranges));
941eda14cbcSMatt Macy 	ndn->dn_allocated_txg = odn->dn_allocated_txg;
942eda14cbcSMatt Macy 	ndn->dn_free_txg = odn->dn_free_txg;
943eda14cbcSMatt Macy 	ndn->dn_assigned_txg = odn->dn_assigned_txg;
944eda14cbcSMatt Macy 	ndn->dn_dirty_txg = odn->dn_dirty_txg;
945eda14cbcSMatt Macy 	ndn->dn_dirtyctx = odn->dn_dirtyctx;
946eda14cbcSMatt Macy 	ndn->dn_dirtyctx_firstset = odn->dn_dirtyctx_firstset;
947eda14cbcSMatt Macy 	ASSERT(zfs_refcount_count(&odn->dn_tx_holds) == 0);
948eda14cbcSMatt Macy 	zfs_refcount_transfer(&ndn->dn_holds, &odn->dn_holds);
949eda14cbcSMatt Macy 	ASSERT(avl_is_empty(&ndn->dn_dbufs));
950eda14cbcSMatt Macy 	avl_swap(&ndn->dn_dbufs, &odn->dn_dbufs);
951eda14cbcSMatt Macy 	ndn->dn_dbufs_count = odn->dn_dbufs_count;
952eda14cbcSMatt Macy 	ndn->dn_bonus = odn->dn_bonus;
953eda14cbcSMatt Macy 	ndn->dn_have_spill = odn->dn_have_spill;
954eda14cbcSMatt Macy 	ndn->dn_zio = odn->dn_zio;
955eda14cbcSMatt Macy 	ndn->dn_oldused = odn->dn_oldused;
956eda14cbcSMatt Macy 	ndn->dn_oldflags = odn->dn_oldflags;
957eda14cbcSMatt Macy 	ndn->dn_olduid = odn->dn_olduid;
958eda14cbcSMatt Macy 	ndn->dn_oldgid = odn->dn_oldgid;
959eda14cbcSMatt Macy 	ndn->dn_oldprojid = odn->dn_oldprojid;
960eda14cbcSMatt Macy 	ndn->dn_newuid = odn->dn_newuid;
961eda14cbcSMatt Macy 	ndn->dn_newgid = odn->dn_newgid;
962eda14cbcSMatt Macy 	ndn->dn_newprojid = odn->dn_newprojid;
963eda14cbcSMatt Macy 	ndn->dn_id_flags = odn->dn_id_flags;
964*ce4dcb97SMartin Matuska 	ndn->dn_storage_type = odn->dn_storage_type;
96516038816SMartin Matuska 	dmu_zfetch_init(&ndn->dn_zfetch, ndn);
966eda14cbcSMatt Macy 
967eda14cbcSMatt Macy 	/*
968eda14cbcSMatt Macy 	 * Update back pointers. Updating the handle fixes the back pointer of
969eda14cbcSMatt Macy 	 * every descendant dbuf as well as the bonus dbuf.
970eda14cbcSMatt Macy 	 */
971eda14cbcSMatt Macy 	ASSERT(ndn->dn_handle->dnh_dnode == odn);
972eda14cbcSMatt Macy 	ndn->dn_handle->dnh_dnode = ndn;
973eda14cbcSMatt Macy 
974eda14cbcSMatt Macy 	/*
975eda14cbcSMatt Macy 	 * Invalidate the original dnode by clearing all of its back pointers.
976eda14cbcSMatt Macy 	 */
977eda14cbcSMatt Macy 	odn->dn_dbuf = NULL;
978eda14cbcSMatt Macy 	odn->dn_handle = NULL;
979eda14cbcSMatt Macy 	avl_create(&odn->dn_dbufs, dbuf_compare, sizeof (dmu_buf_impl_t),
980eda14cbcSMatt Macy 	    offsetof(dmu_buf_impl_t, db_link));
981eda14cbcSMatt Macy 	odn->dn_dbufs_count = 0;
982eda14cbcSMatt Macy 	odn->dn_bonus = NULL;
983eda14cbcSMatt Macy 	dmu_zfetch_fini(&odn->dn_zfetch);
984eda14cbcSMatt Macy 
985eda14cbcSMatt Macy 	/*
986eda14cbcSMatt Macy 	 * Set the low bit of the objset pointer to ensure that dnode_move()
987eda14cbcSMatt Macy 	 * recognizes the dnode as invalid in any subsequent callback.
988eda14cbcSMatt Macy 	 */
989eda14cbcSMatt Macy 	POINTER_INVALIDATE(&odn->dn_objset);
990eda14cbcSMatt Macy 
991eda14cbcSMatt Macy 	/*
992eda14cbcSMatt Macy 	 * Satisfy the destructor.
993eda14cbcSMatt Macy 	 */
994da5137abSMartin Matuska 	for (int i = 0; i < TXG_SIZE; i++) {
995eda14cbcSMatt Macy 		list_create(&odn->dn_dirty_records[i],
996eda14cbcSMatt Macy 		    sizeof (dbuf_dirty_record_t),
997eda14cbcSMatt Macy 		    offsetof(dbuf_dirty_record_t, dr_dirty_node));
998eda14cbcSMatt Macy 		odn->dn_free_ranges[i] = NULL;
999eda14cbcSMatt Macy 		odn->dn_next_nlevels[i] = 0;
1000eda14cbcSMatt Macy 		odn->dn_next_indblkshift[i] = 0;
1001eda14cbcSMatt Macy 		odn->dn_next_bonustype[i] = 0;
1002eda14cbcSMatt Macy 		odn->dn_rm_spillblk[i] = 0;
1003eda14cbcSMatt Macy 		odn->dn_next_bonuslen[i] = 0;
1004eda14cbcSMatt Macy 		odn->dn_next_blksz[i] = 0;
1005eda14cbcSMatt Macy 	}
1006eda14cbcSMatt Macy 	odn->dn_allocated_txg = 0;
1007eda14cbcSMatt Macy 	odn->dn_free_txg = 0;
1008eda14cbcSMatt Macy 	odn->dn_assigned_txg = 0;
1009eda14cbcSMatt Macy 	odn->dn_dirty_txg = 0;
1010eda14cbcSMatt Macy 	odn->dn_dirtyctx = 0;
1011eda14cbcSMatt Macy 	odn->dn_dirtyctx_firstset = NULL;
1012eda14cbcSMatt Macy 	odn->dn_have_spill = B_FALSE;
1013eda14cbcSMatt Macy 	odn->dn_zio = NULL;
1014eda14cbcSMatt Macy 	odn->dn_oldused = 0;
1015eda14cbcSMatt Macy 	odn->dn_oldflags = 0;
1016eda14cbcSMatt Macy 	odn->dn_olduid = 0;
1017eda14cbcSMatt Macy 	odn->dn_oldgid = 0;
1018eda14cbcSMatt Macy 	odn->dn_oldprojid = ZFS_DEFAULT_PROJID;
1019eda14cbcSMatt Macy 	odn->dn_newuid = 0;
1020eda14cbcSMatt Macy 	odn->dn_newgid = 0;
1021eda14cbcSMatt Macy 	odn->dn_newprojid = ZFS_DEFAULT_PROJID;
1022eda14cbcSMatt Macy 	odn->dn_id_flags = 0;
1023*ce4dcb97SMartin Matuska 	odn->dn_storage_type = DMU_OT_NONE;
1024eda14cbcSMatt Macy 
1025eda14cbcSMatt Macy 	/*
1026eda14cbcSMatt Macy 	 * Mark the dnode.
1027eda14cbcSMatt Macy 	 */
1028eda14cbcSMatt Macy 	ndn->dn_moved = 1;
1029eda14cbcSMatt Macy 	odn->dn_moved = (uint8_t)-1;
1030eda14cbcSMatt Macy }
1031eda14cbcSMatt Macy 
1032eda14cbcSMatt Macy static kmem_cbrc_t
1033eda14cbcSMatt Macy dnode_move(void *buf, void *newbuf, size_t size, void *arg)
1034eda14cbcSMatt Macy {
1035eda14cbcSMatt Macy 	dnode_t *odn = buf, *ndn = newbuf;
1036eda14cbcSMatt Macy 	objset_t *os;
1037eda14cbcSMatt Macy 	int64_t refcount;
1038eda14cbcSMatt Macy 	uint32_t dbufs;
1039eda14cbcSMatt Macy 
1040*ce4dcb97SMartin Matuska #ifndef USE_DNODE_HANDLE
1041*ce4dcb97SMartin Matuska 	/*
1042*ce4dcb97SMartin Matuska 	 * We can't move dnodes if dbufs reference them directly without
1043*ce4dcb97SMartin Matuska 	 * using handles and respecitve locking.  Unless USE_DNODE_HANDLE
1044*ce4dcb97SMartin Matuska 	 * is defined the code below is only to make sure it still builds,
1045*ce4dcb97SMartin Matuska 	 * but it should never be used, since it is unsafe.
1046*ce4dcb97SMartin Matuska 	 */
1047*ce4dcb97SMartin Matuska #ifdef ZFS_DEBUG
1048*ce4dcb97SMartin Matuska 	PANIC("dnode_move() called without USE_DNODE_HANDLE");
1049*ce4dcb97SMartin Matuska #endif
1050*ce4dcb97SMartin Matuska 	return (KMEM_CBRC_NO);
1051*ce4dcb97SMartin Matuska #endif
1052*ce4dcb97SMartin Matuska 
1053eda14cbcSMatt Macy 	/*
1054eda14cbcSMatt Macy 	 * The dnode is on the objset's list of known dnodes if the objset
1055eda14cbcSMatt Macy 	 * pointer is valid. We set the low bit of the objset pointer when
1056eda14cbcSMatt Macy 	 * freeing the dnode to invalidate it, and the memory patterns written
1057eda14cbcSMatt Macy 	 * by kmem (baddcafe and deadbeef) set at least one of the two low bits.
1058eda14cbcSMatt Macy 	 * A newly created dnode sets the objset pointer last of all to indicate
1059eda14cbcSMatt Macy 	 * that the dnode is known and in a valid state to be moved by this
1060eda14cbcSMatt Macy 	 * function.
1061eda14cbcSMatt Macy 	 */
1062eda14cbcSMatt Macy 	os = odn->dn_objset;
1063eda14cbcSMatt Macy 	if (!POINTER_IS_VALID(os)) {
1064eda14cbcSMatt Macy 		DNODE_STAT_BUMP(dnode_move_invalid);
1065eda14cbcSMatt Macy 		return (KMEM_CBRC_DONT_KNOW);
1066eda14cbcSMatt Macy 	}
1067eda14cbcSMatt Macy 
1068eda14cbcSMatt Macy 	/*
1069eda14cbcSMatt Macy 	 * Ensure that the objset does not go away during the move.
1070eda14cbcSMatt Macy 	 */
1071eda14cbcSMatt Macy 	rw_enter(&os_lock, RW_WRITER);
1072eda14cbcSMatt Macy 	if (os != odn->dn_objset) {
1073eda14cbcSMatt Macy 		rw_exit(&os_lock);
1074eda14cbcSMatt Macy 		DNODE_STAT_BUMP(dnode_move_recheck1);
1075eda14cbcSMatt Macy 		return (KMEM_CBRC_DONT_KNOW);
1076eda14cbcSMatt Macy 	}
1077eda14cbcSMatt Macy 
1078eda14cbcSMatt Macy 	/*
1079eda14cbcSMatt Macy 	 * If the dnode is still valid, then so is the objset. We know that no
1080eda14cbcSMatt Macy 	 * valid objset can be freed while we hold os_lock, so we can safely
1081eda14cbcSMatt Macy 	 * ensure that the objset remains in use.
1082eda14cbcSMatt Macy 	 */
1083eda14cbcSMatt Macy 	mutex_enter(&os->os_lock);
1084eda14cbcSMatt Macy 
1085eda14cbcSMatt Macy 	/*
1086eda14cbcSMatt Macy 	 * Recheck the objset pointer in case the dnode was removed just before
1087eda14cbcSMatt Macy 	 * acquiring the lock.
1088eda14cbcSMatt Macy 	 */
1089eda14cbcSMatt Macy 	if (os != odn->dn_objset) {
1090eda14cbcSMatt Macy 		mutex_exit(&os->os_lock);
1091eda14cbcSMatt Macy 		rw_exit(&os_lock);
1092eda14cbcSMatt Macy 		DNODE_STAT_BUMP(dnode_move_recheck2);
1093eda14cbcSMatt Macy 		return (KMEM_CBRC_DONT_KNOW);
1094eda14cbcSMatt Macy 	}
1095eda14cbcSMatt Macy 
1096eda14cbcSMatt Macy 	/*
1097eda14cbcSMatt Macy 	 * At this point we know that as long as we hold os->os_lock, the dnode
1098eda14cbcSMatt Macy 	 * cannot be freed and fields within the dnode can be safely accessed.
1099eda14cbcSMatt Macy 	 * The objset listing this dnode cannot go away as long as this dnode is
1100eda14cbcSMatt Macy 	 * on its list.
1101eda14cbcSMatt Macy 	 */
1102eda14cbcSMatt Macy 	rw_exit(&os_lock);
1103eda14cbcSMatt Macy 	if (DMU_OBJECT_IS_SPECIAL(odn->dn_object)) {
1104eda14cbcSMatt Macy 		mutex_exit(&os->os_lock);
1105eda14cbcSMatt Macy 		DNODE_STAT_BUMP(dnode_move_special);
1106eda14cbcSMatt Macy 		return (KMEM_CBRC_NO);
1107eda14cbcSMatt Macy 	}
1108eda14cbcSMatt Macy 	ASSERT(odn->dn_dbuf != NULL); /* only "special" dnodes have no parent */
1109eda14cbcSMatt Macy 
1110eda14cbcSMatt Macy 	/*
1111eda14cbcSMatt Macy 	 * Lock the dnode handle to prevent the dnode from obtaining any new
1112eda14cbcSMatt Macy 	 * holds. This also prevents the descendant dbufs and the bonus dbuf
1113eda14cbcSMatt Macy 	 * from accessing the dnode, so that we can discount their holds. The
1114eda14cbcSMatt Macy 	 * handle is safe to access because we know that while the dnode cannot
1115eda14cbcSMatt Macy 	 * go away, neither can its handle. Once we hold dnh_zrlock, we can
1116eda14cbcSMatt Macy 	 * safely move any dnode referenced only by dbufs.
1117eda14cbcSMatt Macy 	 */
1118eda14cbcSMatt Macy 	if (!zrl_tryenter(&odn->dn_handle->dnh_zrlock)) {
1119eda14cbcSMatt Macy 		mutex_exit(&os->os_lock);
1120eda14cbcSMatt Macy 		DNODE_STAT_BUMP(dnode_move_handle);
1121eda14cbcSMatt Macy 		return (KMEM_CBRC_LATER);
1122eda14cbcSMatt Macy 	}
1123eda14cbcSMatt Macy 
1124eda14cbcSMatt Macy 	/*
1125eda14cbcSMatt Macy 	 * Ensure a consistent view of the dnode's holds and the dnode's dbufs.
1126eda14cbcSMatt Macy 	 * We need to guarantee that there is a hold for every dbuf in order to
1127eda14cbcSMatt Macy 	 * determine whether the dnode is actively referenced. Falsely matching
1128eda14cbcSMatt Macy 	 * a dbuf to an active hold would lead to an unsafe move. It's possible
1129eda14cbcSMatt Macy 	 * that a thread already having an active dnode hold is about to add a
1130eda14cbcSMatt Macy 	 * dbuf, and we can't compare hold and dbuf counts while the add is in
1131eda14cbcSMatt Macy 	 * progress.
1132eda14cbcSMatt Macy 	 */
1133eda14cbcSMatt Macy 	if (!rw_tryenter(&odn->dn_struct_rwlock, RW_WRITER)) {
1134eda14cbcSMatt Macy 		zrl_exit(&odn->dn_handle->dnh_zrlock);
1135eda14cbcSMatt Macy 		mutex_exit(&os->os_lock);
1136eda14cbcSMatt Macy 		DNODE_STAT_BUMP(dnode_move_rwlock);
1137eda14cbcSMatt Macy 		return (KMEM_CBRC_LATER);
1138eda14cbcSMatt Macy 	}
1139eda14cbcSMatt Macy 
1140eda14cbcSMatt Macy 	/*
1141eda14cbcSMatt Macy 	 * A dbuf may be removed (evicted) without an active dnode hold. In that
1142eda14cbcSMatt Macy 	 * case, the dbuf count is decremented under the handle lock before the
1143eda14cbcSMatt Macy 	 * dbuf's hold is released. This order ensures that if we count the hold
1144eda14cbcSMatt Macy 	 * after the dbuf is removed but before its hold is released, we will
1145eda14cbcSMatt Macy 	 * treat the unmatched hold as active and exit safely. If we count the
1146eda14cbcSMatt Macy 	 * hold before the dbuf is removed, the hold is discounted, and the
1147eda14cbcSMatt Macy 	 * removal is blocked until the move completes.
1148eda14cbcSMatt Macy 	 */
1149eda14cbcSMatt Macy 	refcount = zfs_refcount_count(&odn->dn_holds);
1150eda14cbcSMatt Macy 	ASSERT(refcount >= 0);
1151eda14cbcSMatt Macy 	dbufs = DN_DBUFS_COUNT(odn);
1152eda14cbcSMatt Macy 
1153eda14cbcSMatt Macy 	/* We can't have more dbufs than dnode holds. */
1154eda14cbcSMatt Macy 	ASSERT3U(dbufs, <=, refcount);
1155eda14cbcSMatt Macy 	DTRACE_PROBE3(dnode__move, dnode_t *, odn, int64_t, refcount,
1156eda14cbcSMatt Macy 	    uint32_t, dbufs);
1157eda14cbcSMatt Macy 
1158eda14cbcSMatt Macy 	if (refcount > dbufs) {
1159eda14cbcSMatt Macy 		rw_exit(&odn->dn_struct_rwlock);
1160eda14cbcSMatt Macy 		zrl_exit(&odn->dn_handle->dnh_zrlock);
1161eda14cbcSMatt Macy 		mutex_exit(&os->os_lock);
1162eda14cbcSMatt Macy 		DNODE_STAT_BUMP(dnode_move_active);
1163eda14cbcSMatt Macy 		return (KMEM_CBRC_LATER);
1164eda14cbcSMatt Macy 	}
1165eda14cbcSMatt Macy 
1166eda14cbcSMatt Macy 	rw_exit(&odn->dn_struct_rwlock);
1167eda14cbcSMatt Macy 
1168eda14cbcSMatt Macy 	/*
1169eda14cbcSMatt Macy 	 * At this point we know that anyone with a hold on the dnode is not
1170eda14cbcSMatt Macy 	 * actively referencing it. The dnode is known and in a valid state to
1171eda14cbcSMatt Macy 	 * move. We're holding the locks needed to execute the critical section.
1172eda14cbcSMatt Macy 	 */
1173eda14cbcSMatt Macy 	dnode_move_impl(odn, ndn);
1174eda14cbcSMatt Macy 
1175eda14cbcSMatt Macy 	list_link_replace(&odn->dn_link, &ndn->dn_link);
1176eda14cbcSMatt Macy 	/* If the dnode was safe to move, the refcount cannot have changed. */
1177eda14cbcSMatt Macy 	ASSERT(refcount == zfs_refcount_count(&ndn->dn_holds));
1178eda14cbcSMatt Macy 	ASSERT(dbufs == DN_DBUFS_COUNT(ndn));
1179eda14cbcSMatt Macy 	zrl_exit(&ndn->dn_handle->dnh_zrlock); /* handle has moved */
1180eda14cbcSMatt Macy 	mutex_exit(&os->os_lock);
1181eda14cbcSMatt Macy 
1182eda14cbcSMatt Macy 	return (KMEM_CBRC_YES);
1183eda14cbcSMatt Macy }
1184eda14cbcSMatt Macy #endif	/* _KERNEL */
1185eda14cbcSMatt Macy 
1186eda14cbcSMatt Macy static void
1187eda14cbcSMatt Macy dnode_slots_hold(dnode_children_t *children, int idx, int slots)
1188eda14cbcSMatt Macy {
1189eda14cbcSMatt Macy 	ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK);
1190eda14cbcSMatt Macy 
1191eda14cbcSMatt Macy 	for (int i = idx; i < idx + slots; i++) {
1192eda14cbcSMatt Macy 		dnode_handle_t *dnh = &children->dnc_children[i];
1193eda14cbcSMatt Macy 		zrl_add(&dnh->dnh_zrlock);
1194eda14cbcSMatt Macy 	}
1195eda14cbcSMatt Macy }
1196eda14cbcSMatt Macy 
1197eda14cbcSMatt Macy static void
1198eda14cbcSMatt Macy dnode_slots_rele(dnode_children_t *children, int idx, int slots)
1199eda14cbcSMatt Macy {
1200eda14cbcSMatt Macy 	ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK);
1201eda14cbcSMatt Macy 
1202eda14cbcSMatt Macy 	for (int i = idx; i < idx + slots; i++) {
1203eda14cbcSMatt Macy 		dnode_handle_t *dnh = &children->dnc_children[i];
1204eda14cbcSMatt Macy 
1205eda14cbcSMatt Macy 		if (zrl_is_locked(&dnh->dnh_zrlock))
1206eda14cbcSMatt Macy 			zrl_exit(&dnh->dnh_zrlock);
1207eda14cbcSMatt Macy 		else
1208eda14cbcSMatt Macy 			zrl_remove(&dnh->dnh_zrlock);
1209eda14cbcSMatt Macy 	}
1210eda14cbcSMatt Macy }
1211eda14cbcSMatt Macy 
1212eda14cbcSMatt Macy static int
1213eda14cbcSMatt Macy dnode_slots_tryenter(dnode_children_t *children, int idx, int slots)
1214eda14cbcSMatt Macy {
1215eda14cbcSMatt Macy 	ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK);
1216eda14cbcSMatt Macy 
1217eda14cbcSMatt Macy 	for (int i = idx; i < idx + slots; i++) {
1218eda14cbcSMatt Macy 		dnode_handle_t *dnh = &children->dnc_children[i];
1219eda14cbcSMatt Macy 
1220eda14cbcSMatt Macy 		if (!zrl_tryenter(&dnh->dnh_zrlock)) {
1221eda14cbcSMatt Macy 			for (int j = idx; j < i; j++) {
1222eda14cbcSMatt Macy 				dnh = &children->dnc_children[j];
1223eda14cbcSMatt Macy 				zrl_exit(&dnh->dnh_zrlock);
1224eda14cbcSMatt Macy 			}
1225eda14cbcSMatt Macy 
1226eda14cbcSMatt Macy 			return (0);
1227eda14cbcSMatt Macy 		}
1228eda14cbcSMatt Macy 	}
1229eda14cbcSMatt Macy 
1230eda14cbcSMatt Macy 	return (1);
1231eda14cbcSMatt Macy }
1232eda14cbcSMatt Macy 
1233eda14cbcSMatt Macy static void
1234eda14cbcSMatt Macy dnode_set_slots(dnode_children_t *children, int idx, int slots, void *ptr)
1235eda14cbcSMatt Macy {
1236eda14cbcSMatt Macy 	ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK);
1237eda14cbcSMatt Macy 
1238eda14cbcSMatt Macy 	for (int i = idx; i < idx + slots; i++) {
1239eda14cbcSMatt Macy 		dnode_handle_t *dnh = &children->dnc_children[i];
1240eda14cbcSMatt Macy 		dnh->dnh_dnode = ptr;
1241eda14cbcSMatt Macy 	}
1242eda14cbcSMatt Macy }
1243eda14cbcSMatt Macy 
1244eda14cbcSMatt Macy static boolean_t
1245eda14cbcSMatt Macy dnode_check_slots_free(dnode_children_t *children, int idx, int slots)
1246eda14cbcSMatt Macy {
1247eda14cbcSMatt Macy 	ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK);
1248eda14cbcSMatt Macy 
1249eda14cbcSMatt Macy 	/*
1250eda14cbcSMatt Macy 	 * If all dnode slots are either already free or
1251eda14cbcSMatt Macy 	 * evictable return B_TRUE.
1252eda14cbcSMatt Macy 	 */
1253eda14cbcSMatt Macy 	for (int i = idx; i < idx + slots; i++) {
1254eda14cbcSMatt Macy 		dnode_handle_t *dnh = &children->dnc_children[i];
1255eda14cbcSMatt Macy 		dnode_t *dn = dnh->dnh_dnode;
1256eda14cbcSMatt Macy 
1257eda14cbcSMatt Macy 		if (dn == DN_SLOT_FREE) {
1258eda14cbcSMatt Macy 			continue;
1259eda14cbcSMatt Macy 		} else if (DN_SLOT_IS_PTR(dn)) {
1260eda14cbcSMatt Macy 			mutex_enter(&dn->dn_mtx);
1261eda14cbcSMatt Macy 			boolean_t can_free = (dn->dn_type == DMU_OT_NONE &&
1262eda14cbcSMatt Macy 			    zfs_refcount_is_zero(&dn->dn_holds) &&
1263eda14cbcSMatt Macy 			    !DNODE_IS_DIRTY(dn));
1264eda14cbcSMatt Macy 			mutex_exit(&dn->dn_mtx);
1265eda14cbcSMatt Macy 
1266eda14cbcSMatt Macy 			if (!can_free)
1267eda14cbcSMatt Macy 				return (B_FALSE);
1268eda14cbcSMatt Macy 			else
1269eda14cbcSMatt Macy 				continue;
1270eda14cbcSMatt Macy 		} else {
1271eda14cbcSMatt Macy 			return (B_FALSE);
1272eda14cbcSMatt Macy 		}
1273eda14cbcSMatt Macy 	}
1274eda14cbcSMatt Macy 
1275eda14cbcSMatt Macy 	return (B_TRUE);
1276eda14cbcSMatt Macy }
1277eda14cbcSMatt Macy 
1278a2b560ccSMartin Matuska static uint_t
1279eda14cbcSMatt Macy dnode_reclaim_slots(dnode_children_t *children, int idx, int slots)
1280eda14cbcSMatt Macy {
1281a2b560ccSMartin Matuska 	uint_t reclaimed = 0;
1282a2b560ccSMartin Matuska 
1283eda14cbcSMatt Macy 	ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK);
1284eda14cbcSMatt Macy 
1285eda14cbcSMatt Macy 	for (int i = idx; i < idx + slots; i++) {
1286eda14cbcSMatt Macy 		dnode_handle_t *dnh = &children->dnc_children[i];
1287eda14cbcSMatt Macy 
1288eda14cbcSMatt Macy 		ASSERT(zrl_is_locked(&dnh->dnh_zrlock));
1289eda14cbcSMatt Macy 
1290eda14cbcSMatt Macy 		if (DN_SLOT_IS_PTR(dnh->dnh_dnode)) {
1291eda14cbcSMatt Macy 			ASSERT3S(dnh->dnh_dnode->dn_type, ==, DMU_OT_NONE);
1292eda14cbcSMatt Macy 			dnode_destroy(dnh->dnh_dnode);
1293eda14cbcSMatt Macy 			dnh->dnh_dnode = DN_SLOT_FREE;
1294a2b560ccSMartin Matuska 			reclaimed++;
1295eda14cbcSMatt Macy 		}
1296eda14cbcSMatt Macy 	}
1297a2b560ccSMartin Matuska 
1298a2b560ccSMartin Matuska 	return (reclaimed);
1299eda14cbcSMatt Macy }
1300eda14cbcSMatt Macy 
1301eda14cbcSMatt Macy void
1302eda14cbcSMatt Macy dnode_free_interior_slots(dnode_t *dn)
1303eda14cbcSMatt Macy {
1304eda14cbcSMatt Macy 	dnode_children_t *children = dmu_buf_get_user(&dn->dn_dbuf->db);
1305eda14cbcSMatt Macy 	int epb = dn->dn_dbuf->db.db_size >> DNODE_SHIFT;
1306eda14cbcSMatt Macy 	int idx = (dn->dn_object & (epb - 1)) + 1;
1307eda14cbcSMatt Macy 	int slots = dn->dn_num_slots - 1;
1308eda14cbcSMatt Macy 
1309eda14cbcSMatt Macy 	if (slots == 0)
1310eda14cbcSMatt Macy 		return;
1311eda14cbcSMatt Macy 
1312eda14cbcSMatt Macy 	ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK);
1313eda14cbcSMatt Macy 
1314eda14cbcSMatt Macy 	while (!dnode_slots_tryenter(children, idx, slots)) {
1315eda14cbcSMatt Macy 		DNODE_STAT_BUMP(dnode_free_interior_lock_retry);
1316c7046f76SMartin Matuska 		kpreempt(KPREEMPT_SYNC);
1317eda14cbcSMatt Macy 	}
1318eda14cbcSMatt Macy 
1319eda14cbcSMatt Macy 	dnode_set_slots(children, idx, slots, DN_SLOT_FREE);
1320eda14cbcSMatt Macy 	dnode_slots_rele(children, idx, slots);
1321eda14cbcSMatt Macy }
1322eda14cbcSMatt Macy 
1323eda14cbcSMatt Macy void
1324eda14cbcSMatt Macy dnode_special_close(dnode_handle_t *dnh)
1325eda14cbcSMatt Macy {
1326eda14cbcSMatt Macy 	dnode_t *dn = dnh->dnh_dnode;
1327eda14cbcSMatt Macy 
1328eda14cbcSMatt Macy 	/*
1329eda14cbcSMatt Macy 	 * Ensure dnode_rele_and_unlock() has released dn_mtx, after final
1330eda14cbcSMatt Macy 	 * zfs_refcount_remove()
1331eda14cbcSMatt Macy 	 */
1332eda14cbcSMatt Macy 	mutex_enter(&dn->dn_mtx);
1333eda14cbcSMatt Macy 	if (zfs_refcount_count(&dn->dn_holds) > 0)
1334eda14cbcSMatt Macy 		cv_wait(&dn->dn_nodnholds, &dn->dn_mtx);
1335eda14cbcSMatt Macy 	mutex_exit(&dn->dn_mtx);
1336eda14cbcSMatt Macy 	ASSERT3U(zfs_refcount_count(&dn->dn_holds), ==, 0);
1337eda14cbcSMatt Macy 
1338eda14cbcSMatt Macy 	ASSERT(dn->dn_dbuf == NULL ||
1339eda14cbcSMatt Macy 	    dmu_buf_get_user(&dn->dn_dbuf->db) == NULL);
1340eda14cbcSMatt Macy 	zrl_add(&dnh->dnh_zrlock);
1341eda14cbcSMatt Macy 	dnode_destroy(dn); /* implicit zrl_remove() */
1342eda14cbcSMatt Macy 	zrl_destroy(&dnh->dnh_zrlock);
1343eda14cbcSMatt Macy 	dnh->dnh_dnode = NULL;
1344eda14cbcSMatt Macy }
1345eda14cbcSMatt Macy 
1346eda14cbcSMatt Macy void
1347eda14cbcSMatt Macy dnode_special_open(objset_t *os, dnode_phys_t *dnp, uint64_t object,
1348eda14cbcSMatt Macy     dnode_handle_t *dnh)
1349eda14cbcSMatt Macy {
1350eda14cbcSMatt Macy 	dnode_t *dn;
1351eda14cbcSMatt Macy 
1352eda14cbcSMatt Macy 	zrl_init(&dnh->dnh_zrlock);
13532c48331dSMatt Macy 	VERIFY3U(1, ==, zrl_tryenter(&dnh->dnh_zrlock));
1354eda14cbcSMatt Macy 
1355eda14cbcSMatt Macy 	dn = dnode_create(os, dnp, NULL, object, dnh);
1356eda14cbcSMatt Macy 	DNODE_VERIFY(dn);
1357eda14cbcSMatt Macy 
1358eda14cbcSMatt Macy 	zrl_exit(&dnh->dnh_zrlock);
1359eda14cbcSMatt Macy }
1360eda14cbcSMatt Macy 
1361eda14cbcSMatt Macy static void
1362eda14cbcSMatt Macy dnode_buf_evict_async(void *dbu)
1363eda14cbcSMatt Macy {
1364eda14cbcSMatt Macy 	dnode_children_t *dnc = dbu;
1365eda14cbcSMatt Macy 
1366eda14cbcSMatt Macy 	DNODE_STAT_BUMP(dnode_buf_evict);
1367eda14cbcSMatt Macy 
1368eda14cbcSMatt Macy 	for (int i = 0; i < dnc->dnc_count; i++) {
1369eda14cbcSMatt Macy 		dnode_handle_t *dnh = &dnc->dnc_children[i];
1370eda14cbcSMatt Macy 		dnode_t *dn;
1371eda14cbcSMatt Macy 
1372eda14cbcSMatt Macy 		/*
1373eda14cbcSMatt Macy 		 * The dnode handle lock guards against the dnode moving to
1374eda14cbcSMatt Macy 		 * another valid address, so there is no need here to guard
1375eda14cbcSMatt Macy 		 * against changes to or from NULL.
1376eda14cbcSMatt Macy 		 */
1377eda14cbcSMatt Macy 		if (!DN_SLOT_IS_PTR(dnh->dnh_dnode)) {
1378eda14cbcSMatt Macy 			zrl_destroy(&dnh->dnh_zrlock);
1379eda14cbcSMatt Macy 			dnh->dnh_dnode = DN_SLOT_UNINIT;
1380eda14cbcSMatt Macy 			continue;
1381eda14cbcSMatt Macy 		}
1382eda14cbcSMatt Macy 
1383eda14cbcSMatt Macy 		zrl_add(&dnh->dnh_zrlock);
1384eda14cbcSMatt Macy 		dn = dnh->dnh_dnode;
1385eda14cbcSMatt Macy 		/*
1386eda14cbcSMatt Macy 		 * If there are holds on this dnode, then there should
1387eda14cbcSMatt Macy 		 * be holds on the dnode's containing dbuf as well; thus
1388eda14cbcSMatt Macy 		 * it wouldn't be eligible for eviction and this function
1389eda14cbcSMatt Macy 		 * would not have been called.
1390eda14cbcSMatt Macy 		 */
1391eda14cbcSMatt Macy 		ASSERT(zfs_refcount_is_zero(&dn->dn_holds));
1392eda14cbcSMatt Macy 		ASSERT(zfs_refcount_is_zero(&dn->dn_tx_holds));
1393eda14cbcSMatt Macy 
1394eda14cbcSMatt Macy 		dnode_destroy(dn); /* implicit zrl_remove() for first slot */
1395eda14cbcSMatt Macy 		zrl_destroy(&dnh->dnh_zrlock);
1396eda14cbcSMatt Macy 		dnh->dnh_dnode = DN_SLOT_UNINIT;
1397eda14cbcSMatt Macy 	}
1398eda14cbcSMatt Macy 	kmem_free(dnc, sizeof (dnode_children_t) +
1399eda14cbcSMatt Macy 	    dnc->dnc_count * sizeof (dnode_handle_t));
1400eda14cbcSMatt Macy }
1401eda14cbcSMatt Macy 
1402eda14cbcSMatt Macy /*
1403eda14cbcSMatt Macy  * When the DNODE_MUST_BE_FREE flag is set, the "slots" parameter is used
1404eda14cbcSMatt Macy  * to ensure the hole at the specified object offset is large enough to
1405eda14cbcSMatt Macy  * hold the dnode being created. The slots parameter is also used to ensure
1406eda14cbcSMatt Macy  * a dnode does not span multiple dnode blocks. In both of these cases, if
1407eda14cbcSMatt Macy  * a failure occurs, ENOSPC is returned. Keep in mind, these failure cases
1408eda14cbcSMatt Macy  * are only possible when using DNODE_MUST_BE_FREE.
1409eda14cbcSMatt Macy  *
1410eda14cbcSMatt Macy  * If the DNODE_MUST_BE_ALLOCATED flag is set, "slots" must be 0.
1411eda14cbcSMatt Macy  * dnode_hold_impl() will check if the requested dnode is already consumed
1412eda14cbcSMatt Macy  * as an extra dnode slot by an large dnode, in which case it returns
1413eda14cbcSMatt Macy  * ENOENT.
1414eda14cbcSMatt Macy  *
1415eda14cbcSMatt Macy  * If the DNODE_DRY_RUN flag is set, we don't actually hold the dnode, just
1416eda14cbcSMatt Macy  * return whether the hold would succeed or not. tag and dnp should set to
1417eda14cbcSMatt Macy  * NULL in this case.
1418eda14cbcSMatt Macy  *
1419eda14cbcSMatt Macy  * errors:
1420eda14cbcSMatt Macy  * EINVAL - Invalid object number or flags.
1421eda14cbcSMatt Macy  * ENOSPC - Hole too small to fulfill "slots" request (DNODE_MUST_BE_FREE)
1422eda14cbcSMatt Macy  * EEXIST - Refers to an allocated dnode (DNODE_MUST_BE_FREE)
1423eda14cbcSMatt Macy  *        - Refers to a freeing dnode (DNODE_MUST_BE_FREE)
1424eda14cbcSMatt Macy  *        - Refers to an interior dnode slot (DNODE_MUST_BE_ALLOCATED)
1425eda14cbcSMatt Macy  * ENOENT - The requested dnode is not allocated (DNODE_MUST_BE_ALLOCATED)
1426eda14cbcSMatt Macy  *        - The requested dnode is being freed (DNODE_MUST_BE_ALLOCATED)
1427eda14cbcSMatt Macy  * EIO    - I/O error when reading the meta dnode dbuf.
1428eda14cbcSMatt Macy  *
1429eda14cbcSMatt Macy  * succeeds even for free dnodes.
1430eda14cbcSMatt Macy  */
1431eda14cbcSMatt Macy int
1432eda14cbcSMatt Macy dnode_hold_impl(objset_t *os, uint64_t object, int flag, int slots,
1433a0b956f5SMartin Matuska     const void *tag, dnode_t **dnp)
1434eda14cbcSMatt Macy {
1435eda14cbcSMatt Macy 	int epb, idx, err;
1436eda14cbcSMatt Macy 	int drop_struct_lock = FALSE;
1437eda14cbcSMatt Macy 	int type;
1438eda14cbcSMatt Macy 	uint64_t blk;
1439eda14cbcSMatt Macy 	dnode_t *mdn, *dn;
1440eda14cbcSMatt Macy 	dmu_buf_impl_t *db;
1441eda14cbcSMatt Macy 	dnode_children_t *dnc;
1442eda14cbcSMatt Macy 	dnode_phys_t *dn_block;
1443eda14cbcSMatt Macy 	dnode_handle_t *dnh;
1444eda14cbcSMatt Macy 
1445eda14cbcSMatt Macy 	ASSERT(!(flag & DNODE_MUST_BE_ALLOCATED) || (slots == 0));
1446eda14cbcSMatt Macy 	ASSERT(!(flag & DNODE_MUST_BE_FREE) || (slots > 0));
1447eda14cbcSMatt Macy 	IMPLY(flag & DNODE_DRY_RUN, (tag == NULL) && (dnp == NULL));
1448eda14cbcSMatt Macy 
1449eda14cbcSMatt Macy 	/*
1450eda14cbcSMatt Macy 	 * If you are holding the spa config lock as writer, you shouldn't
1451eda14cbcSMatt Macy 	 * be asking the DMU to do *anything* unless it's the root pool
1452eda14cbcSMatt Macy 	 * which may require us to read from the root filesystem while
1453eda14cbcSMatt Macy 	 * holding some (not all) of the locks as writer.
1454eda14cbcSMatt Macy 	 */
1455eda14cbcSMatt Macy 	ASSERT(spa_config_held(os->os_spa, SCL_ALL, RW_WRITER) == 0 ||
1456eda14cbcSMatt Macy 	    (spa_is_root(os->os_spa) &&
1457eda14cbcSMatt Macy 	    spa_config_held(os->os_spa, SCL_STATE, RW_WRITER)));
1458eda14cbcSMatt Macy 
1459eda14cbcSMatt Macy 	ASSERT((flag & DNODE_MUST_BE_ALLOCATED) || (flag & DNODE_MUST_BE_FREE));
1460eda14cbcSMatt Macy 
1461eda14cbcSMatt Macy 	if (object == DMU_USERUSED_OBJECT || object == DMU_GROUPUSED_OBJECT ||
1462eda14cbcSMatt Macy 	    object == DMU_PROJECTUSED_OBJECT) {
1463eda14cbcSMatt Macy 		if (object == DMU_USERUSED_OBJECT)
1464eda14cbcSMatt Macy 			dn = DMU_USERUSED_DNODE(os);
1465eda14cbcSMatt Macy 		else if (object == DMU_GROUPUSED_OBJECT)
1466eda14cbcSMatt Macy 			dn = DMU_GROUPUSED_DNODE(os);
1467eda14cbcSMatt Macy 		else
1468eda14cbcSMatt Macy 			dn = DMU_PROJECTUSED_DNODE(os);
1469eda14cbcSMatt Macy 		if (dn == NULL)
1470eda14cbcSMatt Macy 			return (SET_ERROR(ENOENT));
1471eda14cbcSMatt Macy 		type = dn->dn_type;
1472eda14cbcSMatt Macy 		if ((flag & DNODE_MUST_BE_ALLOCATED) && type == DMU_OT_NONE)
1473eda14cbcSMatt Macy 			return (SET_ERROR(ENOENT));
1474eda14cbcSMatt Macy 		if ((flag & DNODE_MUST_BE_FREE) && type != DMU_OT_NONE)
1475eda14cbcSMatt Macy 			return (SET_ERROR(EEXIST));
1476eda14cbcSMatt Macy 		DNODE_VERIFY(dn);
1477eda14cbcSMatt Macy 		/* Don't actually hold if dry run, just return 0 */
1478eda14cbcSMatt Macy 		if (!(flag & DNODE_DRY_RUN)) {
1479eda14cbcSMatt Macy 			(void) zfs_refcount_add(&dn->dn_holds, tag);
1480eda14cbcSMatt Macy 			*dnp = dn;
1481eda14cbcSMatt Macy 		}
1482eda14cbcSMatt Macy 		return (0);
1483eda14cbcSMatt Macy 	}
1484eda14cbcSMatt Macy 
1485eda14cbcSMatt Macy 	if (object == 0 || object >= DN_MAX_OBJECT)
1486eda14cbcSMatt Macy 		return (SET_ERROR(EINVAL));
1487eda14cbcSMatt Macy 
1488eda14cbcSMatt Macy 	mdn = DMU_META_DNODE(os);
1489eda14cbcSMatt Macy 	ASSERT(mdn->dn_object == DMU_META_DNODE_OBJECT);
1490eda14cbcSMatt Macy 
1491eda14cbcSMatt Macy 	DNODE_VERIFY(mdn);
1492eda14cbcSMatt Macy 
1493eda14cbcSMatt Macy 	if (!RW_WRITE_HELD(&mdn->dn_struct_rwlock)) {
1494eda14cbcSMatt Macy 		rw_enter(&mdn->dn_struct_rwlock, RW_READER);
1495eda14cbcSMatt Macy 		drop_struct_lock = TRUE;
1496eda14cbcSMatt Macy 	}
1497eda14cbcSMatt Macy 
1498eda14cbcSMatt Macy 	blk = dbuf_whichblock(mdn, 0, object * sizeof (dnode_phys_t));
1499eda14cbcSMatt Macy 	db = dbuf_hold(mdn, blk, FTAG);
1500eda14cbcSMatt Macy 	if (drop_struct_lock)
1501eda14cbcSMatt Macy 		rw_exit(&mdn->dn_struct_rwlock);
1502eda14cbcSMatt Macy 	if (db == NULL) {
1503eda14cbcSMatt Macy 		DNODE_STAT_BUMP(dnode_hold_dbuf_hold);
1504eda14cbcSMatt Macy 		return (SET_ERROR(EIO));
1505eda14cbcSMatt Macy 	}
1506eda14cbcSMatt Macy 
1507eda14cbcSMatt Macy 	/*
1508eda14cbcSMatt Macy 	 * We do not need to decrypt to read the dnode so it doesn't matter
1509eda14cbcSMatt Macy 	 * if we get the encrypted or decrypted version.
1510eda14cbcSMatt Macy 	 */
1511c40487d4SMatt Macy 	err = dbuf_read(db, NULL, DB_RF_CANFAIL |
1512c40487d4SMatt Macy 	    DB_RF_NO_DECRYPT | DB_RF_NOPREFETCH);
1513eda14cbcSMatt Macy 	if (err) {
1514eda14cbcSMatt Macy 		DNODE_STAT_BUMP(dnode_hold_dbuf_read);
1515eda14cbcSMatt Macy 		dbuf_rele(db, FTAG);
1516eda14cbcSMatt Macy 		return (err);
1517eda14cbcSMatt Macy 	}
1518eda14cbcSMatt Macy 
1519eda14cbcSMatt Macy 	ASSERT3U(db->db.db_size, >=, 1<<DNODE_SHIFT);
1520eda14cbcSMatt Macy 	epb = db->db.db_size >> DNODE_SHIFT;
1521eda14cbcSMatt Macy 
1522eda14cbcSMatt Macy 	idx = object & (epb - 1);
1523eda14cbcSMatt Macy 	dn_block = (dnode_phys_t *)db->db.db_data;
1524eda14cbcSMatt Macy 
1525eda14cbcSMatt Macy 	ASSERT(DB_DNODE(db)->dn_type == DMU_OT_DNODE);
1526eda14cbcSMatt Macy 	dnc = dmu_buf_get_user(&db->db);
1527eda14cbcSMatt Macy 	dnh = NULL;
1528eda14cbcSMatt Macy 	if (dnc == NULL) {
1529eda14cbcSMatt Macy 		dnode_children_t *winner;
1530eda14cbcSMatt Macy 		int skip = 0;
1531eda14cbcSMatt Macy 
1532eda14cbcSMatt Macy 		dnc = kmem_zalloc(sizeof (dnode_children_t) +
1533eda14cbcSMatt Macy 		    epb * sizeof (dnode_handle_t), KM_SLEEP);
1534eda14cbcSMatt Macy 		dnc->dnc_count = epb;
1535eda14cbcSMatt Macy 		dnh = &dnc->dnc_children[0];
1536eda14cbcSMatt Macy 
1537eda14cbcSMatt Macy 		/* Initialize dnode slot status from dnode_phys_t */
1538eda14cbcSMatt Macy 		for (int i = 0; i < epb; i++) {
1539eda14cbcSMatt Macy 			zrl_init(&dnh[i].dnh_zrlock);
1540eda14cbcSMatt Macy 
1541eda14cbcSMatt Macy 			if (skip) {
1542eda14cbcSMatt Macy 				skip--;
1543eda14cbcSMatt Macy 				continue;
1544eda14cbcSMatt Macy 			}
1545eda14cbcSMatt Macy 
1546eda14cbcSMatt Macy 			if (dn_block[i].dn_type != DMU_OT_NONE) {
1547eda14cbcSMatt Macy 				int interior = dn_block[i].dn_extra_slots;
1548eda14cbcSMatt Macy 
1549eda14cbcSMatt Macy 				dnode_set_slots(dnc, i, 1, DN_SLOT_ALLOCATED);
1550eda14cbcSMatt Macy 				dnode_set_slots(dnc, i + 1, interior,
1551eda14cbcSMatt Macy 				    DN_SLOT_INTERIOR);
1552eda14cbcSMatt Macy 				skip = interior;
1553eda14cbcSMatt Macy 			} else {
1554eda14cbcSMatt Macy 				dnh[i].dnh_dnode = DN_SLOT_FREE;
1555eda14cbcSMatt Macy 				skip = 0;
1556eda14cbcSMatt Macy 			}
1557eda14cbcSMatt Macy 		}
1558eda14cbcSMatt Macy 
1559eda14cbcSMatt Macy 		dmu_buf_init_user(&dnc->dnc_dbu, NULL,
1560eda14cbcSMatt Macy 		    dnode_buf_evict_async, NULL);
1561eda14cbcSMatt Macy 		winner = dmu_buf_set_user(&db->db, &dnc->dnc_dbu);
1562eda14cbcSMatt Macy 		if (winner != NULL) {
1563eda14cbcSMatt Macy 
1564eda14cbcSMatt Macy 			for (int i = 0; i < epb; i++)
1565eda14cbcSMatt Macy 				zrl_destroy(&dnh[i].dnh_zrlock);
1566eda14cbcSMatt Macy 
1567eda14cbcSMatt Macy 			kmem_free(dnc, sizeof (dnode_children_t) +
1568eda14cbcSMatt Macy 			    epb * sizeof (dnode_handle_t));
1569eda14cbcSMatt Macy 			dnc = winner;
1570eda14cbcSMatt Macy 		}
1571eda14cbcSMatt Macy 	}
1572eda14cbcSMatt Macy 
1573eda14cbcSMatt Macy 	ASSERT(dnc->dnc_count == epb);
1574eda14cbcSMatt Macy 
1575eda14cbcSMatt Macy 	if (flag & DNODE_MUST_BE_ALLOCATED) {
1576eda14cbcSMatt Macy 		slots = 1;
1577eda14cbcSMatt Macy 
1578eda14cbcSMatt Macy 		dnode_slots_hold(dnc, idx, slots);
1579eda14cbcSMatt Macy 		dnh = &dnc->dnc_children[idx];
1580eda14cbcSMatt Macy 
1581eda14cbcSMatt Macy 		if (DN_SLOT_IS_PTR(dnh->dnh_dnode)) {
1582eda14cbcSMatt Macy 			dn = dnh->dnh_dnode;
1583eda14cbcSMatt Macy 		} else if (dnh->dnh_dnode == DN_SLOT_INTERIOR) {
1584eda14cbcSMatt Macy 			DNODE_STAT_BUMP(dnode_hold_alloc_interior);
1585eda14cbcSMatt Macy 			dnode_slots_rele(dnc, idx, slots);
1586eda14cbcSMatt Macy 			dbuf_rele(db, FTAG);
1587eda14cbcSMatt Macy 			return (SET_ERROR(EEXIST));
1588eda14cbcSMatt Macy 		} else if (dnh->dnh_dnode != DN_SLOT_ALLOCATED) {
1589eda14cbcSMatt Macy 			DNODE_STAT_BUMP(dnode_hold_alloc_misses);
1590eda14cbcSMatt Macy 			dnode_slots_rele(dnc, idx, slots);
1591eda14cbcSMatt Macy 			dbuf_rele(db, FTAG);
1592eda14cbcSMatt Macy 			return (SET_ERROR(ENOENT));
1593eda14cbcSMatt Macy 		} else {
1594eda14cbcSMatt Macy 			dnode_slots_rele(dnc, idx, slots);
1595eda14cbcSMatt Macy 			while (!dnode_slots_tryenter(dnc, idx, slots)) {
1596eda14cbcSMatt Macy 				DNODE_STAT_BUMP(dnode_hold_alloc_lock_retry);
1597c7046f76SMartin Matuska 				kpreempt(KPREEMPT_SYNC);
1598eda14cbcSMatt Macy 			}
1599eda14cbcSMatt Macy 
1600eda14cbcSMatt Macy 			/*
1601eda14cbcSMatt Macy 			 * Someone else won the race and called dnode_create()
1602eda14cbcSMatt Macy 			 * after we checked DN_SLOT_IS_PTR() above but before
1603eda14cbcSMatt Macy 			 * we acquired the lock.
1604eda14cbcSMatt Macy 			 */
1605eda14cbcSMatt Macy 			if (DN_SLOT_IS_PTR(dnh->dnh_dnode)) {
1606eda14cbcSMatt Macy 				DNODE_STAT_BUMP(dnode_hold_alloc_lock_misses);
1607eda14cbcSMatt Macy 				dn = dnh->dnh_dnode;
1608eda14cbcSMatt Macy 			} else {
1609eda14cbcSMatt Macy 				dn = dnode_create(os, dn_block + idx, db,
1610eda14cbcSMatt Macy 				    object, dnh);
1611a2b560ccSMartin Matuska 				dmu_buf_add_user_size(&db->db,
1612a2b560ccSMartin Matuska 				    sizeof (dnode_t));
1613eda14cbcSMatt Macy 			}
1614eda14cbcSMatt Macy 		}
1615eda14cbcSMatt Macy 
1616eda14cbcSMatt Macy 		mutex_enter(&dn->dn_mtx);
1617eda14cbcSMatt Macy 		if (dn->dn_type == DMU_OT_NONE || dn->dn_free_txg != 0) {
1618eda14cbcSMatt Macy 			DNODE_STAT_BUMP(dnode_hold_alloc_type_none);
1619eda14cbcSMatt Macy 			mutex_exit(&dn->dn_mtx);
1620eda14cbcSMatt Macy 			dnode_slots_rele(dnc, idx, slots);
1621eda14cbcSMatt Macy 			dbuf_rele(db, FTAG);
1622eda14cbcSMatt Macy 			return (SET_ERROR(ENOENT));
1623eda14cbcSMatt Macy 		}
1624eda14cbcSMatt Macy 
1625eda14cbcSMatt Macy 		/* Don't actually hold if dry run, just return 0 */
1626eda14cbcSMatt Macy 		if (flag & DNODE_DRY_RUN) {
1627eda14cbcSMatt Macy 			mutex_exit(&dn->dn_mtx);
1628eda14cbcSMatt Macy 			dnode_slots_rele(dnc, idx, slots);
1629eda14cbcSMatt Macy 			dbuf_rele(db, FTAG);
1630eda14cbcSMatt Macy 			return (0);
1631eda14cbcSMatt Macy 		}
1632eda14cbcSMatt Macy 
1633eda14cbcSMatt Macy 		DNODE_STAT_BUMP(dnode_hold_alloc_hits);
1634eda14cbcSMatt Macy 	} else if (flag & DNODE_MUST_BE_FREE) {
1635eda14cbcSMatt Macy 
1636eda14cbcSMatt Macy 		if (idx + slots - 1 >= DNODES_PER_BLOCK) {
1637eda14cbcSMatt Macy 			DNODE_STAT_BUMP(dnode_hold_free_overflow);
1638eda14cbcSMatt Macy 			dbuf_rele(db, FTAG);
1639eda14cbcSMatt Macy 			return (SET_ERROR(ENOSPC));
1640eda14cbcSMatt Macy 		}
1641eda14cbcSMatt Macy 
1642eda14cbcSMatt Macy 		dnode_slots_hold(dnc, idx, slots);
1643eda14cbcSMatt Macy 
1644eda14cbcSMatt Macy 		if (!dnode_check_slots_free(dnc, idx, slots)) {
1645eda14cbcSMatt Macy 			DNODE_STAT_BUMP(dnode_hold_free_misses);
1646eda14cbcSMatt Macy 			dnode_slots_rele(dnc, idx, slots);
1647eda14cbcSMatt Macy 			dbuf_rele(db, FTAG);
1648eda14cbcSMatt Macy 			return (SET_ERROR(ENOSPC));
1649eda14cbcSMatt Macy 		}
1650eda14cbcSMatt Macy 
1651eda14cbcSMatt Macy 		dnode_slots_rele(dnc, idx, slots);
1652eda14cbcSMatt Macy 		while (!dnode_slots_tryenter(dnc, idx, slots)) {
1653eda14cbcSMatt Macy 			DNODE_STAT_BUMP(dnode_hold_free_lock_retry);
1654c7046f76SMartin Matuska 			kpreempt(KPREEMPT_SYNC);
1655eda14cbcSMatt Macy 		}
1656eda14cbcSMatt Macy 
1657eda14cbcSMatt Macy 		if (!dnode_check_slots_free(dnc, idx, slots)) {
1658eda14cbcSMatt Macy 			DNODE_STAT_BUMP(dnode_hold_free_lock_misses);
1659eda14cbcSMatt Macy 			dnode_slots_rele(dnc, idx, slots);
1660eda14cbcSMatt Macy 			dbuf_rele(db, FTAG);
1661eda14cbcSMatt Macy 			return (SET_ERROR(ENOSPC));
1662eda14cbcSMatt Macy 		}
1663eda14cbcSMatt Macy 
1664eda14cbcSMatt Macy 		/*
1665eda14cbcSMatt Macy 		 * Allocated but otherwise free dnodes which would
1666eda14cbcSMatt Macy 		 * be in the interior of a multi-slot dnodes need
1667eda14cbcSMatt Macy 		 * to be freed.  Single slot dnodes can be safely
1668eda14cbcSMatt Macy 		 * re-purposed as a performance optimization.
1669eda14cbcSMatt Macy 		 */
1670a2b560ccSMartin Matuska 		if (slots > 1) {
1671a2b560ccSMartin Matuska 			uint_t reclaimed =
1672eda14cbcSMatt Macy 			    dnode_reclaim_slots(dnc, idx + 1, slots - 1);
1673a2b560ccSMartin Matuska 			if (reclaimed > 0)
1674a2b560ccSMartin Matuska 				dmu_buf_sub_user_size(&db->db,
1675a2b560ccSMartin Matuska 				    reclaimed * sizeof (dnode_t));
1676a2b560ccSMartin Matuska 		}
1677eda14cbcSMatt Macy 
1678eda14cbcSMatt Macy 		dnh = &dnc->dnc_children[idx];
1679eda14cbcSMatt Macy 		if (DN_SLOT_IS_PTR(dnh->dnh_dnode)) {
1680eda14cbcSMatt Macy 			dn = dnh->dnh_dnode;
1681eda14cbcSMatt Macy 		} else {
1682eda14cbcSMatt Macy 			dn = dnode_create(os, dn_block + idx, db,
1683eda14cbcSMatt Macy 			    object, dnh);
1684a2b560ccSMartin Matuska 			dmu_buf_add_user_size(&db->db, sizeof (dnode_t));
1685eda14cbcSMatt Macy 		}
1686eda14cbcSMatt Macy 
1687eda14cbcSMatt Macy 		mutex_enter(&dn->dn_mtx);
1688eda14cbcSMatt Macy 		if (!zfs_refcount_is_zero(&dn->dn_holds) || dn->dn_free_txg) {
1689eda14cbcSMatt Macy 			DNODE_STAT_BUMP(dnode_hold_free_refcount);
1690eda14cbcSMatt Macy 			mutex_exit(&dn->dn_mtx);
1691eda14cbcSMatt Macy 			dnode_slots_rele(dnc, idx, slots);
1692eda14cbcSMatt Macy 			dbuf_rele(db, FTAG);
1693eda14cbcSMatt Macy 			return (SET_ERROR(EEXIST));
1694eda14cbcSMatt Macy 		}
1695eda14cbcSMatt Macy 
1696eda14cbcSMatt Macy 		/* Don't actually hold if dry run, just return 0 */
1697eda14cbcSMatt Macy 		if (flag & DNODE_DRY_RUN) {
1698eda14cbcSMatt Macy 			mutex_exit(&dn->dn_mtx);
1699eda14cbcSMatt Macy 			dnode_slots_rele(dnc, idx, slots);
1700eda14cbcSMatt Macy 			dbuf_rele(db, FTAG);
1701eda14cbcSMatt Macy 			return (0);
1702eda14cbcSMatt Macy 		}
1703eda14cbcSMatt Macy 
1704eda14cbcSMatt Macy 		dnode_set_slots(dnc, idx + 1, slots - 1, DN_SLOT_INTERIOR);
1705eda14cbcSMatt Macy 		DNODE_STAT_BUMP(dnode_hold_free_hits);
1706eda14cbcSMatt Macy 	} else {
1707eda14cbcSMatt Macy 		dbuf_rele(db, FTAG);
1708eda14cbcSMatt Macy 		return (SET_ERROR(EINVAL));
1709eda14cbcSMatt Macy 	}
1710eda14cbcSMatt Macy 
1711eda14cbcSMatt Macy 	ASSERT0(dn->dn_free_txg);
1712eda14cbcSMatt Macy 
1713eda14cbcSMatt Macy 	if (zfs_refcount_add(&dn->dn_holds, tag) == 1)
1714eda14cbcSMatt Macy 		dbuf_add_ref(db, dnh);
1715eda14cbcSMatt Macy 
1716eda14cbcSMatt Macy 	mutex_exit(&dn->dn_mtx);
1717eda14cbcSMatt Macy 
1718eda14cbcSMatt Macy 	/* Now we can rely on the hold to prevent the dnode from moving. */
1719eda14cbcSMatt Macy 	dnode_slots_rele(dnc, idx, slots);
1720eda14cbcSMatt Macy 
1721eda14cbcSMatt Macy 	DNODE_VERIFY(dn);
1722eda14cbcSMatt Macy 	ASSERT3P(dnp, !=, NULL);
1723eda14cbcSMatt Macy 	ASSERT3P(dn->dn_dbuf, ==, db);
1724eda14cbcSMatt Macy 	ASSERT3U(dn->dn_object, ==, object);
1725eda14cbcSMatt Macy 	dbuf_rele(db, FTAG);
1726eda14cbcSMatt Macy 
1727eda14cbcSMatt Macy 	*dnp = dn;
1728eda14cbcSMatt Macy 	return (0);
1729eda14cbcSMatt Macy }
1730eda14cbcSMatt Macy 
1731eda14cbcSMatt Macy /*
1732eda14cbcSMatt Macy  * Return held dnode if the object is allocated, NULL if not.
1733eda14cbcSMatt Macy  */
1734eda14cbcSMatt Macy int
1735a0b956f5SMartin Matuska dnode_hold(objset_t *os, uint64_t object, const void *tag, dnode_t **dnp)
1736eda14cbcSMatt Macy {
1737eda14cbcSMatt Macy 	return (dnode_hold_impl(os, object, DNODE_MUST_BE_ALLOCATED, 0, tag,
1738eda14cbcSMatt Macy 	    dnp));
1739eda14cbcSMatt Macy }
1740eda14cbcSMatt Macy 
1741eda14cbcSMatt Macy /*
1742eda14cbcSMatt Macy  * Can only add a reference if there is already at least one
1743eda14cbcSMatt Macy  * reference on the dnode.  Returns FALSE if unable to add a
1744eda14cbcSMatt Macy  * new reference.
1745eda14cbcSMatt Macy  */
1746eda14cbcSMatt Macy boolean_t
1747a0b956f5SMartin Matuska dnode_add_ref(dnode_t *dn, const void *tag)
1748eda14cbcSMatt Macy {
1749eda14cbcSMatt Macy 	mutex_enter(&dn->dn_mtx);
1750eda14cbcSMatt Macy 	if (zfs_refcount_is_zero(&dn->dn_holds)) {
1751eda14cbcSMatt Macy 		mutex_exit(&dn->dn_mtx);
1752eda14cbcSMatt Macy 		return (FALSE);
1753eda14cbcSMatt Macy 	}
1754eda14cbcSMatt Macy 	VERIFY(1 < zfs_refcount_add(&dn->dn_holds, tag));
1755eda14cbcSMatt Macy 	mutex_exit(&dn->dn_mtx);
1756eda14cbcSMatt Macy 	return (TRUE);
1757eda14cbcSMatt Macy }
1758eda14cbcSMatt Macy 
1759eda14cbcSMatt Macy void
1760a0b956f5SMartin Matuska dnode_rele(dnode_t *dn, const void *tag)
1761eda14cbcSMatt Macy {
1762eda14cbcSMatt Macy 	mutex_enter(&dn->dn_mtx);
1763eda14cbcSMatt Macy 	dnode_rele_and_unlock(dn, tag, B_FALSE);
1764eda14cbcSMatt Macy }
1765eda14cbcSMatt Macy 
1766eda14cbcSMatt Macy void
1767a0b956f5SMartin Matuska dnode_rele_and_unlock(dnode_t *dn, const void *tag, boolean_t evicting)
1768eda14cbcSMatt Macy {
1769eda14cbcSMatt Macy 	uint64_t refs;
1770eda14cbcSMatt Macy 	/* Get while the hold prevents the dnode from moving. */
1771eda14cbcSMatt Macy 	dmu_buf_impl_t *db = dn->dn_dbuf;
1772eda14cbcSMatt Macy 	dnode_handle_t *dnh = dn->dn_handle;
1773eda14cbcSMatt Macy 
1774eda14cbcSMatt Macy 	refs = zfs_refcount_remove(&dn->dn_holds, tag);
1775eda14cbcSMatt Macy 	if (refs == 0)
1776eda14cbcSMatt Macy 		cv_broadcast(&dn->dn_nodnholds);
1777eda14cbcSMatt Macy 	mutex_exit(&dn->dn_mtx);
1778eda14cbcSMatt Macy 	/* dnode could get destroyed at this point, so don't use it anymore */
1779eda14cbcSMatt Macy 
1780eda14cbcSMatt Macy 	/*
1781eda14cbcSMatt Macy 	 * It's unsafe to release the last hold on a dnode by dnode_rele() or
1782eda14cbcSMatt Macy 	 * indirectly by dbuf_rele() while relying on the dnode handle to
1783eda14cbcSMatt Macy 	 * prevent the dnode from moving, since releasing the last hold could
1784eda14cbcSMatt Macy 	 * result in the dnode's parent dbuf evicting its dnode handles. For
1785eda14cbcSMatt Macy 	 * that reason anyone calling dnode_rele() or dbuf_rele() without some
1786eda14cbcSMatt Macy 	 * other direct or indirect hold on the dnode must first drop the dnode
1787eda14cbcSMatt Macy 	 * handle.
1788eda14cbcSMatt Macy 	 */
1789e92ffd9bSMartin Matuska #ifdef ZFS_DEBUG
1790*ce4dcb97SMartin Matuska 	ASSERT(refs > 0 || zrl_owner(&dnh->dnh_zrlock) != curthread);
1791e92ffd9bSMartin Matuska #endif
1792eda14cbcSMatt Macy 
1793eda14cbcSMatt Macy 	/* NOTE: the DNODE_DNODE does not have a dn_dbuf */
1794eda14cbcSMatt Macy 	if (refs == 0 && db != NULL) {
1795eda14cbcSMatt Macy 		/*
1796eda14cbcSMatt Macy 		 * Another thread could add a hold to the dnode handle in
1797eda14cbcSMatt Macy 		 * dnode_hold_impl() while holding the parent dbuf. Since the
1798eda14cbcSMatt Macy 		 * hold on the parent dbuf prevents the handle from being
1799eda14cbcSMatt Macy 		 * destroyed, the hold on the handle is OK. We can't yet assert
1800eda14cbcSMatt Macy 		 * that the handle has zero references, but that will be
1801eda14cbcSMatt Macy 		 * asserted anyway when the handle gets destroyed.
1802eda14cbcSMatt Macy 		 */
1803eda14cbcSMatt Macy 		mutex_enter(&db->db_mtx);
1804eda14cbcSMatt Macy 		dbuf_rele_and_unlock(db, dnh, evicting);
1805eda14cbcSMatt Macy 	}
1806eda14cbcSMatt Macy }
1807eda14cbcSMatt Macy 
1808eda14cbcSMatt Macy /*
1809eda14cbcSMatt Macy  * Test whether we can create a dnode at the specified location.
1810eda14cbcSMatt Macy  */
1811eda14cbcSMatt Macy int
1812eda14cbcSMatt Macy dnode_try_claim(objset_t *os, uint64_t object, int slots)
1813eda14cbcSMatt Macy {
1814eda14cbcSMatt Macy 	return (dnode_hold_impl(os, object, DNODE_MUST_BE_FREE | DNODE_DRY_RUN,
1815eda14cbcSMatt Macy 	    slots, NULL, NULL));
1816eda14cbcSMatt Macy }
1817eda14cbcSMatt Macy 
181881b22a98SMartin Matuska /*
18192276e539SMartin Matuska  * Checks if the dnode itself is dirty, or is carrying any uncommitted records.
18202276e539SMartin Matuska  * It is important to check both conditions, as some operations (eg appending
18212276e539SMartin Matuska  * to a file) can dirty both as a single logical unit, but they are not synced
18222276e539SMartin Matuska  * out atomically, so checking one and not the other can result in an object
18232276e539SMartin Matuska  * appearing to be clean mid-way through a commit.
18242276e539SMartin Matuska  *
18252276e539SMartin Matuska  * Do not change this lightly! If you get it wrong, dmu_offset_next() can
18262276e539SMartin Matuska  * detect a hole where there is really data, leading to silent corruption.
182781b22a98SMartin Matuska  */
182881b22a98SMartin Matuska boolean_t
182981b22a98SMartin Matuska dnode_is_dirty(dnode_t *dn)
183081b22a98SMartin Matuska {
183181b22a98SMartin Matuska 	mutex_enter(&dn->dn_mtx);
183263ee747fSMateusz Guzik 
183381b22a98SMartin Matuska 	for (int i = 0; i < TXG_SIZE; i++) {
18342276e539SMartin Matuska 		if (multilist_link_active(&dn->dn_dirty_link[i]) ||
18352276e539SMartin Matuska 		    !list_is_empty(&dn->dn_dirty_records[i])) {
183681b22a98SMartin Matuska 			mutex_exit(&dn->dn_mtx);
183781b22a98SMartin Matuska 			return (B_TRUE);
183881b22a98SMartin Matuska 		}
183981b22a98SMartin Matuska 	}
184063ee747fSMateusz Guzik 
184181b22a98SMartin Matuska 	mutex_exit(&dn->dn_mtx);
184281b22a98SMartin Matuska 
184381b22a98SMartin Matuska 	return (B_FALSE);
184481b22a98SMartin Matuska }
184581b22a98SMartin Matuska 
1846eda14cbcSMatt Macy void
1847eda14cbcSMatt Macy dnode_setdirty(dnode_t *dn, dmu_tx_t *tx)
1848eda14cbcSMatt Macy {
1849eda14cbcSMatt Macy 	objset_t *os = dn->dn_objset;
1850eda14cbcSMatt Macy 	uint64_t txg = tx->tx_txg;
1851eda14cbcSMatt Macy 
1852eda14cbcSMatt Macy 	if (DMU_OBJECT_IS_SPECIAL(dn->dn_object)) {
1853eda14cbcSMatt Macy 		dsl_dataset_dirty(os->os_dsl_dataset, tx);
1854eda14cbcSMatt Macy 		return;
1855eda14cbcSMatt Macy 	}
1856eda14cbcSMatt Macy 
1857eda14cbcSMatt Macy 	DNODE_VERIFY(dn);
1858eda14cbcSMatt Macy 
1859eda14cbcSMatt Macy #ifdef ZFS_DEBUG
1860eda14cbcSMatt Macy 	mutex_enter(&dn->dn_mtx);
1861eda14cbcSMatt Macy 	ASSERT(dn->dn_phys->dn_type || dn->dn_allocated_txg);
1862eda14cbcSMatt Macy 	ASSERT(dn->dn_free_txg == 0 || dn->dn_free_txg >= txg);
1863eda14cbcSMatt Macy 	mutex_exit(&dn->dn_mtx);
1864eda14cbcSMatt Macy #endif
1865eda14cbcSMatt Macy 
1866eda14cbcSMatt Macy 	/*
1867eda14cbcSMatt Macy 	 * Determine old uid/gid when necessary
1868eda14cbcSMatt Macy 	 */
1869eda14cbcSMatt Macy 	dmu_objset_userquota_get_ids(dn, B_TRUE, tx);
1870eda14cbcSMatt Macy 
18713ff01b23SMartin Matuska 	multilist_t *dirtylist = &os->os_dirty_dnodes[txg & TXG_MASK];
1872eda14cbcSMatt Macy 	multilist_sublist_t *mls = multilist_sublist_lock_obj(dirtylist, dn);
1873eda14cbcSMatt Macy 
1874eda14cbcSMatt Macy 	/*
1875eda14cbcSMatt Macy 	 * If we are already marked dirty, we're done.
1876eda14cbcSMatt Macy 	 */
1877eda14cbcSMatt Macy 	if (multilist_link_active(&dn->dn_dirty_link[txg & TXG_MASK])) {
1878eda14cbcSMatt Macy 		multilist_sublist_unlock(mls);
1879eda14cbcSMatt Macy 		return;
1880eda14cbcSMatt Macy 	}
1881eda14cbcSMatt Macy 
1882eda14cbcSMatt Macy 	ASSERT(!zfs_refcount_is_zero(&dn->dn_holds) ||
1883eda14cbcSMatt Macy 	    !avl_is_empty(&dn->dn_dbufs));
1884eda14cbcSMatt Macy 	ASSERT(dn->dn_datablksz != 0);
1885eda14cbcSMatt Macy 	ASSERT0(dn->dn_next_bonuslen[txg & TXG_MASK]);
1886eda14cbcSMatt Macy 	ASSERT0(dn->dn_next_blksz[txg & TXG_MASK]);
1887eda14cbcSMatt Macy 	ASSERT0(dn->dn_next_bonustype[txg & TXG_MASK]);
1888eda14cbcSMatt Macy 
1889eda14cbcSMatt Macy 	dprintf_ds(os->os_dsl_dataset, "obj=%llu txg=%llu\n",
189033b8c039SMartin Matuska 	    (u_longlong_t)dn->dn_object, (u_longlong_t)txg);
1891eda14cbcSMatt Macy 
1892eda14cbcSMatt Macy 	multilist_sublist_insert_head(mls, dn);
1893eda14cbcSMatt Macy 
1894eda14cbcSMatt Macy 	multilist_sublist_unlock(mls);
1895eda14cbcSMatt Macy 
1896eda14cbcSMatt Macy 	/*
1897eda14cbcSMatt Macy 	 * The dnode maintains a hold on its containing dbuf as
1898eda14cbcSMatt Macy 	 * long as there are holds on it.  Each instantiated child
1899eda14cbcSMatt Macy 	 * dbuf maintains a hold on the dnode.  When the last child
1900eda14cbcSMatt Macy 	 * drops its hold, the dnode will drop its hold on the
1901eda14cbcSMatt Macy 	 * containing dbuf. We add a "dirty hold" here so that the
1902eda14cbcSMatt Macy 	 * dnode will hang around after we finish processing its
1903eda14cbcSMatt Macy 	 * children.
1904eda14cbcSMatt Macy 	 */
1905eda14cbcSMatt Macy 	VERIFY(dnode_add_ref(dn, (void *)(uintptr_t)tx->tx_txg));
1906eda14cbcSMatt Macy 
1907eda14cbcSMatt Macy 	(void) dbuf_dirty(dn->dn_dbuf, tx);
1908eda14cbcSMatt Macy 
1909eda14cbcSMatt Macy 	dsl_dataset_dirty(os->os_dsl_dataset, tx);
1910eda14cbcSMatt Macy }
1911eda14cbcSMatt Macy 
1912eda14cbcSMatt Macy void
1913eda14cbcSMatt Macy dnode_free(dnode_t *dn, dmu_tx_t *tx)
1914eda14cbcSMatt Macy {
1915eda14cbcSMatt Macy 	mutex_enter(&dn->dn_mtx);
1916eda14cbcSMatt Macy 	if (dn->dn_type == DMU_OT_NONE || dn->dn_free_txg) {
1917eda14cbcSMatt Macy 		mutex_exit(&dn->dn_mtx);
1918eda14cbcSMatt Macy 		return;
1919eda14cbcSMatt Macy 	}
1920eda14cbcSMatt Macy 	dn->dn_free_txg = tx->tx_txg;
1921eda14cbcSMatt Macy 	mutex_exit(&dn->dn_mtx);
1922eda14cbcSMatt Macy 
1923eda14cbcSMatt Macy 	dnode_setdirty(dn, tx);
1924eda14cbcSMatt Macy }
1925eda14cbcSMatt Macy 
1926eda14cbcSMatt Macy /*
1927eda14cbcSMatt Macy  * Try to change the block size for the indicated dnode.  This can only
1928eda14cbcSMatt Macy  * succeed if there are no blocks allocated or dirty beyond first block
1929eda14cbcSMatt Macy  */
1930eda14cbcSMatt Macy int
1931eda14cbcSMatt Macy dnode_set_blksz(dnode_t *dn, uint64_t size, int ibs, dmu_tx_t *tx)
1932eda14cbcSMatt Macy {
1933eda14cbcSMatt Macy 	dmu_buf_impl_t *db;
1934eda14cbcSMatt Macy 	int err;
1935eda14cbcSMatt Macy 
1936eda14cbcSMatt Macy 	ASSERT3U(size, <=, spa_maxblocksize(dmu_objset_spa(dn->dn_objset)));
1937eda14cbcSMatt Macy 	if (size == 0)
1938eda14cbcSMatt Macy 		size = SPA_MINBLOCKSIZE;
1939eda14cbcSMatt Macy 	else
1940eda14cbcSMatt Macy 		size = P2ROUNDUP(size, SPA_MINBLOCKSIZE);
1941eda14cbcSMatt Macy 
1942eda14cbcSMatt Macy 	if (ibs == dn->dn_indblkshift)
1943eda14cbcSMatt Macy 		ibs = 0;
1944eda14cbcSMatt Macy 
1945315ee00fSMartin Matuska 	if (size == dn->dn_datablksz && ibs == 0)
1946eda14cbcSMatt Macy 		return (0);
1947eda14cbcSMatt Macy 
1948eda14cbcSMatt Macy 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
1949eda14cbcSMatt Macy 
1950eda14cbcSMatt Macy 	/* Check for any allocated blocks beyond the first */
1951eda14cbcSMatt Macy 	if (dn->dn_maxblkid != 0)
1952eda14cbcSMatt Macy 		goto fail;
1953eda14cbcSMatt Macy 
1954eda14cbcSMatt Macy 	mutex_enter(&dn->dn_dbufs_mtx);
1955eda14cbcSMatt Macy 	for (db = avl_first(&dn->dn_dbufs); db != NULL;
1956eda14cbcSMatt Macy 	    db = AVL_NEXT(&dn->dn_dbufs, db)) {
1957eda14cbcSMatt Macy 		if (db->db_blkid != 0 && db->db_blkid != DMU_BONUS_BLKID &&
1958eda14cbcSMatt Macy 		    db->db_blkid != DMU_SPILL_BLKID) {
1959eda14cbcSMatt Macy 			mutex_exit(&dn->dn_dbufs_mtx);
1960eda14cbcSMatt Macy 			goto fail;
1961eda14cbcSMatt Macy 		}
1962eda14cbcSMatt Macy 	}
1963eda14cbcSMatt Macy 	mutex_exit(&dn->dn_dbufs_mtx);
1964eda14cbcSMatt Macy 
1965eda14cbcSMatt Macy 	if (ibs && dn->dn_nlevels != 1)
1966eda14cbcSMatt Macy 		goto fail;
1967eda14cbcSMatt Macy 
1968315ee00fSMartin Matuska 	dnode_setdirty(dn, tx);
1969315ee00fSMartin Matuska 	if (size != dn->dn_datablksz) {
1970eda14cbcSMatt Macy 		/* resize the old block */
1971eda14cbcSMatt Macy 		err = dbuf_hold_impl(dn, 0, 0, TRUE, FALSE, FTAG, &db);
1972eda14cbcSMatt Macy 		if (err == 0) {
1973eda14cbcSMatt Macy 			dbuf_new_size(db, size, tx);
1974eda14cbcSMatt Macy 		} else if (err != ENOENT) {
1975eda14cbcSMatt Macy 			goto fail;
1976eda14cbcSMatt Macy 		}
1977eda14cbcSMatt Macy 
1978eda14cbcSMatt Macy 		dnode_setdblksz(dn, size);
1979eda14cbcSMatt Macy 		dn->dn_next_blksz[tx->tx_txg & TXG_MASK] = size;
1980315ee00fSMartin Matuska 		if (db)
1981315ee00fSMartin Matuska 			dbuf_rele(db, FTAG);
1982315ee00fSMartin Matuska 	}
1983eda14cbcSMatt Macy 	if (ibs) {
1984eda14cbcSMatt Macy 		dn->dn_indblkshift = ibs;
1985eda14cbcSMatt Macy 		dn->dn_next_indblkshift[tx->tx_txg & TXG_MASK] = ibs;
1986eda14cbcSMatt Macy 	}
1987eda14cbcSMatt Macy 
1988eda14cbcSMatt Macy 	rw_exit(&dn->dn_struct_rwlock);
1989eda14cbcSMatt Macy 	return (0);
1990eda14cbcSMatt Macy 
1991eda14cbcSMatt Macy fail:
1992eda14cbcSMatt Macy 	rw_exit(&dn->dn_struct_rwlock);
1993eda14cbcSMatt Macy 	return (SET_ERROR(ENOTSUP));
1994eda14cbcSMatt Macy }
1995eda14cbcSMatt Macy 
1996eda14cbcSMatt Macy static void
1997eda14cbcSMatt Macy dnode_set_nlevels_impl(dnode_t *dn, int new_nlevels, dmu_tx_t *tx)
1998eda14cbcSMatt Macy {
1999eda14cbcSMatt Macy 	uint64_t txgoff = tx->tx_txg & TXG_MASK;
2000eda14cbcSMatt Macy 	int old_nlevels = dn->dn_nlevels;
2001eda14cbcSMatt Macy 	dmu_buf_impl_t *db;
2002eda14cbcSMatt Macy 	list_t *list;
2003eda14cbcSMatt Macy 	dbuf_dirty_record_t *new, *dr, *dr_next;
2004eda14cbcSMatt Macy 
2005eda14cbcSMatt Macy 	ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock));
2006eda14cbcSMatt Macy 
20077877fdebSMatt Macy 	ASSERT3U(new_nlevels, >, dn->dn_nlevels);
2008eda14cbcSMatt Macy 	dn->dn_nlevels = new_nlevels;
2009eda14cbcSMatt Macy 
2010eda14cbcSMatt Macy 	ASSERT3U(new_nlevels, >, dn->dn_next_nlevels[txgoff]);
2011eda14cbcSMatt Macy 	dn->dn_next_nlevels[txgoff] = new_nlevels;
2012eda14cbcSMatt Macy 
2013eda14cbcSMatt Macy 	/* dirty the left indirects */
2014eda14cbcSMatt Macy 	db = dbuf_hold_level(dn, old_nlevels, 0, FTAG);
2015eda14cbcSMatt Macy 	ASSERT(db != NULL);
2016eda14cbcSMatt Macy 	new = dbuf_dirty(db, tx);
2017eda14cbcSMatt Macy 	dbuf_rele(db, FTAG);
2018eda14cbcSMatt Macy 
2019eda14cbcSMatt Macy 	/* transfer the dirty records to the new indirect */
2020eda14cbcSMatt Macy 	mutex_enter(&dn->dn_mtx);
2021eda14cbcSMatt Macy 	mutex_enter(&new->dt.di.dr_mtx);
2022eda14cbcSMatt Macy 	list = &dn->dn_dirty_records[txgoff];
2023eda14cbcSMatt Macy 	for (dr = list_head(list); dr; dr = dr_next) {
2024eda14cbcSMatt Macy 		dr_next = list_next(&dn->dn_dirty_records[txgoff], dr);
20257877fdebSMatt Macy 
20267877fdebSMatt Macy 		IMPLY(dr->dr_dbuf == NULL, old_nlevels == 1);
20277877fdebSMatt Macy 		if (dr->dr_dbuf == NULL ||
20287877fdebSMatt Macy 		    (dr->dr_dbuf->db_level == old_nlevels - 1 &&
2029eda14cbcSMatt Macy 		    dr->dr_dbuf->db_blkid != DMU_BONUS_BLKID &&
20307877fdebSMatt Macy 		    dr->dr_dbuf->db_blkid != DMU_SPILL_BLKID)) {
2031eda14cbcSMatt Macy 			list_remove(&dn->dn_dirty_records[txgoff], dr);
2032eda14cbcSMatt Macy 			list_insert_tail(&new->dt.di.dr_children, dr);
2033eda14cbcSMatt Macy 			dr->dr_parent = new;
2034eda14cbcSMatt Macy 		}
2035eda14cbcSMatt Macy 	}
2036eda14cbcSMatt Macy 	mutex_exit(&new->dt.di.dr_mtx);
2037eda14cbcSMatt Macy 	mutex_exit(&dn->dn_mtx);
2038eda14cbcSMatt Macy }
2039eda14cbcSMatt Macy 
2040eda14cbcSMatt Macy int
2041eda14cbcSMatt Macy dnode_set_nlevels(dnode_t *dn, int nlevels, dmu_tx_t *tx)
2042eda14cbcSMatt Macy {
2043eda14cbcSMatt Macy 	int ret = 0;
2044eda14cbcSMatt Macy 
2045eda14cbcSMatt Macy 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
2046eda14cbcSMatt Macy 
2047eda14cbcSMatt Macy 	if (dn->dn_nlevels == nlevels) {
2048eda14cbcSMatt Macy 		ret = 0;
2049eda14cbcSMatt Macy 		goto out;
2050eda14cbcSMatt Macy 	} else if (nlevels < dn->dn_nlevels) {
2051eda14cbcSMatt Macy 		ret = SET_ERROR(EINVAL);
2052eda14cbcSMatt Macy 		goto out;
2053eda14cbcSMatt Macy 	}
2054eda14cbcSMatt Macy 
2055eda14cbcSMatt Macy 	dnode_set_nlevels_impl(dn, nlevels, tx);
2056eda14cbcSMatt Macy 
2057eda14cbcSMatt Macy out:
2058eda14cbcSMatt Macy 	rw_exit(&dn->dn_struct_rwlock);
2059eda14cbcSMatt Macy 	return (ret);
2060eda14cbcSMatt Macy }
2061eda14cbcSMatt Macy 
2062eda14cbcSMatt Macy /* read-holding callers must not rely on the lock being continuously held */
2063eda14cbcSMatt Macy void
2064eda14cbcSMatt Macy dnode_new_blkid(dnode_t *dn, uint64_t blkid, dmu_tx_t *tx, boolean_t have_read,
2065eda14cbcSMatt Macy     boolean_t force)
2066eda14cbcSMatt Macy {
2067eda14cbcSMatt Macy 	int epbs, new_nlevels;
2068eda14cbcSMatt Macy 	uint64_t sz;
2069eda14cbcSMatt Macy 
2070eda14cbcSMatt Macy 	ASSERT(blkid != DMU_BONUS_BLKID);
2071eda14cbcSMatt Macy 
2072eda14cbcSMatt Macy 	ASSERT(have_read ?
2073eda14cbcSMatt Macy 	    RW_READ_HELD(&dn->dn_struct_rwlock) :
2074eda14cbcSMatt Macy 	    RW_WRITE_HELD(&dn->dn_struct_rwlock));
2075eda14cbcSMatt Macy 
2076eda14cbcSMatt Macy 	/*
2077eda14cbcSMatt Macy 	 * if we have a read-lock, check to see if we need to do any work
2078eda14cbcSMatt Macy 	 * before upgrading to a write-lock.
2079eda14cbcSMatt Macy 	 */
2080eda14cbcSMatt Macy 	if (have_read) {
2081eda14cbcSMatt Macy 		if (blkid <= dn->dn_maxblkid)
2082eda14cbcSMatt Macy 			return;
2083eda14cbcSMatt Macy 
2084eda14cbcSMatt Macy 		if (!rw_tryupgrade(&dn->dn_struct_rwlock)) {
2085eda14cbcSMatt Macy 			rw_exit(&dn->dn_struct_rwlock);
2086eda14cbcSMatt Macy 			rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
2087eda14cbcSMatt Macy 		}
2088eda14cbcSMatt Macy 	}
2089eda14cbcSMatt Macy 
2090eda14cbcSMatt Macy 	/*
2091eda14cbcSMatt Macy 	 * Raw sends (indicated by the force flag) require that we take the
2092eda14cbcSMatt Macy 	 * given blkid even if the value is lower than the current value.
2093eda14cbcSMatt Macy 	 */
2094eda14cbcSMatt Macy 	if (!force && blkid <= dn->dn_maxblkid)
2095eda14cbcSMatt Macy 		goto out;
2096eda14cbcSMatt Macy 
2097eda14cbcSMatt Macy 	/*
2098eda14cbcSMatt Macy 	 * We use the (otherwise unused) top bit of dn_next_maxblkid[txgoff]
2099eda14cbcSMatt Macy 	 * to indicate that this field is set. This allows us to set the
2100eda14cbcSMatt Macy 	 * maxblkid to 0 on an existing object in dnode_sync().
2101eda14cbcSMatt Macy 	 */
2102eda14cbcSMatt Macy 	dn->dn_maxblkid = blkid;
2103eda14cbcSMatt Macy 	dn->dn_next_maxblkid[tx->tx_txg & TXG_MASK] =
2104eda14cbcSMatt Macy 	    blkid | DMU_NEXT_MAXBLKID_SET;
2105eda14cbcSMatt Macy 
2106eda14cbcSMatt Macy 	/*
2107eda14cbcSMatt Macy 	 * Compute the number of levels necessary to support the new maxblkid.
2108eda14cbcSMatt Macy 	 * Raw sends will ensure nlevels is set correctly for us.
2109eda14cbcSMatt Macy 	 */
2110eda14cbcSMatt Macy 	new_nlevels = 1;
2111eda14cbcSMatt Macy 	epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
2112eda14cbcSMatt Macy 	for (sz = dn->dn_nblkptr;
2113eda14cbcSMatt Macy 	    sz <= blkid && sz >= dn->dn_nblkptr; sz <<= epbs)
2114eda14cbcSMatt Macy 		new_nlevels++;
2115eda14cbcSMatt Macy 
2116eda14cbcSMatt Macy 	ASSERT3U(new_nlevels, <=, DN_MAX_LEVELS);
2117eda14cbcSMatt Macy 
2118eda14cbcSMatt Macy 	if (!force) {
2119eda14cbcSMatt Macy 		if (new_nlevels > dn->dn_nlevels)
2120eda14cbcSMatt Macy 			dnode_set_nlevels_impl(dn, new_nlevels, tx);
2121eda14cbcSMatt Macy 	} else {
2122eda14cbcSMatt Macy 		ASSERT3U(dn->dn_nlevels, >=, new_nlevels);
2123eda14cbcSMatt Macy 	}
2124eda14cbcSMatt Macy 
2125eda14cbcSMatt Macy out:
2126eda14cbcSMatt Macy 	if (have_read)
2127eda14cbcSMatt Macy 		rw_downgrade(&dn->dn_struct_rwlock);
2128eda14cbcSMatt Macy }
2129eda14cbcSMatt Macy 
2130eda14cbcSMatt Macy static void
2131eda14cbcSMatt Macy dnode_dirty_l1(dnode_t *dn, uint64_t l1blkid, dmu_tx_t *tx)
2132eda14cbcSMatt Macy {
2133eda14cbcSMatt Macy 	dmu_buf_impl_t *db = dbuf_hold_level(dn, 1, l1blkid, FTAG);
2134eda14cbcSMatt Macy 	if (db != NULL) {
2135eda14cbcSMatt Macy 		dmu_buf_will_dirty(&db->db, tx);
2136eda14cbcSMatt Macy 		dbuf_rele(db, FTAG);
2137eda14cbcSMatt Macy 	}
2138eda14cbcSMatt Macy }
2139eda14cbcSMatt Macy 
2140eda14cbcSMatt Macy /*
2141eda14cbcSMatt Macy  * Dirty all the in-core level-1 dbufs in the range specified by start_blkid
2142eda14cbcSMatt Macy  * and end_blkid.
2143eda14cbcSMatt Macy  */
2144eda14cbcSMatt Macy static void
2145eda14cbcSMatt Macy dnode_dirty_l1range(dnode_t *dn, uint64_t start_blkid, uint64_t end_blkid,
2146eda14cbcSMatt Macy     dmu_tx_t *tx)
2147eda14cbcSMatt Macy {
21482c48331dSMatt Macy 	dmu_buf_impl_t *db_search;
2149eda14cbcSMatt Macy 	dmu_buf_impl_t *db;
2150eda14cbcSMatt Macy 	avl_index_t where;
2151eda14cbcSMatt Macy 
21522c48331dSMatt Macy 	db_search = kmem_zalloc(sizeof (dmu_buf_impl_t), KM_SLEEP);
21532c48331dSMatt Macy 
2154eda14cbcSMatt Macy 	mutex_enter(&dn->dn_dbufs_mtx);
2155eda14cbcSMatt Macy 
21562c48331dSMatt Macy 	db_search->db_level = 1;
21572c48331dSMatt Macy 	db_search->db_blkid = start_blkid + 1;
21582c48331dSMatt Macy 	db_search->db_state = DB_SEARCH;
2159eda14cbcSMatt Macy 	for (;;) {
2160eda14cbcSMatt Macy 
21612c48331dSMatt Macy 		db = avl_find(&dn->dn_dbufs, db_search, &where);
2162eda14cbcSMatt Macy 		if (db == NULL)
2163eda14cbcSMatt Macy 			db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER);
2164eda14cbcSMatt Macy 
2165eda14cbcSMatt Macy 		if (db == NULL || db->db_level != 1 ||
2166eda14cbcSMatt Macy 		    db->db_blkid >= end_blkid) {
2167eda14cbcSMatt Macy 			break;
2168eda14cbcSMatt Macy 		}
2169eda14cbcSMatt Macy 
2170eda14cbcSMatt Macy 		/*
2171eda14cbcSMatt Macy 		 * Setup the next blkid we want to search for.
2172eda14cbcSMatt Macy 		 */
21732c48331dSMatt Macy 		db_search->db_blkid = db->db_blkid + 1;
2174eda14cbcSMatt Macy 		ASSERT3U(db->db_blkid, >=, start_blkid);
2175eda14cbcSMatt Macy 
2176eda14cbcSMatt Macy 		/*
2177eda14cbcSMatt Macy 		 * If the dbuf transitions to DB_EVICTING while we're trying
2178eda14cbcSMatt Macy 		 * to dirty it, then we will be unable to discover it in
2179eda14cbcSMatt Macy 		 * the dbuf hash table. This will result in a call to
2180eda14cbcSMatt Macy 		 * dbuf_create() which needs to acquire the dn_dbufs_mtx
2181eda14cbcSMatt Macy 		 * lock. To avoid a deadlock, we drop the lock before
2182eda14cbcSMatt Macy 		 * dirtying the level-1 dbuf.
2183eda14cbcSMatt Macy 		 */
2184eda14cbcSMatt Macy 		mutex_exit(&dn->dn_dbufs_mtx);
2185eda14cbcSMatt Macy 		dnode_dirty_l1(dn, db->db_blkid, tx);
2186eda14cbcSMatt Macy 		mutex_enter(&dn->dn_dbufs_mtx);
2187eda14cbcSMatt Macy 	}
2188eda14cbcSMatt Macy 
2189eda14cbcSMatt Macy #ifdef ZFS_DEBUG
2190eda14cbcSMatt Macy 	/*
2191eda14cbcSMatt Macy 	 * Walk all the in-core level-1 dbufs and verify they have been dirtied.
2192eda14cbcSMatt Macy 	 */
21932c48331dSMatt Macy 	db_search->db_level = 1;
21942c48331dSMatt Macy 	db_search->db_blkid = start_blkid + 1;
21952c48331dSMatt Macy 	db_search->db_state = DB_SEARCH;
21962c48331dSMatt Macy 	db = avl_find(&dn->dn_dbufs, db_search, &where);
2197eda14cbcSMatt Macy 	if (db == NULL)
2198eda14cbcSMatt Macy 		db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER);
2199eda14cbcSMatt Macy 	for (; db != NULL; db = AVL_NEXT(&dn->dn_dbufs, db)) {
2200eda14cbcSMatt Macy 		if (db->db_level != 1 || db->db_blkid >= end_blkid)
2201eda14cbcSMatt Macy 			break;
2202eda14cbcSMatt Macy 		if (db->db_state != DB_EVICTING)
2203eda14cbcSMatt Macy 			ASSERT(db->db_dirtycnt > 0);
2204eda14cbcSMatt Macy 	}
2205eda14cbcSMatt Macy #endif
22062c48331dSMatt Macy 	kmem_free(db_search, sizeof (dmu_buf_impl_t));
2207eda14cbcSMatt Macy 	mutex_exit(&dn->dn_dbufs_mtx);
2208eda14cbcSMatt Macy }
2209eda14cbcSMatt Macy 
2210eda14cbcSMatt Macy void
2211a0b956f5SMartin Matuska dnode_set_dirtyctx(dnode_t *dn, dmu_tx_t *tx, const void *tag)
2212eda14cbcSMatt Macy {
2213eda14cbcSMatt Macy 	/*
2214eda14cbcSMatt Macy 	 * Don't set dirtyctx to SYNC if we're just modifying this as we
2215eda14cbcSMatt Macy 	 * initialize the objset.
2216eda14cbcSMatt Macy 	 */
2217eda14cbcSMatt Macy 	if (dn->dn_dirtyctx == DN_UNDIRTIED) {
2218eda14cbcSMatt Macy 		dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset;
2219eda14cbcSMatt Macy 
2220eda14cbcSMatt Macy 		if (ds != NULL) {
2221eda14cbcSMatt Macy 			rrw_enter(&ds->ds_bp_rwlock, RW_READER, tag);
2222eda14cbcSMatt Macy 		}
2223eda14cbcSMatt Macy 		if (!BP_IS_HOLE(dn->dn_objset->os_rootbp)) {
2224eda14cbcSMatt Macy 			if (dmu_tx_is_syncing(tx))
2225eda14cbcSMatt Macy 				dn->dn_dirtyctx = DN_DIRTY_SYNC;
2226eda14cbcSMatt Macy 			else
2227eda14cbcSMatt Macy 				dn->dn_dirtyctx = DN_DIRTY_OPEN;
2228eda14cbcSMatt Macy 			dn->dn_dirtyctx_firstset = tag;
2229eda14cbcSMatt Macy 		}
2230eda14cbcSMatt Macy 		if (ds != NULL) {
2231eda14cbcSMatt Macy 			rrw_exit(&ds->ds_bp_rwlock, tag);
2232eda14cbcSMatt Macy 		}
2233eda14cbcSMatt Macy 	}
2234eda14cbcSMatt Macy }
2235eda14cbcSMatt Macy 
22366ba2210eSMartin Matuska static void
22376ba2210eSMartin Matuska dnode_partial_zero(dnode_t *dn, uint64_t off, uint64_t blkoff, uint64_t len,
22386ba2210eSMartin Matuska     dmu_tx_t *tx)
22396ba2210eSMartin Matuska {
22406ba2210eSMartin Matuska 	dmu_buf_impl_t *db;
22416ba2210eSMartin Matuska 	int res;
22426ba2210eSMartin Matuska 
22436ba2210eSMartin Matuska 	rw_enter(&dn->dn_struct_rwlock, RW_READER);
22446ba2210eSMartin Matuska 	res = dbuf_hold_impl(dn, 0, dbuf_whichblock(dn, 0, off), TRUE, FALSE,
22456ba2210eSMartin Matuska 	    FTAG, &db);
22466ba2210eSMartin Matuska 	rw_exit(&dn->dn_struct_rwlock);
22476ba2210eSMartin Matuska 	if (res == 0) {
22486ba2210eSMartin Matuska 		db_lock_type_t dblt;
22496ba2210eSMartin Matuska 		boolean_t dirty;
22506ba2210eSMartin Matuska 
22516ba2210eSMartin Matuska 		dblt = dmu_buf_lock_parent(db, RW_READER, FTAG);
22526ba2210eSMartin Matuska 		/* don't dirty if not on disk and not dirty */
22536ba2210eSMartin Matuska 		dirty = !list_is_empty(&db->db_dirty_records) ||
22546ba2210eSMartin Matuska 		    (db->db_blkptr && !BP_IS_HOLE(db->db_blkptr));
22556ba2210eSMartin Matuska 		dmu_buf_unlock_parent(db, dblt, FTAG);
22566ba2210eSMartin Matuska 		if (dirty) {
22576ba2210eSMartin Matuska 			caddr_t data;
22586ba2210eSMartin Matuska 
22596ba2210eSMartin Matuska 			dmu_buf_will_dirty(&db->db, tx);
22606ba2210eSMartin Matuska 			data = db->db.db_data;
2261da5137abSMartin Matuska 			memset(data + blkoff, 0, len);
22626ba2210eSMartin Matuska 		}
22636ba2210eSMartin Matuska 		dbuf_rele(db, FTAG);
22646ba2210eSMartin Matuska 	}
22656ba2210eSMartin Matuska }
22666ba2210eSMartin Matuska 
2267eda14cbcSMatt Macy void
2268eda14cbcSMatt Macy dnode_free_range(dnode_t *dn, uint64_t off, uint64_t len, dmu_tx_t *tx)
2269eda14cbcSMatt Macy {
2270eda14cbcSMatt Macy 	uint64_t blkoff, blkid, nblks;
2271eda14cbcSMatt Macy 	int blksz, blkshift, head, tail;
2272eda14cbcSMatt Macy 	int trunc = FALSE;
2273eda14cbcSMatt Macy 	int epbs;
2274eda14cbcSMatt Macy 
2275eda14cbcSMatt Macy 	blksz = dn->dn_datablksz;
2276eda14cbcSMatt Macy 	blkshift = dn->dn_datablkshift;
2277eda14cbcSMatt Macy 	epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT;
2278eda14cbcSMatt Macy 
2279eda14cbcSMatt Macy 	if (len == DMU_OBJECT_END) {
2280eda14cbcSMatt Macy 		len = UINT64_MAX - off;
2281eda14cbcSMatt Macy 		trunc = TRUE;
2282eda14cbcSMatt Macy 	}
2283eda14cbcSMatt Macy 
2284eda14cbcSMatt Macy 	/*
2285eda14cbcSMatt Macy 	 * First, block align the region to free:
2286eda14cbcSMatt Macy 	 */
2287eda14cbcSMatt Macy 	if (ISP2(blksz)) {
2288eda14cbcSMatt Macy 		head = P2NPHASE(off, blksz);
2289eda14cbcSMatt Macy 		blkoff = P2PHASE(off, blksz);
2290eda14cbcSMatt Macy 		if ((off >> blkshift) > dn->dn_maxblkid)
2291eda14cbcSMatt Macy 			return;
2292eda14cbcSMatt Macy 	} else {
2293eda14cbcSMatt Macy 		ASSERT(dn->dn_maxblkid == 0);
2294eda14cbcSMatt Macy 		if (off == 0 && len >= blksz) {
2295eda14cbcSMatt Macy 			/*
2296eda14cbcSMatt Macy 			 * Freeing the whole block; fast-track this request.
2297eda14cbcSMatt Macy 			 */
2298eda14cbcSMatt Macy 			blkid = 0;
2299eda14cbcSMatt Macy 			nblks = 1;
2300eda14cbcSMatt Macy 			if (dn->dn_nlevels > 1) {
2301eda14cbcSMatt Macy 				rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
2302eda14cbcSMatt Macy 				dnode_dirty_l1(dn, 0, tx);
2303eda14cbcSMatt Macy 				rw_exit(&dn->dn_struct_rwlock);
2304eda14cbcSMatt Macy 			}
2305eda14cbcSMatt Macy 			goto done;
2306eda14cbcSMatt Macy 		} else if (off >= blksz) {
2307eda14cbcSMatt Macy 			/* Freeing past end-of-data */
2308eda14cbcSMatt Macy 			return;
2309eda14cbcSMatt Macy 		} else {
2310eda14cbcSMatt Macy 			/* Freeing part of the block. */
2311eda14cbcSMatt Macy 			head = blksz - off;
2312eda14cbcSMatt Macy 			ASSERT3U(head, >, 0);
2313eda14cbcSMatt Macy 		}
2314eda14cbcSMatt Macy 		blkoff = off;
2315eda14cbcSMatt Macy 	}
2316eda14cbcSMatt Macy 	/* zero out any partial block data at the start of the range */
2317eda14cbcSMatt Macy 	if (head) {
2318eda14cbcSMatt Macy 		ASSERT3U(blkoff + head, ==, blksz);
2319eda14cbcSMatt Macy 		if (len < head)
2320eda14cbcSMatt Macy 			head = len;
23216ba2210eSMartin Matuska 		dnode_partial_zero(dn, off, blkoff, head, tx);
2322eda14cbcSMatt Macy 		off += head;
2323eda14cbcSMatt Macy 		len -= head;
2324eda14cbcSMatt Macy 	}
2325eda14cbcSMatt Macy 
2326eda14cbcSMatt Macy 	/* If the range was less than one block, we're done */
2327eda14cbcSMatt Macy 	if (len == 0)
2328eda14cbcSMatt Macy 		return;
2329eda14cbcSMatt Macy 
2330eda14cbcSMatt Macy 	/* If the remaining range is past end of file, we're done */
2331eda14cbcSMatt Macy 	if ((off >> blkshift) > dn->dn_maxblkid)
2332eda14cbcSMatt Macy 		return;
2333eda14cbcSMatt Macy 
2334eda14cbcSMatt Macy 	ASSERT(ISP2(blksz));
2335eda14cbcSMatt Macy 	if (trunc)
2336eda14cbcSMatt Macy 		tail = 0;
2337eda14cbcSMatt Macy 	else
2338eda14cbcSMatt Macy 		tail = P2PHASE(len, blksz);
2339eda14cbcSMatt Macy 
2340eda14cbcSMatt Macy 	ASSERT0(P2PHASE(off, blksz));
2341eda14cbcSMatt Macy 	/* zero out any partial block data at the end of the range */
2342eda14cbcSMatt Macy 	if (tail) {
2343eda14cbcSMatt Macy 		if (len < tail)
2344eda14cbcSMatt Macy 			tail = len;
23456ba2210eSMartin Matuska 		dnode_partial_zero(dn, off + len, 0, tail, tx);
2346eda14cbcSMatt Macy 		len -= tail;
2347eda14cbcSMatt Macy 	}
2348eda14cbcSMatt Macy 
2349eda14cbcSMatt Macy 	/* If the range did not include a full block, we are done */
2350eda14cbcSMatt Macy 	if (len == 0)
2351eda14cbcSMatt Macy 		return;
2352eda14cbcSMatt Macy 
2353eda14cbcSMatt Macy 	ASSERT(IS_P2ALIGNED(off, blksz));
2354eda14cbcSMatt Macy 	ASSERT(trunc || IS_P2ALIGNED(len, blksz));
2355eda14cbcSMatt Macy 	blkid = off >> blkshift;
2356eda14cbcSMatt Macy 	nblks = len >> blkshift;
2357eda14cbcSMatt Macy 	if (trunc)
2358eda14cbcSMatt Macy 		nblks += 1;
2359eda14cbcSMatt Macy 
2360eda14cbcSMatt Macy 	/*
2361eda14cbcSMatt Macy 	 * Dirty all the indirect blocks in this range.  Note that only
2362eda14cbcSMatt Macy 	 * the first and last indirect blocks can actually be written
2363eda14cbcSMatt Macy 	 * (if they were partially freed) -- they must be dirtied, even if
2364eda14cbcSMatt Macy 	 * they do not exist on disk yet.  The interior blocks will
2365eda14cbcSMatt Macy 	 * be freed by free_children(), so they will not actually be written.
2366eda14cbcSMatt Macy 	 * Even though these interior blocks will not be written, we
2367eda14cbcSMatt Macy 	 * dirty them for two reasons:
2368eda14cbcSMatt Macy 	 *
2369eda14cbcSMatt Macy 	 *  - It ensures that the indirect blocks remain in memory until
2370eda14cbcSMatt Macy 	 *    syncing context.  (They have already been prefetched by
2371eda14cbcSMatt Macy 	 *    dmu_tx_hold_free(), so we don't have to worry about reading
2372eda14cbcSMatt Macy 	 *    them serially here.)
2373eda14cbcSMatt Macy 	 *
2374eda14cbcSMatt Macy 	 *  - The dirty space accounting will put pressure on the txg sync
2375eda14cbcSMatt Macy 	 *    mechanism to begin syncing, and to delay transactions if there
2376eda14cbcSMatt Macy 	 *    is a large amount of freeing.  Even though these indirect
2377eda14cbcSMatt Macy 	 *    blocks will not be written, we could need to write the same
2378eda14cbcSMatt Macy 	 *    amount of space if we copy the freed BPs into deadlists.
2379eda14cbcSMatt Macy 	 */
2380eda14cbcSMatt Macy 	if (dn->dn_nlevels > 1) {
2381eda14cbcSMatt Macy 		rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
2382eda14cbcSMatt Macy 		uint64_t first, last;
2383eda14cbcSMatt Macy 
2384eda14cbcSMatt Macy 		first = blkid >> epbs;
2385eda14cbcSMatt Macy 		dnode_dirty_l1(dn, first, tx);
2386eda14cbcSMatt Macy 		if (trunc)
2387eda14cbcSMatt Macy 			last = dn->dn_maxblkid >> epbs;
2388eda14cbcSMatt Macy 		else
2389eda14cbcSMatt Macy 			last = (blkid + nblks - 1) >> epbs;
2390eda14cbcSMatt Macy 		if (last != first)
2391eda14cbcSMatt Macy 			dnode_dirty_l1(dn, last, tx);
2392eda14cbcSMatt Macy 
2393eda14cbcSMatt Macy 		dnode_dirty_l1range(dn, first, last, tx);
2394eda14cbcSMatt Macy 
2395eda14cbcSMatt Macy 		int shift = dn->dn_datablkshift + dn->dn_indblkshift -
2396eda14cbcSMatt Macy 		    SPA_BLKPTRSHIFT;
2397eda14cbcSMatt Macy 		for (uint64_t i = first + 1; i < last; i++) {
2398eda14cbcSMatt Macy 			/*
2399eda14cbcSMatt Macy 			 * Set i to the blockid of the next non-hole
2400eda14cbcSMatt Macy 			 * level-1 indirect block at or after i.  Note
2401eda14cbcSMatt Macy 			 * that dnode_next_offset() operates in terms of
2402eda14cbcSMatt Macy 			 * level-0-equivalent bytes.
2403eda14cbcSMatt Macy 			 */
2404eda14cbcSMatt Macy 			uint64_t ibyte = i << shift;
2405eda14cbcSMatt Macy 			int err = dnode_next_offset(dn, DNODE_FIND_HAVELOCK,
2406eda14cbcSMatt Macy 			    &ibyte, 2, 1, 0);
2407eda14cbcSMatt Macy 			i = ibyte >> shift;
2408eda14cbcSMatt Macy 			if (i >= last)
2409eda14cbcSMatt Macy 				break;
2410eda14cbcSMatt Macy 
2411eda14cbcSMatt Macy 			/*
2412eda14cbcSMatt Macy 			 * Normally we should not see an error, either
2413eda14cbcSMatt Macy 			 * from dnode_next_offset() or dbuf_hold_level()
2414eda14cbcSMatt Macy 			 * (except for ESRCH from dnode_next_offset).
2415eda14cbcSMatt Macy 			 * If there is an i/o error, then when we read
2416eda14cbcSMatt Macy 			 * this block in syncing context, it will use
2417eda14cbcSMatt Macy 			 * ZIO_FLAG_MUSTSUCCEED, and thus hang/panic according
2418eda14cbcSMatt Macy 			 * to the "failmode" property.  dnode_next_offset()
2419eda14cbcSMatt Macy 			 * doesn't have a flag to indicate MUSTSUCCEED.
2420eda14cbcSMatt Macy 			 */
2421eda14cbcSMatt Macy 			if (err != 0)
2422eda14cbcSMatt Macy 				break;
2423eda14cbcSMatt Macy 
2424eda14cbcSMatt Macy 			dnode_dirty_l1(dn, i, tx);
2425eda14cbcSMatt Macy 		}
2426eda14cbcSMatt Macy 		rw_exit(&dn->dn_struct_rwlock);
2427eda14cbcSMatt Macy 	}
2428eda14cbcSMatt Macy 
2429eda14cbcSMatt Macy done:
2430eda14cbcSMatt Macy 	/*
2431eda14cbcSMatt Macy 	 * Add this range to the dnode range list.
2432eda14cbcSMatt Macy 	 * We will finish up this free operation in the syncing phase.
2433eda14cbcSMatt Macy 	 */
2434eda14cbcSMatt Macy 	mutex_enter(&dn->dn_mtx);
2435eda14cbcSMatt Macy 	{
2436eda14cbcSMatt Macy 		int txgoff = tx->tx_txg & TXG_MASK;
2437eda14cbcSMatt Macy 		if (dn->dn_free_ranges[txgoff] == NULL) {
2438eda14cbcSMatt Macy 			dn->dn_free_ranges[txgoff] = range_tree_create(NULL,
2439eda14cbcSMatt Macy 			    RANGE_SEG64, NULL, 0, 0);
2440eda14cbcSMatt Macy 		}
2441eda14cbcSMatt Macy 		range_tree_clear(dn->dn_free_ranges[txgoff], blkid, nblks);
2442eda14cbcSMatt Macy 		range_tree_add(dn->dn_free_ranges[txgoff], blkid, nblks);
2443eda14cbcSMatt Macy 	}
2444eda14cbcSMatt Macy 	dprintf_dnode(dn, "blkid=%llu nblks=%llu txg=%llu\n",
244533b8c039SMartin Matuska 	    (u_longlong_t)blkid, (u_longlong_t)nblks,
244633b8c039SMartin Matuska 	    (u_longlong_t)tx->tx_txg);
2447eda14cbcSMatt Macy 	mutex_exit(&dn->dn_mtx);
2448eda14cbcSMatt Macy 
2449eda14cbcSMatt Macy 	dbuf_free_range(dn, blkid, blkid + nblks - 1, tx);
2450eda14cbcSMatt Macy 	dnode_setdirty(dn, tx);
2451eda14cbcSMatt Macy }
2452eda14cbcSMatt Macy 
2453eda14cbcSMatt Macy static boolean_t
2454eda14cbcSMatt Macy dnode_spill_freed(dnode_t *dn)
2455eda14cbcSMatt Macy {
2456eda14cbcSMatt Macy 	int i;
2457eda14cbcSMatt Macy 
2458eda14cbcSMatt Macy 	mutex_enter(&dn->dn_mtx);
2459eda14cbcSMatt Macy 	for (i = 0; i < TXG_SIZE; i++) {
2460eda14cbcSMatt Macy 		if (dn->dn_rm_spillblk[i] == DN_KILL_SPILLBLK)
2461eda14cbcSMatt Macy 			break;
2462eda14cbcSMatt Macy 	}
2463eda14cbcSMatt Macy 	mutex_exit(&dn->dn_mtx);
2464eda14cbcSMatt Macy 	return (i < TXG_SIZE);
2465eda14cbcSMatt Macy }
2466eda14cbcSMatt Macy 
2467eda14cbcSMatt Macy /* return TRUE if this blkid was freed in a recent txg, or FALSE if it wasn't */
2468eda14cbcSMatt Macy uint64_t
2469eda14cbcSMatt Macy dnode_block_freed(dnode_t *dn, uint64_t blkid)
2470eda14cbcSMatt Macy {
2471eda14cbcSMatt Macy 	int i;
2472eda14cbcSMatt Macy 
2473eda14cbcSMatt Macy 	if (blkid == DMU_BONUS_BLKID)
2474eda14cbcSMatt Macy 		return (FALSE);
2475eda14cbcSMatt Macy 
2476eda14cbcSMatt Macy 	if (dn->dn_free_txg)
2477eda14cbcSMatt Macy 		return (TRUE);
2478eda14cbcSMatt Macy 
2479eda14cbcSMatt Macy 	if (blkid == DMU_SPILL_BLKID)
2480eda14cbcSMatt Macy 		return (dnode_spill_freed(dn));
2481eda14cbcSMatt Macy 
2482eda14cbcSMatt Macy 	mutex_enter(&dn->dn_mtx);
2483eda14cbcSMatt Macy 	for (i = 0; i < TXG_SIZE; i++) {
2484eda14cbcSMatt Macy 		if (dn->dn_free_ranges[i] != NULL &&
2485eda14cbcSMatt Macy 		    range_tree_contains(dn->dn_free_ranges[i], blkid, 1))
2486eda14cbcSMatt Macy 			break;
2487eda14cbcSMatt Macy 	}
2488eda14cbcSMatt Macy 	mutex_exit(&dn->dn_mtx);
2489eda14cbcSMatt Macy 	return (i < TXG_SIZE);
2490eda14cbcSMatt Macy }
2491eda14cbcSMatt Macy 
2492eda14cbcSMatt Macy /* call from syncing context when we actually write/free space for this dnode */
2493eda14cbcSMatt Macy void
2494eda14cbcSMatt Macy dnode_diduse_space(dnode_t *dn, int64_t delta)
2495eda14cbcSMatt Macy {
2496eda14cbcSMatt Macy 	uint64_t space;
2497eda14cbcSMatt Macy 	dprintf_dnode(dn, "dn=%p dnp=%p used=%llu delta=%lld\n",
2498eda14cbcSMatt Macy 	    dn, dn->dn_phys,
2499eda14cbcSMatt Macy 	    (u_longlong_t)dn->dn_phys->dn_used,
2500eda14cbcSMatt Macy 	    (longlong_t)delta);
2501eda14cbcSMatt Macy 
2502eda14cbcSMatt Macy 	mutex_enter(&dn->dn_mtx);
2503eda14cbcSMatt Macy 	space = DN_USED_BYTES(dn->dn_phys);
2504eda14cbcSMatt Macy 	if (delta > 0) {
2505eda14cbcSMatt Macy 		ASSERT3U(space + delta, >=, space); /* no overflow */
2506eda14cbcSMatt Macy 	} else {
2507eda14cbcSMatt Macy 		ASSERT3U(space, >=, -delta); /* no underflow */
2508eda14cbcSMatt Macy 	}
2509eda14cbcSMatt Macy 	space += delta;
2510eda14cbcSMatt Macy 	if (spa_version(dn->dn_objset->os_spa) < SPA_VERSION_DNODE_BYTES) {
2511eda14cbcSMatt Macy 		ASSERT((dn->dn_phys->dn_flags & DNODE_FLAG_USED_BYTES) == 0);
2512eda14cbcSMatt Macy 		ASSERT0(P2PHASE(space, 1<<DEV_BSHIFT));
2513eda14cbcSMatt Macy 		dn->dn_phys->dn_used = space >> DEV_BSHIFT;
2514eda14cbcSMatt Macy 	} else {
2515eda14cbcSMatt Macy 		dn->dn_phys->dn_used = space;
2516eda14cbcSMatt Macy 		dn->dn_phys->dn_flags |= DNODE_FLAG_USED_BYTES;
2517eda14cbcSMatt Macy 	}
2518eda14cbcSMatt Macy 	mutex_exit(&dn->dn_mtx);
2519eda14cbcSMatt Macy }
2520eda14cbcSMatt Macy 
2521eda14cbcSMatt Macy /*
2522eda14cbcSMatt Macy  * Scans a block at the indicated "level" looking for a hole or data,
2523eda14cbcSMatt Macy  * depending on 'flags'.
2524eda14cbcSMatt Macy  *
2525eda14cbcSMatt Macy  * If level > 0, then we are scanning an indirect block looking at its
2526eda14cbcSMatt Macy  * pointers.  If level == 0, then we are looking at a block of dnodes.
2527eda14cbcSMatt Macy  *
2528eda14cbcSMatt Macy  * If we don't find what we are looking for in the block, we return ESRCH.
2529eda14cbcSMatt Macy  * Otherwise, return with *offset pointing to the beginning (if searching
2530eda14cbcSMatt Macy  * forwards) or end (if searching backwards) of the range covered by the
2531eda14cbcSMatt Macy  * block pointer we matched on (or dnode).
2532eda14cbcSMatt Macy  *
2533eda14cbcSMatt Macy  * The basic search algorithm used below by dnode_next_offset() is to
2534eda14cbcSMatt Macy  * use this function to search up the block tree (widen the search) until
2535eda14cbcSMatt Macy  * we find something (i.e., we don't return ESRCH) and then search back
2536eda14cbcSMatt Macy  * down the tree (narrow the search) until we reach our original search
2537eda14cbcSMatt Macy  * level.
2538eda14cbcSMatt Macy  */
2539eda14cbcSMatt Macy static int
2540eda14cbcSMatt Macy dnode_next_offset_level(dnode_t *dn, int flags, uint64_t *offset,
2541eda14cbcSMatt Macy     int lvl, uint64_t blkfill, uint64_t txg)
2542eda14cbcSMatt Macy {
2543eda14cbcSMatt Macy 	dmu_buf_impl_t *db = NULL;
2544eda14cbcSMatt Macy 	void *data = NULL;
2545eda14cbcSMatt Macy 	uint64_t epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT;
2546eda14cbcSMatt Macy 	uint64_t epb = 1ULL << epbs;
2547eda14cbcSMatt Macy 	uint64_t minfill, maxfill;
2548eda14cbcSMatt Macy 	boolean_t hole;
2549eda14cbcSMatt Macy 	int i, inc, error, span;
2550eda14cbcSMatt Macy 
2551eda14cbcSMatt Macy 	ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock));
2552eda14cbcSMatt Macy 
2553eda14cbcSMatt Macy 	hole = ((flags & DNODE_FIND_HOLE) != 0);
2554eda14cbcSMatt Macy 	inc = (flags & DNODE_FIND_BACKWARDS) ? -1 : 1;
2555eda14cbcSMatt Macy 	ASSERT(txg == 0 || !hole);
2556eda14cbcSMatt Macy 
2557eda14cbcSMatt Macy 	if (lvl == dn->dn_phys->dn_nlevels) {
2558eda14cbcSMatt Macy 		error = 0;
2559eda14cbcSMatt Macy 		epb = dn->dn_phys->dn_nblkptr;
2560eda14cbcSMatt Macy 		data = dn->dn_phys->dn_blkptr;
2561eda14cbcSMatt Macy 	} else {
2562eda14cbcSMatt Macy 		uint64_t blkid = dbuf_whichblock(dn, lvl, *offset);
2563eda14cbcSMatt Macy 		error = dbuf_hold_impl(dn, lvl, blkid, TRUE, FALSE, FTAG, &db);
2564eda14cbcSMatt Macy 		if (error) {
2565eda14cbcSMatt Macy 			if (error != ENOENT)
2566eda14cbcSMatt Macy 				return (error);
2567eda14cbcSMatt Macy 			if (hole)
2568eda14cbcSMatt Macy 				return (0);
2569eda14cbcSMatt Macy 			/*
2570eda14cbcSMatt Macy 			 * This can only happen when we are searching up
2571eda14cbcSMatt Macy 			 * the block tree for data.  We don't really need to
2572eda14cbcSMatt Macy 			 * adjust the offset, as we will just end up looking
2573eda14cbcSMatt Macy 			 * at the pointer to this block in its parent, and its
2574eda14cbcSMatt Macy 			 * going to be unallocated, so we will skip over it.
2575eda14cbcSMatt Macy 			 */
2576eda14cbcSMatt Macy 			return (SET_ERROR(ESRCH));
2577eda14cbcSMatt Macy 		}
2578eda14cbcSMatt Macy 		error = dbuf_read(db, NULL,
2579c40487d4SMatt Macy 		    DB_RF_CANFAIL | DB_RF_HAVESTRUCT |
2580c40487d4SMatt Macy 		    DB_RF_NO_DECRYPT | DB_RF_NOPREFETCH);
2581eda14cbcSMatt Macy 		if (error) {
2582eda14cbcSMatt Macy 			dbuf_rele(db, FTAG);
2583eda14cbcSMatt Macy 			return (error);
2584eda14cbcSMatt Macy 		}
2585eda14cbcSMatt Macy 		data = db->db.db_data;
2586eda14cbcSMatt Macy 		rw_enter(&db->db_rwlock, RW_READER);
2587eda14cbcSMatt Macy 	}
2588eda14cbcSMatt Macy 
2589eda14cbcSMatt Macy 	if (db != NULL && txg != 0 && (db->db_blkptr == NULL ||
2590783d3ff6SMartin Matuska 	    BP_GET_LOGICAL_BIRTH(db->db_blkptr) <= txg ||
2591eda14cbcSMatt Macy 	    BP_IS_HOLE(db->db_blkptr))) {
2592eda14cbcSMatt Macy 		/*
2593eda14cbcSMatt Macy 		 * This can only happen when we are searching up the tree
2594eda14cbcSMatt Macy 		 * and these conditions mean that we need to keep climbing.
2595eda14cbcSMatt Macy 		 */
2596eda14cbcSMatt Macy 		error = SET_ERROR(ESRCH);
2597eda14cbcSMatt Macy 	} else if (lvl == 0) {
2598eda14cbcSMatt Macy 		dnode_phys_t *dnp = data;
2599eda14cbcSMatt Macy 
2600eda14cbcSMatt Macy 		ASSERT(dn->dn_type == DMU_OT_DNODE);
2601eda14cbcSMatt Macy 		ASSERT(!(flags & DNODE_FIND_BACKWARDS));
2602eda14cbcSMatt Macy 
2603eda14cbcSMatt Macy 		for (i = (*offset >> DNODE_SHIFT) & (blkfill - 1);
2604eda14cbcSMatt Macy 		    i < blkfill; i += dnp[i].dn_extra_slots + 1) {
2605eda14cbcSMatt Macy 			if ((dnp[i].dn_type == DMU_OT_NONE) == hole)
2606eda14cbcSMatt Macy 				break;
2607eda14cbcSMatt Macy 		}
2608eda14cbcSMatt Macy 
2609eda14cbcSMatt Macy 		if (i == blkfill)
2610eda14cbcSMatt Macy 			error = SET_ERROR(ESRCH);
2611eda14cbcSMatt Macy 
2612eda14cbcSMatt Macy 		*offset = (*offset & ~(DNODE_BLOCK_SIZE - 1)) +
2613eda14cbcSMatt Macy 		    (i << DNODE_SHIFT);
2614eda14cbcSMatt Macy 	} else {
2615eda14cbcSMatt Macy 		blkptr_t *bp = data;
2616eda14cbcSMatt Macy 		uint64_t start = *offset;
2617eda14cbcSMatt Macy 		span = (lvl - 1) * epbs + dn->dn_datablkshift;
2618eda14cbcSMatt Macy 		minfill = 0;
2619eda14cbcSMatt Macy 		maxfill = blkfill << ((lvl - 1) * epbs);
2620eda14cbcSMatt Macy 
2621eda14cbcSMatt Macy 		if (hole)
2622eda14cbcSMatt Macy 			maxfill--;
2623eda14cbcSMatt Macy 		else
2624eda14cbcSMatt Macy 			minfill++;
2625eda14cbcSMatt Macy 
2626eda14cbcSMatt Macy 		if (span >= 8 * sizeof (*offset)) {
2627eda14cbcSMatt Macy 			/* This only happens on the highest indirection level */
2628eda14cbcSMatt Macy 			ASSERT3U((lvl - 1), ==, dn->dn_phys->dn_nlevels - 1);
2629eda14cbcSMatt Macy 			*offset = 0;
2630eda14cbcSMatt Macy 		} else {
2631eda14cbcSMatt Macy 			*offset = *offset >> span;
2632eda14cbcSMatt Macy 		}
2633eda14cbcSMatt Macy 
2634eda14cbcSMatt Macy 		for (i = BF64_GET(*offset, 0, epbs);
2635eda14cbcSMatt Macy 		    i >= 0 && i < epb; i += inc) {
2636eda14cbcSMatt Macy 			if (BP_GET_FILL(&bp[i]) >= minfill &&
2637eda14cbcSMatt Macy 			    BP_GET_FILL(&bp[i]) <= maxfill &&
2638783d3ff6SMartin Matuska 			    (hole || BP_GET_LOGICAL_BIRTH(&bp[i]) > txg))
2639eda14cbcSMatt Macy 				break;
2640eda14cbcSMatt Macy 			if (inc > 0 || *offset > 0)
2641eda14cbcSMatt Macy 				*offset += inc;
2642eda14cbcSMatt Macy 		}
2643eda14cbcSMatt Macy 
2644eda14cbcSMatt Macy 		if (span >= 8 * sizeof (*offset)) {
2645eda14cbcSMatt Macy 			*offset = start;
2646eda14cbcSMatt Macy 		} else {
2647eda14cbcSMatt Macy 			*offset = *offset << span;
2648eda14cbcSMatt Macy 		}
2649eda14cbcSMatt Macy 
2650eda14cbcSMatt Macy 		if (inc < 0) {
2651eda14cbcSMatt Macy 			/* traversing backwards; position offset at the end */
26522a58b312SMartin Matuska 			if (span < 8 * sizeof (*offset))
26532a58b312SMartin Matuska 				*offset = MIN(*offset + (1ULL << span) - 1,
26542a58b312SMartin Matuska 				    start);
2655eda14cbcSMatt Macy 		} else if (*offset < start) {
2656eda14cbcSMatt Macy 			*offset = start;
2657eda14cbcSMatt Macy 		}
2658eda14cbcSMatt Macy 		if (i < 0 || i >= epb)
2659eda14cbcSMatt Macy 			error = SET_ERROR(ESRCH);
2660eda14cbcSMatt Macy 	}
2661eda14cbcSMatt Macy 
2662eda14cbcSMatt Macy 	if (db != NULL) {
2663eda14cbcSMatt Macy 		rw_exit(&db->db_rwlock);
2664eda14cbcSMatt Macy 		dbuf_rele(db, FTAG);
2665eda14cbcSMatt Macy 	}
2666eda14cbcSMatt Macy 
2667eda14cbcSMatt Macy 	return (error);
2668eda14cbcSMatt Macy }
2669eda14cbcSMatt Macy 
2670eda14cbcSMatt Macy /*
2671eda14cbcSMatt Macy  * Find the next hole, data, or sparse region at or after *offset.
2672eda14cbcSMatt Macy  * The value 'blkfill' tells us how many items we expect to find
2673eda14cbcSMatt Macy  * in an L0 data block; this value is 1 for normal objects,
2674eda14cbcSMatt Macy  * DNODES_PER_BLOCK for the meta dnode, and some fraction of
2675eda14cbcSMatt Macy  * DNODES_PER_BLOCK when searching for sparse regions thereof.
2676eda14cbcSMatt Macy  *
2677eda14cbcSMatt Macy  * Examples:
2678eda14cbcSMatt Macy  *
2679eda14cbcSMatt Macy  * dnode_next_offset(dn, flags, offset, 1, 1, 0);
2680eda14cbcSMatt Macy  *	Finds the next/previous hole/data in a file.
2681eda14cbcSMatt Macy  *	Used in dmu_offset_next().
2682eda14cbcSMatt Macy  *
2683eda14cbcSMatt Macy  * dnode_next_offset(mdn, flags, offset, 0, DNODES_PER_BLOCK, txg);
2684eda14cbcSMatt Macy  *	Finds the next free/allocated dnode an objset's meta-dnode.
2685eda14cbcSMatt Macy  *	Only finds objects that have new contents since txg (ie.
2686eda14cbcSMatt Macy  *	bonus buffer changes and content removal are ignored).
2687eda14cbcSMatt Macy  *	Used in dmu_object_next().
2688eda14cbcSMatt Macy  *
2689eda14cbcSMatt Macy  * dnode_next_offset(mdn, DNODE_FIND_HOLE, offset, 2, DNODES_PER_BLOCK >> 2, 0);
2690eda14cbcSMatt Macy  *	Finds the next L2 meta-dnode bp that's at most 1/4 full.
2691eda14cbcSMatt Macy  *	Used in dmu_object_alloc().
2692eda14cbcSMatt Macy  */
2693eda14cbcSMatt Macy int
2694eda14cbcSMatt Macy dnode_next_offset(dnode_t *dn, int flags, uint64_t *offset,
2695eda14cbcSMatt Macy     int minlvl, uint64_t blkfill, uint64_t txg)
2696eda14cbcSMatt Macy {
2697eda14cbcSMatt Macy 	uint64_t initial_offset = *offset;
2698eda14cbcSMatt Macy 	int lvl, maxlvl;
2699eda14cbcSMatt Macy 	int error = 0;
2700eda14cbcSMatt Macy 
2701eda14cbcSMatt Macy 	if (!(flags & DNODE_FIND_HAVELOCK))
2702eda14cbcSMatt Macy 		rw_enter(&dn->dn_struct_rwlock, RW_READER);
2703eda14cbcSMatt Macy 
2704eda14cbcSMatt Macy 	if (dn->dn_phys->dn_nlevels == 0) {
2705eda14cbcSMatt Macy 		error = SET_ERROR(ESRCH);
2706eda14cbcSMatt Macy 		goto out;
2707eda14cbcSMatt Macy 	}
2708eda14cbcSMatt Macy 
2709eda14cbcSMatt Macy 	if (dn->dn_datablkshift == 0) {
2710eda14cbcSMatt Macy 		if (*offset < dn->dn_datablksz) {
2711eda14cbcSMatt Macy 			if (flags & DNODE_FIND_HOLE)
2712eda14cbcSMatt Macy 				*offset = dn->dn_datablksz;
2713eda14cbcSMatt Macy 		} else {
2714eda14cbcSMatt Macy 			error = SET_ERROR(ESRCH);
2715eda14cbcSMatt Macy 		}
2716eda14cbcSMatt Macy 		goto out;
2717eda14cbcSMatt Macy 	}
2718eda14cbcSMatt Macy 
2719eda14cbcSMatt Macy 	maxlvl = dn->dn_phys->dn_nlevels;
2720eda14cbcSMatt Macy 
2721eda14cbcSMatt Macy 	for (lvl = minlvl; lvl <= maxlvl; lvl++) {
2722eda14cbcSMatt Macy 		error = dnode_next_offset_level(dn,
2723eda14cbcSMatt Macy 		    flags, offset, lvl, blkfill, txg);
2724eda14cbcSMatt Macy 		if (error != ESRCH)
2725eda14cbcSMatt Macy 			break;
2726eda14cbcSMatt Macy 	}
2727eda14cbcSMatt Macy 
2728eda14cbcSMatt Macy 	while (error == 0 && --lvl >= minlvl) {
2729eda14cbcSMatt Macy 		error = dnode_next_offset_level(dn,
2730eda14cbcSMatt Macy 		    flags, offset, lvl, blkfill, txg);
2731eda14cbcSMatt Macy 	}
2732eda14cbcSMatt Macy 
2733eda14cbcSMatt Macy 	/*
2734eda14cbcSMatt Macy 	 * There's always a "virtual hole" at the end of the object, even
2735eda14cbcSMatt Macy 	 * if all BP's which physically exist are non-holes.
2736eda14cbcSMatt Macy 	 */
2737eda14cbcSMatt Macy 	if ((flags & DNODE_FIND_HOLE) && error == ESRCH && txg == 0 &&
2738eda14cbcSMatt Macy 	    minlvl == 1 && blkfill == 1 && !(flags & DNODE_FIND_BACKWARDS)) {
2739eda14cbcSMatt Macy 		error = 0;
2740eda14cbcSMatt Macy 	}
2741eda14cbcSMatt Macy 
2742eda14cbcSMatt Macy 	if (error == 0 && (flags & DNODE_FIND_BACKWARDS ?
2743eda14cbcSMatt Macy 	    initial_offset < *offset : initial_offset > *offset))
2744eda14cbcSMatt Macy 		error = SET_ERROR(ESRCH);
2745eda14cbcSMatt Macy out:
2746eda14cbcSMatt Macy 	if (!(flags & DNODE_FIND_HAVELOCK))
2747eda14cbcSMatt Macy 		rw_exit(&dn->dn_struct_rwlock);
2748eda14cbcSMatt Macy 
2749eda14cbcSMatt Macy 	return (error);
2750eda14cbcSMatt Macy }
2751eda14cbcSMatt Macy 
2752eda14cbcSMatt Macy #if defined(_KERNEL)
2753eda14cbcSMatt Macy EXPORT_SYMBOL(dnode_hold);
2754eda14cbcSMatt Macy EXPORT_SYMBOL(dnode_rele);
2755eda14cbcSMatt Macy EXPORT_SYMBOL(dnode_set_nlevels);
2756eda14cbcSMatt Macy EXPORT_SYMBOL(dnode_set_blksz);
2757eda14cbcSMatt Macy EXPORT_SYMBOL(dnode_free_range);
2758eda14cbcSMatt Macy EXPORT_SYMBOL(dnode_evict_dbufs);
2759eda14cbcSMatt Macy EXPORT_SYMBOL(dnode_evict_bonus);
2760eda14cbcSMatt Macy #endif
276115f0b8c3SMartin Matuska 
276215f0b8c3SMartin Matuska ZFS_MODULE_PARAM(zfs, zfs_, default_bs, INT, ZMOD_RW,
276315f0b8c3SMartin Matuska 	"Default dnode block shift");
276415f0b8c3SMartin Matuska ZFS_MODULE_PARAM(zfs, zfs_, default_ibs, INT, ZMOD_RW,
276515f0b8c3SMartin Matuska 	"Default dnode indirect block shift");
2766