Lines Matching refs:dn
66 dnode_t *dn = arg; in dnode_cons() local
69 rw_init(&dn->dn_struct_rwlock, NULL, RW_DEFAULT, NULL); in dnode_cons()
70 mutex_init(&dn->dn_mtx, NULL, MUTEX_DEFAULT, NULL); in dnode_cons()
71 mutex_init(&dn->dn_dbufs_mtx, NULL, MUTEX_DEFAULT, NULL); in dnode_cons()
72 cv_init(&dn->dn_notxholds, NULL, CV_DEFAULT, NULL); in dnode_cons()
74 refcount_create(&dn->dn_holds); in dnode_cons()
75 refcount_create(&dn->dn_tx_holds); in dnode_cons()
76 list_link_init(&dn->dn_link); in dnode_cons()
78 bzero(&dn->dn_next_nblkptr[0], sizeof (dn->dn_next_nblkptr)); in dnode_cons()
79 bzero(&dn->dn_next_nlevels[0], sizeof (dn->dn_next_nlevels)); in dnode_cons()
80 bzero(&dn->dn_next_indblkshift[0], sizeof (dn->dn_next_indblkshift)); in dnode_cons()
81 bzero(&dn->dn_next_bonustype[0], sizeof (dn->dn_next_bonustype)); in dnode_cons()
82 bzero(&dn->dn_rm_spillblk[0], sizeof (dn->dn_rm_spillblk)); in dnode_cons()
83 bzero(&dn->dn_next_bonuslen[0], sizeof (dn->dn_next_bonuslen)); in dnode_cons()
84 bzero(&dn->dn_next_blksz[0], sizeof (dn->dn_next_blksz)); in dnode_cons()
87 list_link_init(&dn->dn_dirty_link[i]); in dnode_cons()
88 avl_create(&dn->dn_ranges[i], free_range_compar, in dnode_cons()
91 list_create(&dn->dn_dirty_records[i], in dnode_cons()
96 dn->dn_allocated_txg = 0; in dnode_cons()
97 dn->dn_free_txg = 0; in dnode_cons()
98 dn->dn_assigned_txg = 0; in dnode_cons()
99 dn->dn_dirtyctx = 0; in dnode_cons()
100 dn->dn_dirtyctx_firstset = NULL; in dnode_cons()
101 dn->dn_bonus = NULL; in dnode_cons()
102 dn->dn_have_spill = B_FALSE; in dnode_cons()
103 dn->dn_zio = NULL; in dnode_cons()
104 dn->dn_oldused = 0; in dnode_cons()
105 dn->dn_oldflags = 0; in dnode_cons()
106 dn->dn_olduid = 0; in dnode_cons()
107 dn->dn_oldgid = 0; in dnode_cons()
108 dn->dn_newuid = 0; in dnode_cons()
109 dn->dn_newgid = 0; in dnode_cons()
110 dn->dn_id_flags = 0; in dnode_cons()
112 dn->dn_dbufs_count = 0; in dnode_cons()
113 list_create(&dn->dn_dbufs, sizeof (dmu_buf_impl_t), in dnode_cons()
116 dn->dn_moved = 0; in dnode_cons()
125 dnode_t *dn = arg; in dnode_dest() local
127 rw_destroy(&dn->dn_struct_rwlock); in dnode_dest()
128 mutex_destroy(&dn->dn_mtx); in dnode_dest()
129 mutex_destroy(&dn->dn_dbufs_mtx); in dnode_dest()
130 cv_destroy(&dn->dn_notxholds); in dnode_dest()
131 refcount_destroy(&dn->dn_holds); in dnode_dest()
132 refcount_destroy(&dn->dn_tx_holds); in dnode_dest()
133 ASSERT(!list_link_active(&dn->dn_link)); in dnode_dest()
136 ASSERT(!list_link_active(&dn->dn_dirty_link[i])); in dnode_dest()
137 avl_destroy(&dn->dn_ranges[i]); in dnode_dest()
138 list_destroy(&dn->dn_dirty_records[i]); in dnode_dest()
139 ASSERT3U(dn->dn_next_nblkptr[i], ==, 0); in dnode_dest()
140 ASSERT3U(dn->dn_next_nlevels[i], ==, 0); in dnode_dest()
141 ASSERT3U(dn->dn_next_indblkshift[i], ==, 0); in dnode_dest()
142 ASSERT3U(dn->dn_next_bonustype[i], ==, 0); in dnode_dest()
143 ASSERT3U(dn->dn_rm_spillblk[i], ==, 0); in dnode_dest()
144 ASSERT3U(dn->dn_next_bonuslen[i], ==, 0); in dnode_dest()
145 ASSERT3U(dn->dn_next_blksz[i], ==, 0); in dnode_dest()
148 ASSERT3U(dn->dn_allocated_txg, ==, 0); in dnode_dest()
149 ASSERT3U(dn->dn_free_txg, ==, 0); in dnode_dest()
150 ASSERT3U(dn->dn_assigned_txg, ==, 0); in dnode_dest()
151 ASSERT3U(dn->dn_dirtyctx, ==, 0); in dnode_dest()
152 ASSERT3P(dn->dn_dirtyctx_firstset, ==, NULL); in dnode_dest()
153 ASSERT3P(dn->dn_bonus, ==, NULL); in dnode_dest()
154 ASSERT(!dn->dn_have_spill); in dnode_dest()
155 ASSERT3P(dn->dn_zio, ==, NULL); in dnode_dest()
156 ASSERT3U(dn->dn_oldused, ==, 0); in dnode_dest()
157 ASSERT3U(dn->dn_oldflags, ==, 0); in dnode_dest()
158 ASSERT3U(dn->dn_olduid, ==, 0); in dnode_dest()
159 ASSERT3U(dn->dn_oldgid, ==, 0); in dnode_dest()
160 ASSERT3U(dn->dn_newuid, ==, 0); in dnode_dest()
161 ASSERT3U(dn->dn_newgid, ==, 0); in dnode_dest()
162 ASSERT3U(dn->dn_id_flags, ==, 0); in dnode_dest()
164 ASSERT3U(dn->dn_dbufs_count, ==, 0); in dnode_dest()
165 list_destroy(&dn->dn_dbufs); in dnode_dest()
188 dnode_verify(dnode_t *dn) in dnode_verify() argument
192 ASSERT(dn->dn_phys); in dnode_verify()
193 ASSERT(dn->dn_objset); in dnode_verify()
194 ASSERT(dn->dn_handle->dnh_dnode == dn); in dnode_verify()
196 ASSERT(dn->dn_phys->dn_type < DMU_OT_NUMTYPES); in dnode_verify()
201 if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { in dnode_verify()
202 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dnode_verify()
205 if (dn->dn_phys->dn_type != DMU_OT_NONE || dn->dn_allocated_txg != 0) { in dnode_verify()
207 ASSERT3U(dn->dn_indblkshift, >=, 0); in dnode_verify()
208 ASSERT3U(dn->dn_indblkshift, <=, SPA_MAXBLOCKSHIFT); in dnode_verify()
209 if (dn->dn_datablkshift) { in dnode_verify()
210 ASSERT3U(dn->dn_datablkshift, >=, SPA_MINBLOCKSHIFT); in dnode_verify()
211 ASSERT3U(dn->dn_datablkshift, <=, SPA_MAXBLOCKSHIFT); in dnode_verify()
212 ASSERT3U(1<<dn->dn_datablkshift, ==, dn->dn_datablksz); in dnode_verify()
214 ASSERT3U(dn->dn_nlevels, <=, 30); in dnode_verify()
215 ASSERT3U(dn->dn_type, <=, DMU_OT_NUMTYPES); in dnode_verify()
216 ASSERT3U(dn->dn_nblkptr, >=, 1); in dnode_verify()
217 ASSERT3U(dn->dn_nblkptr, <=, DN_MAX_NBLKPTR); in dnode_verify()
218 ASSERT3U(dn->dn_bonuslen, <=, DN_MAX_BONUSLEN); in dnode_verify()
219 ASSERT3U(dn->dn_datablksz, ==, in dnode_verify()
220 dn->dn_datablkszsec << SPA_MINBLOCKSHIFT); in dnode_verify()
221 ASSERT3U(ISP2(dn->dn_datablksz), ==, dn->dn_datablkshift != 0); in dnode_verify()
222 ASSERT3U((dn->dn_nblkptr - 1) * sizeof (blkptr_t) + in dnode_verify()
223 dn->dn_bonuslen, <=, DN_MAX_BONUSLEN); in dnode_verify()
225 ASSERT3U(dn->dn_next_nlevels[i], <=, dn->dn_nlevels); in dnode_verify()
228 if (dn->dn_phys->dn_type != DMU_OT_NONE) in dnode_verify()
229 ASSERT3U(dn->dn_phys->dn_nlevels, <=, dn->dn_nlevels); in dnode_verify()
230 ASSERT(DMU_OBJECT_IS_SPECIAL(dn->dn_object) || dn->dn_dbuf != NULL); in dnode_verify()
231 if (dn->dn_dbuf != NULL) { in dnode_verify()
232 ASSERT3P(dn->dn_phys, ==, in dnode_verify()
233 (dnode_phys_t *)dn->dn_dbuf->db.db_data + in dnode_verify()
234 (dn->dn_object % (dn->dn_dbuf->db.db_size >> DNODE_SHIFT))); in dnode_verify()
237 rw_exit(&dn->dn_struct_rwlock); in dnode_verify()
321 dnode_setbonuslen(dnode_t *dn, int newsize, dmu_tx_t *tx) in dnode_setbonuslen() argument
323 ASSERT3U(refcount_count(&dn->dn_holds), >=, 1); in dnode_setbonuslen()
325 dnode_setdirty(dn, tx); in dnode_setbonuslen()
326 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dnode_setbonuslen()
328 (dn->dn_nblkptr-1) * sizeof (blkptr_t)); in dnode_setbonuslen()
329 dn->dn_bonuslen = newsize; in dnode_setbonuslen()
331 dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = DN_ZERO_BONUSLEN; in dnode_setbonuslen()
333 dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = dn->dn_bonuslen; in dnode_setbonuslen()
334 rw_exit(&dn->dn_struct_rwlock); in dnode_setbonuslen()
338 dnode_setbonus_type(dnode_t *dn, dmu_object_type_t newtype, dmu_tx_t *tx) in dnode_setbonus_type() argument
340 ASSERT3U(refcount_count(&dn->dn_holds), >=, 1); in dnode_setbonus_type()
341 dnode_setdirty(dn, tx); in dnode_setbonus_type()
342 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dnode_setbonus_type()
343 dn->dn_bonustype = newtype; in dnode_setbonus_type()
344 dn->dn_next_bonustype[tx->tx_txg & TXG_MASK] = dn->dn_bonustype; in dnode_setbonus_type()
345 rw_exit(&dn->dn_struct_rwlock); in dnode_setbonus_type()
349 dnode_rm_spill(dnode_t *dn, dmu_tx_t *tx) in dnode_rm_spill() argument
351 ASSERT3U(refcount_count(&dn->dn_holds), >=, 1); in dnode_rm_spill()
352 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); in dnode_rm_spill()
353 dnode_setdirty(dn, tx); in dnode_rm_spill()
354 dn->dn_rm_spillblk[tx->tx_txg&TXG_MASK] = DN_KILL_SPILLBLK; in dnode_rm_spill()
355 dn->dn_have_spill = B_FALSE; in dnode_rm_spill()
359 dnode_setdblksz(dnode_t *dn, int size) in dnode_setdblksz() argument
365 1<<(sizeof (dn->dn_phys->dn_datablkszsec) * 8)); in dnode_setdblksz()
366 dn->dn_datablksz = size; in dnode_setdblksz()
367 dn->dn_datablkszsec = size >> SPA_MINBLOCKSHIFT; in dnode_setdblksz()
368 dn->dn_datablkshift = ISP2(size) ? highbit(size - 1) : 0; in dnode_setdblksz()
375 dnode_t *dn = kmem_cache_alloc(dnode_cache, KM_SLEEP); in dnode_create() local
377 ASSERT(!POINTER_IS_VALID(dn->dn_objset)); in dnode_create()
378 dn->dn_moved = 0; in dnode_create()
384 dn->dn_object = object; in dnode_create()
385 dn->dn_dbuf = db; in dnode_create()
386 dn->dn_handle = dnh; in dnode_create()
387 dn->dn_phys = dnp; in dnode_create()
390 dnode_setdblksz(dn, dnp->dn_datablkszsec << SPA_MINBLOCKSHIFT); in dnode_create()
392 dn->dn_datablksz = 0; in dnode_create()
393 dn->dn_datablkszsec = 0; in dnode_create()
394 dn->dn_datablkshift = 0; in dnode_create()
396 dn->dn_indblkshift = dnp->dn_indblkshift; in dnode_create()
397 dn->dn_nlevels = dnp->dn_nlevels; in dnode_create()
398 dn->dn_type = dnp->dn_type; in dnode_create()
399 dn->dn_nblkptr = dnp->dn_nblkptr; in dnode_create()
400 dn->dn_checksum = dnp->dn_checksum; in dnode_create()
401 dn->dn_compress = dnp->dn_compress; in dnode_create()
402 dn->dn_bonustype = dnp->dn_bonustype; in dnode_create()
403 dn->dn_bonuslen = dnp->dn_bonuslen; in dnode_create()
404 dn->dn_maxblkid = dnp->dn_maxblkid; in dnode_create()
405 dn->dn_have_spill = ((dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) != 0); in dnode_create()
406 dn->dn_id_flags = 0; in dnode_create()
408 dmu_zfetch_init(&dn->dn_zfetch, dn); in dnode_create()
410 ASSERT(dn->dn_phys->dn_type < DMU_OT_NUMTYPES); in dnode_create()
413 list_insert_head(&os->os_dnodes, dn); in dnode_create()
419 dn->dn_objset = os; in dnode_create()
423 return (dn); in dnode_create()
430 dnode_destroy(dnode_t *dn) in dnode_destroy() argument
432 objset_t *os = dn->dn_objset; in dnode_destroy()
434 ASSERT((dn->dn_id_flags & DN_ID_NEW_EXIST) == 0); in dnode_destroy()
437 POINTER_INVALIDATE(&dn->dn_objset); in dnode_destroy()
438 list_remove(&os->os_dnodes, dn); in dnode_destroy()
442 zrl_remove(&dn->dn_handle->dnh_zrlock); in dnode_destroy()
444 dn->dn_allocated_txg = 0; in dnode_destroy()
445 dn->dn_free_txg = 0; in dnode_destroy()
446 dn->dn_assigned_txg = 0; in dnode_destroy()
448 dn->dn_dirtyctx = 0; in dnode_destroy()
449 if (dn->dn_dirtyctx_firstset != NULL) { in dnode_destroy()
450 kmem_free(dn->dn_dirtyctx_firstset, 1); in dnode_destroy()
451 dn->dn_dirtyctx_firstset = NULL; in dnode_destroy()
453 if (dn->dn_bonus != NULL) { in dnode_destroy()
454 mutex_enter(&dn->dn_bonus->db_mtx); in dnode_destroy()
455 dbuf_evict(dn->dn_bonus); in dnode_destroy()
456 dn->dn_bonus = NULL; in dnode_destroy()
458 dn->dn_zio = NULL; in dnode_destroy()
460 dn->dn_have_spill = B_FALSE; in dnode_destroy()
461 dn->dn_oldused = 0; in dnode_destroy()
462 dn->dn_oldflags = 0; in dnode_destroy()
463 dn->dn_olduid = 0; in dnode_destroy()
464 dn->dn_oldgid = 0; in dnode_destroy()
465 dn->dn_newuid = 0; in dnode_destroy()
466 dn->dn_newgid = 0; in dnode_destroy()
467 dn->dn_id_flags = 0; in dnode_destroy()
469 dmu_zfetch_rele(&dn->dn_zfetch); in dnode_destroy()
470 kmem_cache_free(dnode_cache, dn); in dnode_destroy()
475 dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs, in dnode_allocate() argument
492 dprintf("os=%p obj=%llu txg=%llu blocksize=%d ibs=%d\n", dn->dn_objset, in dnode_allocate()
493 dn->dn_object, tx->tx_txg, blocksize, ibs); in dnode_allocate()
495 ASSERT(dn->dn_type == DMU_OT_NONE); in dnode_allocate()
496 ASSERT(bcmp(dn->dn_phys, &dnode_phys_zero, sizeof (dnode_phys_t)) == 0); in dnode_allocate()
497 ASSERT(dn->dn_phys->dn_type == DMU_OT_NONE); in dnode_allocate()
505 ASSERT(dn->dn_type == DMU_OT_NONE); in dnode_allocate()
506 ASSERT3U(dn->dn_maxblkid, ==, 0); in dnode_allocate()
507 ASSERT3U(dn->dn_allocated_txg, ==, 0); in dnode_allocate()
508 ASSERT3U(dn->dn_assigned_txg, ==, 0); in dnode_allocate()
509 ASSERT(refcount_is_zero(&dn->dn_tx_holds)); in dnode_allocate()
510 ASSERT3U(refcount_count(&dn->dn_holds), <=, 1); in dnode_allocate()
511 ASSERT3P(list_head(&dn->dn_dbufs), ==, NULL); in dnode_allocate()
514 ASSERT3U(dn->dn_next_nblkptr[i], ==, 0); in dnode_allocate()
515 ASSERT3U(dn->dn_next_nlevels[i], ==, 0); in dnode_allocate()
516 ASSERT3U(dn->dn_next_indblkshift[i], ==, 0); in dnode_allocate()
517 ASSERT3U(dn->dn_next_bonuslen[i], ==, 0); in dnode_allocate()
518 ASSERT3U(dn->dn_next_bonustype[i], ==, 0); in dnode_allocate()
519 ASSERT3U(dn->dn_rm_spillblk[i], ==, 0); in dnode_allocate()
520 ASSERT3U(dn->dn_next_blksz[i], ==, 0); in dnode_allocate()
521 ASSERT(!list_link_active(&dn->dn_dirty_link[i])); in dnode_allocate()
522 ASSERT3P(list_head(&dn->dn_dirty_records[i]), ==, NULL); in dnode_allocate()
523 ASSERT3U(avl_numnodes(&dn->dn_ranges[i]), ==, 0); in dnode_allocate()
526 dn->dn_type = ot; in dnode_allocate()
527 dnode_setdblksz(dn, blocksize); in dnode_allocate()
528 dn->dn_indblkshift = ibs; in dnode_allocate()
529 dn->dn_nlevels = 1; in dnode_allocate()
531 dn->dn_nblkptr = 1; in dnode_allocate()
533 dn->dn_nblkptr = 1 + in dnode_allocate()
535 dn->dn_bonustype = bonustype; in dnode_allocate()
536 dn->dn_bonuslen = bonuslen; in dnode_allocate()
537 dn->dn_checksum = ZIO_CHECKSUM_INHERIT; in dnode_allocate()
538 dn->dn_compress = ZIO_COMPRESS_INHERIT; in dnode_allocate()
539 dn->dn_dirtyctx = 0; in dnode_allocate()
541 dn->dn_free_txg = 0; in dnode_allocate()
542 if (dn->dn_dirtyctx_firstset) { in dnode_allocate()
543 kmem_free(dn->dn_dirtyctx_firstset, 1); in dnode_allocate()
544 dn->dn_dirtyctx_firstset = NULL; in dnode_allocate()
547 dn->dn_allocated_txg = tx->tx_txg; in dnode_allocate()
548 dn->dn_id_flags = 0; in dnode_allocate()
550 dnode_setdirty(dn, tx); in dnode_allocate()
551 dn->dn_next_indblkshift[tx->tx_txg & TXG_MASK] = ibs; in dnode_allocate()
552 dn->dn_next_bonuslen[tx->tx_txg & TXG_MASK] = dn->dn_bonuslen; in dnode_allocate()
553 dn->dn_next_bonustype[tx->tx_txg & TXG_MASK] = dn->dn_bonustype; in dnode_allocate()
554 dn->dn_next_blksz[tx->tx_txg & TXG_MASK] = dn->dn_datablksz; in dnode_allocate()
558 dnode_reallocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, in dnode_reallocate() argument
566 ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT || dmu_tx_private_ok(tx)); in dnode_reallocate()
575 dnode_evict_dbufs(dn); in dnode_reallocate()
577 dn->dn_id_flags = 0; in dnode_reallocate()
579 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dnode_reallocate()
580 dnode_setdirty(dn, tx); in dnode_reallocate()
581 if (dn->dn_datablksz != blocksize) { in dnode_reallocate()
583 ASSERT(dn->dn_maxblkid == 0 && in dnode_reallocate()
584 (BP_IS_HOLE(&dn->dn_phys->dn_blkptr[0]) || in dnode_reallocate()
585 dnode_block_freed(dn, 0))); in dnode_reallocate()
586 dnode_setdblksz(dn, blocksize); in dnode_reallocate()
587 dn->dn_next_blksz[tx->tx_txg&TXG_MASK] = blocksize; in dnode_reallocate()
589 if (dn->dn_bonuslen != bonuslen) in dnode_reallocate()
590 dn->dn_next_bonuslen[tx->tx_txg&TXG_MASK] = bonuslen; in dnode_reallocate()
596 if (dn->dn_bonustype != bonustype) in dnode_reallocate()
597 dn->dn_next_bonustype[tx->tx_txg&TXG_MASK] = bonustype; in dnode_reallocate()
598 if (dn->dn_nblkptr != nblkptr) in dnode_reallocate()
599 dn->dn_next_nblkptr[tx->tx_txg&TXG_MASK] = nblkptr; in dnode_reallocate()
600 if (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR) { in dnode_reallocate()
601 dbuf_rm_spill(dn, tx); in dnode_reallocate()
602 dnode_rm_spill(dn, tx); in dnode_reallocate()
604 rw_exit(&dn->dn_struct_rwlock); in dnode_reallocate()
607 dn->dn_type = ot; in dnode_reallocate()
610 mutex_enter(&dn->dn_mtx); in dnode_reallocate()
611 dn->dn_bonustype = bonustype; in dnode_reallocate()
612 dn->dn_bonuslen = bonuslen; in dnode_reallocate()
613 dn->dn_nblkptr = nblkptr; in dnode_reallocate()
614 dn->dn_checksum = ZIO_CHECKSUM_INHERIT; in dnode_reallocate()
615 dn->dn_compress = ZIO_COMPRESS_INHERIT; in dnode_reallocate()
616 ASSERT3U(dn->dn_nblkptr, <=, DN_MAX_NBLKPTR); in dnode_reallocate()
619 if (dn->dn_bonus) { in dnode_reallocate()
620 dn->dn_bonus->db.db_size = in dnode_reallocate()
621 DN_MAX_BONUSLEN - (dn->dn_nblkptr-1) * sizeof (blkptr_t); in dnode_reallocate()
622 ASSERT(dn->dn_bonuslen <= dn->dn_bonus->db.db_size); in dnode_reallocate()
625 dn->dn_allocated_txg = tx->tx_txg; in dnode_reallocate()
626 mutex_exit(&dn->dn_mtx); in dnode_reallocate()
925 dnode_t *dn = dnh->dnh_dnode; in dnode_special_close() local
933 while (refcount_count(&dn->dn_holds) > 0) in dnode_special_close()
936 dnode_destroy(dn); /* implicit zrl_remove() */ in dnode_special_close()
945 dnode_t *dn = dnode_create(os, dnp, NULL, object, dnh); in dnode_special_open() local
946 dnh->dnh_dnode = dn; in dnode_special_open()
948 DNODE_VERIFY(dn); in dnode_special_open()
949 return (dn); in dnode_special_open()
963 dnode_t *dn; in dnode_buf_pageout() local
976 dn = dnh->dnh_dnode; in dnode_buf_pageout()
983 ASSERT(refcount_is_zero(&dn->dn_holds)); in dnode_buf_pageout()
984 ASSERT(refcount_is_zero(&dn->dn_tx_holds)); in dnode_buf_pageout()
986 dnode_destroy(dn); /* implicit zrl_remove() */ in dnode_buf_pageout()
1008 dnode_t *mdn, *dn; in dnode_hold_impl() local
1024 dn = (object == DMU_USERUSED_OBJECT) ? in dnode_hold_impl()
1026 if (dn == NULL) in dnode_hold_impl()
1028 type = dn->dn_type; in dnode_hold_impl()
1033 DNODE_VERIFY(dn); in dnode_hold_impl()
1034 (void) refcount_add(&dn->dn_holds, tag); in dnode_hold_impl()
1035 *dnp = dn; in dnode_hold_impl()
1094 if ((dn = dnh->dnh_dnode) == NULL) { in dnode_hold_impl()
1098 dn = dnode_create(os, phys, db, object, dnh); in dnode_hold_impl()
1099 winner = atomic_cas_ptr(&dnh->dnh_dnode, NULL, dn); in dnode_hold_impl()
1102 dnode_destroy(dn); /* implicit zrl_remove() */ in dnode_hold_impl()
1103 dn = winner; in dnode_hold_impl()
1107 mutex_enter(&dn->dn_mtx); in dnode_hold_impl()
1108 type = dn->dn_type; in dnode_hold_impl()
1109 if (dn->dn_free_txg || in dnode_hold_impl()
1112 (type != DMU_OT_NONE || !refcount_is_zero(&dn->dn_holds)))) { in dnode_hold_impl()
1113 mutex_exit(&dn->dn_mtx); in dnode_hold_impl()
1118 mutex_exit(&dn->dn_mtx); in dnode_hold_impl()
1120 if (refcount_add(&dn->dn_holds, tag) == 1) in dnode_hold_impl()
1125 DNODE_VERIFY(dn); in dnode_hold_impl()
1126 ASSERT3P(dn->dn_dbuf, ==, db); in dnode_hold_impl()
1127 ASSERT3U(dn->dn_object, ==, object); in dnode_hold_impl()
1130 *dnp = dn; in dnode_hold_impl()
1149 dnode_add_ref(dnode_t *dn, void *tag) in dnode_add_ref() argument
1151 mutex_enter(&dn->dn_mtx); in dnode_add_ref()
1152 if (refcount_is_zero(&dn->dn_holds)) { in dnode_add_ref()
1153 mutex_exit(&dn->dn_mtx); in dnode_add_ref()
1156 VERIFY(1 < refcount_add(&dn->dn_holds, tag)); in dnode_add_ref()
1157 mutex_exit(&dn->dn_mtx); in dnode_add_ref()
1162 dnode_rele(dnode_t *dn, void *tag) in dnode_rele() argument
1166 dmu_buf_impl_t *db = dn->dn_dbuf; in dnode_rele()
1167 dnode_handle_t *dnh = dn->dn_handle; in dnode_rele()
1169 mutex_enter(&dn->dn_mtx); in dnode_rele()
1170 refs = refcount_remove(&dn->dn_holds, tag); in dnode_rele()
1171 mutex_exit(&dn->dn_mtx); in dnode_rele()
1199 dnode_setdirty(dnode_t *dn, dmu_tx_t *tx) in dnode_setdirty() argument
1201 objset_t *os = dn->dn_objset; in dnode_setdirty()
1204 if (DMU_OBJECT_IS_SPECIAL(dn->dn_object)) { in dnode_setdirty()
1209 DNODE_VERIFY(dn); in dnode_setdirty()
1212 mutex_enter(&dn->dn_mtx); in dnode_setdirty()
1213 ASSERT(dn->dn_phys->dn_type || dn->dn_allocated_txg); in dnode_setdirty()
1214 ASSERT(dn->dn_free_txg == 0 || dn->dn_free_txg >= txg); in dnode_setdirty()
1215 mutex_exit(&dn->dn_mtx); in dnode_setdirty()
1221 dmu_objset_userquota_get_ids(dn, B_TRUE, tx); in dnode_setdirty()
1228 if (list_link_active(&dn->dn_dirty_link[txg & TXG_MASK])) { in dnode_setdirty()
1233 ASSERT(!refcount_is_zero(&dn->dn_holds) || list_head(&dn->dn_dbufs)); in dnode_setdirty()
1234 ASSERT(dn->dn_datablksz != 0); in dnode_setdirty()
1235 ASSERT3U(dn->dn_next_bonuslen[txg&TXG_MASK], ==, 0); in dnode_setdirty()
1236 ASSERT3U(dn->dn_next_blksz[txg&TXG_MASK], ==, 0); in dnode_setdirty()
1237 ASSERT3U(dn->dn_next_bonustype[txg&TXG_MASK], ==, 0); in dnode_setdirty()
1240 dn->dn_object, txg); in dnode_setdirty()
1242 if (dn->dn_free_txg > 0 && dn->dn_free_txg <= txg) { in dnode_setdirty()
1243 list_insert_tail(&os->os_free_dnodes[txg&TXG_MASK], dn); in dnode_setdirty()
1245 list_insert_tail(&os->os_dirty_dnodes[txg&TXG_MASK], dn); in dnode_setdirty()
1259 VERIFY(dnode_add_ref(dn, (void *)(uintptr_t)tx->tx_txg)); in dnode_setdirty()
1261 (void) dbuf_dirty(dn->dn_dbuf, tx); in dnode_setdirty()
1267 dnode_free(dnode_t *dn, dmu_tx_t *tx) in dnode_free() argument
1271 dprintf("dn=%p txg=%llu\n", dn, tx->tx_txg); in dnode_free()
1276 mutex_enter(&dn->dn_mtx); in dnode_free()
1277 if (dn->dn_type == DMU_OT_NONE || dn->dn_free_txg) { in dnode_free()
1278 mutex_exit(&dn->dn_mtx); in dnode_free()
1281 dn->dn_free_txg = tx->tx_txg; in dnode_free()
1282 mutex_exit(&dn->dn_mtx); in dnode_free()
1288 mutex_enter(&dn->dn_objset->os_lock); in dnode_free()
1289 if (list_link_active(&dn->dn_dirty_link[txgoff])) { in dnode_free()
1290 list_remove(&dn->dn_objset->os_dirty_dnodes[txgoff], dn); in dnode_free()
1291 list_insert_tail(&dn->dn_objset->os_free_dnodes[txgoff], dn); in dnode_free()
1292 mutex_exit(&dn->dn_objset->os_lock); in dnode_free()
1294 mutex_exit(&dn->dn_objset->os_lock); in dnode_free()
1295 dnode_setdirty(dn, tx); in dnode_free()
1304 dnode_set_blksz(dnode_t *dn, uint64_t size, int ibs, dmu_tx_t *tx) in dnode_set_blksz() argument
1316 if (ibs == dn->dn_indblkshift) in dnode_set_blksz()
1319 if (size >> SPA_MINBLOCKSHIFT == dn->dn_datablkszsec && ibs == 0) in dnode_set_blksz()
1322 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dnode_set_blksz()
1325 if (dn->dn_phys->dn_maxblkid != 0) in dnode_set_blksz()
1328 mutex_enter(&dn->dn_dbufs_mtx); in dnode_set_blksz()
1329 for (db = list_head(&dn->dn_dbufs); db; db = db_next) { in dnode_set_blksz()
1330 db_next = list_next(&dn->dn_dbufs, db); in dnode_set_blksz()
1334 mutex_exit(&dn->dn_dbufs_mtx); in dnode_set_blksz()
1338 mutex_exit(&dn->dn_dbufs_mtx); in dnode_set_blksz()
1340 if (ibs && dn->dn_nlevels != 1) in dnode_set_blksz()
1344 err = dbuf_hold_impl(dn, 0, 0, TRUE, FTAG, &db); in dnode_set_blksz()
1350 dnode_setdblksz(dn, size); in dnode_set_blksz()
1351 dnode_setdirty(dn, tx); in dnode_set_blksz()
1352 dn->dn_next_blksz[tx->tx_txg&TXG_MASK] = size; in dnode_set_blksz()
1354 dn->dn_indblkshift = ibs; in dnode_set_blksz()
1355 dn->dn_next_indblkshift[tx->tx_txg&TXG_MASK] = ibs; in dnode_set_blksz()
1361 rw_exit(&dn->dn_struct_rwlock); in dnode_set_blksz()
1365 rw_exit(&dn->dn_struct_rwlock); in dnode_set_blksz()
1371 dnode_new_blkid(dnode_t *dn, uint64_t blkid, dmu_tx_t *tx, boolean_t have_read) in dnode_new_blkid() argument
1380 RW_READ_HELD(&dn->dn_struct_rwlock) : in dnode_new_blkid()
1381 RW_WRITE_HELD(&dn->dn_struct_rwlock)); in dnode_new_blkid()
1388 if (blkid <= dn->dn_maxblkid) in dnode_new_blkid()
1391 if (!rw_tryupgrade(&dn->dn_struct_rwlock)) { in dnode_new_blkid()
1392 rw_exit(&dn->dn_struct_rwlock); in dnode_new_blkid()
1393 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dnode_new_blkid()
1397 if (blkid <= dn->dn_maxblkid) in dnode_new_blkid()
1400 dn->dn_maxblkid = blkid; in dnode_new_blkid()
1406 epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; in dnode_new_blkid()
1407 for (sz = dn->dn_nblkptr; in dnode_new_blkid()
1408 sz <= blkid && sz >= dn->dn_nblkptr; sz <<= epbs) in dnode_new_blkid()
1411 if (new_nlevels > dn->dn_nlevels) { in dnode_new_blkid()
1412 int old_nlevels = dn->dn_nlevels; in dnode_new_blkid()
1417 dn->dn_nlevels = new_nlevels; in dnode_new_blkid()
1419 ASSERT3U(new_nlevels, >, dn->dn_next_nlevels[txgoff]); in dnode_new_blkid()
1420 dn->dn_next_nlevels[txgoff] = new_nlevels; in dnode_new_blkid()
1423 db = dbuf_hold_level(dn, old_nlevels, 0, FTAG); in dnode_new_blkid()
1429 mutex_enter(&dn->dn_mtx); in dnode_new_blkid()
1431 list = &dn->dn_dirty_records[txgoff]; in dnode_new_blkid()
1433 dr_next = list_next(&dn->dn_dirty_records[txgoff], dr); in dnode_new_blkid()
1438 list_remove(&dn->dn_dirty_records[txgoff], dr); in dnode_new_blkid()
1444 mutex_exit(&dn->dn_mtx); in dnode_new_blkid()
1449 rw_downgrade(&dn->dn_struct_rwlock); in dnode_new_blkid()
1453 dnode_clear_range(dnode_t *dn, uint64_t blkid, uint64_t nblks, dmu_tx_t *tx) in dnode_clear_range() argument
1455 avl_tree_t *tree = &dn->dn_ranges[tx->tx_txg&TXG_MASK]; in dnode_clear_range()
1461 ASSERT(MUTEX_HELD(&dn->dn_mtx)); in dnode_clear_range()
1464 dprintf_dnode(dn, "blkid=%llu nblks=%llu txg=%llu\n", in dnode_clear_range()
1506 dnode_free_range(dnode_t *dn, uint64_t off, uint64_t len, dmu_tx_t *tx) in dnode_free_range() argument
1514 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dnode_free_range()
1515 blksz = dn->dn_datablksz; in dnode_free_range()
1516 blkshift = dn->dn_datablkshift; in dnode_free_range()
1517 epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; in dnode_free_range()
1530 if ((off >> blkshift) > dn->dn_maxblkid) in dnode_free_range()
1533 ASSERT(dn->dn_maxblkid == 0); in dnode_free_range()
1554 if (dbuf_hold_impl(dn, 0, dbuf_whichblock(dn, off), TRUE, in dnode_free_range()
1561 rw_exit(&dn->dn_struct_rwlock); in dnode_free_range()
1563 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dnode_free_range()
1578 if ((off >> blkshift) > dn->dn_maxblkid) in dnode_free_range()
1592 if (dbuf_hold_impl(dn, 0, dbuf_whichblock(dn, off+len), in dnode_free_range()
1597 rw_exit(&dn->dn_struct_rwlock); in dnode_free_range()
1599 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); in dnode_free_range()
1624 if (dn->dn_nlevels > 1) { in dnode_free_range()
1626 int shift = epbs + dn->dn_datablkshift; in dnode_free_range()
1629 if (db = dbuf_hold_level(dn, 1, first, FTAG)) { in dnode_free_range()
1634 last = dn->dn_maxblkid >> epbs; in dnode_free_range()
1637 if (last > first && (db = dbuf_hold_level(dn, 1, last, FTAG))) { in dnode_free_range()
1645 err = dnode_next_offset(dn, in dnode_free_range()
1651 db = dbuf_hold_level(dn, 1, i, FTAG); in dnode_free_range()
1663 mutex_enter(&dn->dn_mtx); in dnode_free_range()
1664 dnode_clear_range(dn, blkid, nblks, tx); in dnode_free_range()
1668 avl_tree_t *tree = &dn->dn_ranges[tx->tx_txg&TXG_MASK]; in dnode_free_range()
1677 dprintf_dnode(dn, "blkid=%llu nblks=%llu txg=%llu\n", in dnode_free_range()
1680 mutex_exit(&dn->dn_mtx); in dnode_free_range()
1682 dbuf_free_range(dn, blkid, blkid + nblks - 1, tx); in dnode_free_range()
1683 dnode_setdirty(dn, tx); in dnode_free_range()
1685 if (trunc && dn->dn_maxblkid >= (off >> blkshift)) in dnode_free_range()
1686 dn->dn_maxblkid = (off >> blkshift ? (off >> blkshift) - 1 : 0); in dnode_free_range()
1688 rw_exit(&dn->dn_struct_rwlock); in dnode_free_range()
1692 dnode_spill_freed(dnode_t *dn) in dnode_spill_freed() argument
1696 mutex_enter(&dn->dn_mtx); in dnode_spill_freed()
1698 if (dn->dn_rm_spillblk[i] == DN_KILL_SPILLBLK) in dnode_spill_freed()
1701 mutex_exit(&dn->dn_mtx); in dnode_spill_freed()
1707 dnode_block_freed(dnode_t *dn, uint64_t blkid) in dnode_block_freed() argument
1710 void *dp = spa_get_dsl(dn->dn_objset->os_spa); in dnode_block_freed()
1723 if (dn->dn_free_txg) in dnode_block_freed()
1727 return (dnode_spill_freed(dn)); in dnode_block_freed()
1730 mutex_enter(&dn->dn_mtx); in dnode_block_freed()
1735 range_found = avl_find(&dn->dn_ranges[i], &range_tofind, &idx); in dnode_block_freed()
1740 range_found = avl_nearest(&dn->dn_ranges[i], idx, AVL_BEFORE); in dnode_block_freed()
1745 mutex_exit(&dn->dn_mtx); in dnode_block_freed()
1751 dnode_diduse_space(dnode_t *dn, int64_t delta) in dnode_diduse_space() argument
1754 dprintf_dnode(dn, "dn=%p dnp=%p used=%llu delta=%lld\n", in dnode_diduse_space()
1755 dn, dn->dn_phys, in dnode_diduse_space()
1756 (u_longlong_t)dn->dn_phys->dn_used, in dnode_diduse_space()
1759 mutex_enter(&dn->dn_mtx); in dnode_diduse_space()
1760 space = DN_USED_BYTES(dn->dn_phys); in dnode_diduse_space()
1767 if (spa_version(dn->dn_objset->os_spa) < SPA_VERSION_DNODE_BYTES) { in dnode_diduse_space()
1768 ASSERT((dn->dn_phys->dn_flags & DNODE_FLAG_USED_BYTES) == 0); in dnode_diduse_space()
1770 dn->dn_phys->dn_used = space >> DEV_BSHIFT; in dnode_diduse_space()
1772 dn->dn_phys->dn_used = space; in dnode_diduse_space()
1773 dn->dn_phys->dn_flags |= DNODE_FLAG_USED_BYTES; in dnode_diduse_space()
1775 mutex_exit(&dn->dn_mtx); in dnode_diduse_space()
1784 dnode_willuse_space(dnode_t *dn, int64_t space, dmu_tx_t *tx) in dnode_willuse_space() argument
1786 objset_t *os = dn->dn_objset; in dnode_willuse_space()
1815 dnode_next_offset_level(dnode_t *dn, int flags, uint64_t *offset, in dnode_next_offset_level() argument
1820 uint64_t epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; in dnode_next_offset_level()
1827 dn->dn_object, *offset, lvl, dn->dn_phys->dn_nlevels); in dnode_next_offset_level()
1833 if (lvl == dn->dn_phys->dn_nlevels) { in dnode_next_offset_level()
1835 epb = dn->dn_phys->dn_nblkptr; in dnode_next_offset_level()
1836 data = dn->dn_phys->dn_blkptr; in dnode_next_offset_level()
1838 uint64_t blkid = dbuf_whichblock(dn, *offset) >> (epbs * lvl); in dnode_next_offset_level()
1839 error = dbuf_hold_impl(dn, lvl, blkid, TRUE, FTAG, &db); in dnode_next_offset_level()
1872 ASSERT(dn->dn_type == DMU_OT_DNODE); in dnode_next_offset_level()
1885 span = (lvl - 1) * epbs + dn->dn_datablkshift; in dnode_next_offset_level()
1946 dnode_next_offset(dnode_t *dn, int flags, uint64_t *offset, in dnode_next_offset() argument
1954 rw_enter(&dn->dn_struct_rwlock, RW_READER); in dnode_next_offset()
1956 if (dn->dn_phys->dn_nlevels == 0) { in dnode_next_offset()
1961 if (dn->dn_datablkshift == 0) { in dnode_next_offset()
1962 if (*offset < dn->dn_datablksz) { in dnode_next_offset()
1964 *offset = dn->dn_datablksz; in dnode_next_offset()
1971 maxlvl = dn->dn_phys->dn_nlevels; in dnode_next_offset()
1974 error = dnode_next_offset_level(dn, in dnode_next_offset()
1981 error = dnode_next_offset_level(dn, in dnode_next_offset()
1990 rw_exit(&dn->dn_struct_rwlock); in dnode_next_offset()