1eda14cbcSMatt Macy /* 2eda14cbcSMatt Macy * CDDL HEADER START 3eda14cbcSMatt Macy * 4eda14cbcSMatt Macy * The contents of this file are subject to the terms of the 5eda14cbcSMatt Macy * Common Development and Distribution License (the "License"). 6eda14cbcSMatt Macy * You may not use this file except in compliance with the License. 7eda14cbcSMatt Macy * 8eda14cbcSMatt Macy * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9271171e0SMartin Matuska * or https://opensource.org/licenses/CDDL-1.0. 10eda14cbcSMatt Macy * See the License for the specific language governing permissions 11eda14cbcSMatt Macy * and limitations under the License. 12eda14cbcSMatt Macy * 13eda14cbcSMatt Macy * When distributing Covered Code, include this CDDL HEADER in each 14eda14cbcSMatt Macy * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15eda14cbcSMatt Macy * If applicable, add the following below this CDDL HEADER, with the 16eda14cbcSMatt Macy * fields enclosed by brackets "[]" replaced with your own identifying 17eda14cbcSMatt Macy * information: Portions Copyright [yyyy] [name of copyright owner] 18eda14cbcSMatt Macy * 19eda14cbcSMatt Macy * CDDL HEADER END 20eda14cbcSMatt Macy */ 21eda14cbcSMatt Macy /* 22eda14cbcSMatt Macy * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23eda14cbcSMatt Macy * Copyright (c) 2011, 2020 by Delphix. All rights reserved. 24eda14cbcSMatt Macy * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. 25eda14cbcSMatt Macy * Copyright (c) 2013, Joyent, Inc. All rights reserved. 26eda14cbcSMatt Macy * Copyright (c) 2016, Nexenta Systems, Inc. All rights reserved. 27eda14cbcSMatt Macy * Copyright (c) 2015 by Chunwei Chen. All rights reserved. 28eda14cbcSMatt Macy * Copyright (c) 2019 Datto Inc. 29ce4dcb97SMartin Matuska * Copyright (c) 2019, 2023, Klara Inc. 30eda14cbcSMatt Macy * Copyright (c) 2019, Allan Jude 31dbd5678dSMartin Matuska * Copyright (c) 2022 Hewlett Packard Enterprise Development LP. 322a58b312SMartin Matuska * Copyright (c) 2021, 2022 by Pawel Jakub Dawidek 33eda14cbcSMatt Macy */ 34eda14cbcSMatt Macy 35eda14cbcSMatt Macy #include <sys/dmu.h> 36eda14cbcSMatt Macy #include <sys/dmu_impl.h> 37eda14cbcSMatt Macy #include <sys/dmu_tx.h> 38eda14cbcSMatt Macy #include <sys/dbuf.h> 39eda14cbcSMatt Macy #include <sys/dnode.h> 40eda14cbcSMatt Macy #include <sys/zfs_context.h> 41eda14cbcSMatt Macy #include <sys/dmu_objset.h> 42eda14cbcSMatt Macy #include <sys/dmu_traverse.h> 43eda14cbcSMatt Macy #include <sys/dsl_dataset.h> 44eda14cbcSMatt Macy #include <sys/dsl_dir.h> 45eda14cbcSMatt Macy #include <sys/dsl_pool.h> 46eda14cbcSMatt Macy #include <sys/dsl_synctask.h> 47eda14cbcSMatt Macy #include <sys/dsl_prop.h> 48eda14cbcSMatt Macy #include <sys/dmu_zfetch.h> 49eda14cbcSMatt Macy #include <sys/zfs_ioctl.h> 50eda14cbcSMatt Macy #include <sys/zap.h> 51eda14cbcSMatt Macy #include <sys/zio_checksum.h> 52eda14cbcSMatt Macy #include <sys/zio_compress.h> 53eda14cbcSMatt Macy #include <sys/sa.h> 54eda14cbcSMatt Macy #include <sys/zfeature.h> 55eda14cbcSMatt Macy #include <sys/abd.h> 562a58b312SMartin Matuska #include <sys/brt.h> 57eda14cbcSMatt Macy #include <sys/trace_zfs.h> 58ba27dd8bSMartin Matuska #include <sys/zfs_racct.h> 59eda14cbcSMatt Macy #include <sys/zfs_rlock.h> 60eda14cbcSMatt Macy #ifdef _KERNEL 61eda14cbcSMatt Macy #include <sys/vmsystm.h> 62eda14cbcSMatt Macy #include <sys/zfs_znode.h> 63eda14cbcSMatt Macy #endif 64eda14cbcSMatt Macy 65eda14cbcSMatt Macy /* 66eda14cbcSMatt Macy * Enable/disable nopwrite feature. 67eda14cbcSMatt Macy */ 68e92ffd9bSMartin Matuska static int zfs_nopwrite_enabled = 1; 69eda14cbcSMatt Macy 70eda14cbcSMatt Macy /* 71eda14cbcSMatt Macy * Tunable to control percentage of dirtied L1 blocks from frees allowed into 72eda14cbcSMatt Macy * one TXG. After this threshold is crossed, additional dirty blocks from frees 73eda14cbcSMatt Macy * will wait until the next TXG. 74eda14cbcSMatt Macy * A value of zero will disable this throttle. 75eda14cbcSMatt Macy */ 76dbd5678dSMartin Matuska static uint_t zfs_per_txg_dirty_frees_percent = 30; 77eda14cbcSMatt Macy 78eda14cbcSMatt Macy /* 79681ce946SMartin Matuska * Enable/disable forcing txg sync when dirty checking for holes with lseek(). 80681ce946SMartin Matuska * By default this is enabled to ensure accurate hole reporting, it can result 81681ce946SMartin Matuska * in a significant performance penalty for lseek(SEEK_HOLE) heavy workloads. 82681ce946SMartin Matuska * Disabling this option will result in holes never being reported in dirty 83681ce946SMartin Matuska * files which is always safe. 84eda14cbcSMatt Macy */ 85e92ffd9bSMartin Matuska static int zfs_dmu_offset_next_sync = 1; 86eda14cbcSMatt Macy 87eda14cbcSMatt Macy /* 88eda14cbcSMatt Macy * Limit the amount we can prefetch with one call to this amount. This 89eda14cbcSMatt Macy * helps to limit the amount of memory that can be used by prefetching. 90eda14cbcSMatt Macy * Larger objects should be prefetched a bit at a time. 91eda14cbcSMatt Macy */ 92315ee00fSMartin Matuska #ifdef _ILP32 93315ee00fSMartin Matuska uint_t dmu_prefetch_max = 8 * 1024 * 1024; 94315ee00fSMartin Matuska #else 95be181ee2SMartin Matuska uint_t dmu_prefetch_max = 8 * SPA_MAXBLOCKSIZE; 96315ee00fSMartin Matuska #endif 97eda14cbcSMatt Macy 98e2df9bb4SMartin Matuska /* 99e2df9bb4SMartin Matuska * Override copies= for dedup state objects. 0 means the traditional behaviour 100e2df9bb4SMartin Matuska * (ie the default for the containing objset ie 3 for the MOS). 101e2df9bb4SMartin Matuska */ 102e2df9bb4SMartin Matuska uint_t dmu_ddt_copies = 0; 103e2df9bb4SMartin Matuska 104eda14cbcSMatt Macy const dmu_object_type_info_t dmu_ot[DMU_OT_NUMTYPES] = { 105eda14cbcSMatt Macy {DMU_BSWAP_UINT8, TRUE, FALSE, FALSE, "unallocated" }, 106eda14cbcSMatt Macy {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "object directory" }, 107eda14cbcSMatt Macy {DMU_BSWAP_UINT64, TRUE, TRUE, FALSE, "object array" }, 108eda14cbcSMatt Macy {DMU_BSWAP_UINT8, TRUE, FALSE, FALSE, "packed nvlist" }, 109eda14cbcSMatt Macy {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "packed nvlist size" }, 110eda14cbcSMatt Macy {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "bpobj" }, 111eda14cbcSMatt Macy {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "bpobj header" }, 112eda14cbcSMatt Macy {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "SPA space map header" }, 113eda14cbcSMatt Macy {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "SPA space map" }, 114eda14cbcSMatt Macy {DMU_BSWAP_UINT64, TRUE, FALSE, TRUE, "ZIL intent log" }, 115eda14cbcSMatt Macy {DMU_BSWAP_DNODE, TRUE, FALSE, TRUE, "DMU dnode" }, 116eda14cbcSMatt Macy {DMU_BSWAP_OBJSET, TRUE, TRUE, FALSE, "DMU objset" }, 117eda14cbcSMatt Macy {DMU_BSWAP_UINT64, TRUE, TRUE, FALSE, "DSL directory" }, 118eda14cbcSMatt Macy {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL directory child map"}, 119eda14cbcSMatt Macy {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL dataset snap map" }, 120eda14cbcSMatt Macy {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL props" }, 121eda14cbcSMatt Macy {DMU_BSWAP_UINT64, TRUE, TRUE, FALSE, "DSL dataset" }, 122eda14cbcSMatt Macy {DMU_BSWAP_ZNODE, TRUE, FALSE, FALSE, "ZFS znode" }, 123eda14cbcSMatt Macy {DMU_BSWAP_OLDACL, TRUE, FALSE, TRUE, "ZFS V0 ACL" }, 124eda14cbcSMatt Macy {DMU_BSWAP_UINT8, FALSE, FALSE, TRUE, "ZFS plain file" }, 125eda14cbcSMatt Macy {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "ZFS directory" }, 126eda14cbcSMatt Macy {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "ZFS master node" }, 127eda14cbcSMatt Macy {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "ZFS delete queue" }, 128eda14cbcSMatt Macy {DMU_BSWAP_UINT8, FALSE, FALSE, TRUE, "zvol object" }, 129eda14cbcSMatt Macy {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "zvol prop" }, 130eda14cbcSMatt Macy {DMU_BSWAP_UINT8, FALSE, FALSE, TRUE, "other uint8[]" }, 131eda14cbcSMatt Macy {DMU_BSWAP_UINT64, FALSE, FALSE, TRUE, "other uint64[]" }, 132eda14cbcSMatt Macy {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "other ZAP" }, 133eda14cbcSMatt Macy {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "persistent error log" }, 134eda14cbcSMatt Macy {DMU_BSWAP_UINT8, TRUE, FALSE, FALSE, "SPA history" }, 135eda14cbcSMatt Macy {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "SPA history offsets" }, 136eda14cbcSMatt Macy {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "Pool properties" }, 137eda14cbcSMatt Macy {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL permissions" }, 138eda14cbcSMatt Macy {DMU_BSWAP_ACL, TRUE, FALSE, TRUE, "ZFS ACL" }, 139eda14cbcSMatt Macy {DMU_BSWAP_UINT8, TRUE, FALSE, TRUE, "ZFS SYSACL" }, 140eda14cbcSMatt Macy {DMU_BSWAP_UINT8, TRUE, FALSE, TRUE, "FUID table" }, 141eda14cbcSMatt Macy {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "FUID table size" }, 142eda14cbcSMatt Macy {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL dataset next clones"}, 143eda14cbcSMatt Macy {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "scan work queue" }, 144eda14cbcSMatt Macy {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "ZFS user/group/project used" }, 145eda14cbcSMatt Macy {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "ZFS user/group/project quota"}, 146eda14cbcSMatt Macy {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "snapshot refcount tags"}, 147eda14cbcSMatt Macy {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "DDT ZAP algorithm" }, 148eda14cbcSMatt Macy {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "DDT statistics" }, 149eda14cbcSMatt Macy {DMU_BSWAP_UINT8, TRUE, FALSE, TRUE, "System attributes" }, 150eda14cbcSMatt Macy {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "SA master node" }, 151eda14cbcSMatt Macy {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "SA attr registration" }, 152eda14cbcSMatt Macy {DMU_BSWAP_ZAP, TRUE, FALSE, TRUE, "SA attr layouts" }, 153eda14cbcSMatt Macy {DMU_BSWAP_ZAP, TRUE, FALSE, FALSE, "scan translations" }, 154eda14cbcSMatt Macy {DMU_BSWAP_UINT8, FALSE, FALSE, TRUE, "deduplicated block" }, 155eda14cbcSMatt Macy {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL deadlist map" }, 156eda14cbcSMatt Macy {DMU_BSWAP_UINT64, TRUE, TRUE, FALSE, "DSL deadlist map hdr" }, 157eda14cbcSMatt Macy {DMU_BSWAP_ZAP, TRUE, TRUE, FALSE, "DSL dir clones" }, 158eda14cbcSMatt Macy {DMU_BSWAP_UINT64, TRUE, FALSE, FALSE, "bpobj subobj" } 159eda14cbcSMatt Macy }; 160eda14cbcSMatt Macy 161be181ee2SMartin Matuska dmu_object_byteswap_info_t dmu_ot_byteswap[DMU_BSWAP_NUMFUNCS] = { 162eda14cbcSMatt Macy { byteswap_uint8_array, "uint8" }, 163eda14cbcSMatt Macy { byteswap_uint16_array, "uint16" }, 164eda14cbcSMatt Macy { byteswap_uint32_array, "uint32" }, 165eda14cbcSMatt Macy { byteswap_uint64_array, "uint64" }, 166eda14cbcSMatt Macy { zap_byteswap, "zap" }, 167eda14cbcSMatt Macy { dnode_buf_byteswap, "dnode" }, 168eda14cbcSMatt Macy { dmu_objset_byteswap, "objset" }, 169eda14cbcSMatt Macy { zfs_znode_byteswap, "znode" }, 170eda14cbcSMatt Macy { zfs_oldacl_byteswap, "oldacl" }, 171eda14cbcSMatt Macy { zfs_acl_byteswap, "acl" } 172eda14cbcSMatt Macy }; 173eda14cbcSMatt Macy 174315ee00fSMartin Matuska int 175eda14cbcSMatt Macy dmu_buf_hold_noread_by_dnode(dnode_t *dn, uint64_t offset, 176a0b956f5SMartin Matuska const void *tag, dmu_buf_t **dbp) 177eda14cbcSMatt Macy { 178eda14cbcSMatt Macy uint64_t blkid; 179eda14cbcSMatt Macy dmu_buf_impl_t *db; 180eda14cbcSMatt Macy 181eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_READER); 182eda14cbcSMatt Macy blkid = dbuf_whichblock(dn, 0, offset); 183eda14cbcSMatt Macy db = dbuf_hold(dn, blkid, tag); 184eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock); 185eda14cbcSMatt Macy 186eda14cbcSMatt Macy if (db == NULL) { 187eda14cbcSMatt Macy *dbp = NULL; 188eda14cbcSMatt Macy return (SET_ERROR(EIO)); 189eda14cbcSMatt Macy } 190eda14cbcSMatt Macy 191eda14cbcSMatt Macy *dbp = &db->db; 192eda14cbcSMatt Macy return (0); 193eda14cbcSMatt Macy } 194315ee00fSMartin Matuska 195eda14cbcSMatt Macy int 196eda14cbcSMatt Macy dmu_buf_hold_noread(objset_t *os, uint64_t object, uint64_t offset, 197a0b956f5SMartin Matuska const void *tag, dmu_buf_t **dbp) 198eda14cbcSMatt Macy { 199eda14cbcSMatt Macy dnode_t *dn; 200eda14cbcSMatt Macy uint64_t blkid; 201eda14cbcSMatt Macy dmu_buf_impl_t *db; 202eda14cbcSMatt Macy int err; 203eda14cbcSMatt Macy 204eda14cbcSMatt Macy err = dnode_hold(os, object, FTAG, &dn); 205eda14cbcSMatt Macy if (err) 206eda14cbcSMatt Macy return (err); 207eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_READER); 208eda14cbcSMatt Macy blkid = dbuf_whichblock(dn, 0, offset); 209eda14cbcSMatt Macy db = dbuf_hold(dn, blkid, tag); 210eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock); 211eda14cbcSMatt Macy dnode_rele(dn, FTAG); 212eda14cbcSMatt Macy 213eda14cbcSMatt Macy if (db == NULL) { 214eda14cbcSMatt Macy *dbp = NULL; 215eda14cbcSMatt Macy return (SET_ERROR(EIO)); 216eda14cbcSMatt Macy } 217eda14cbcSMatt Macy 218eda14cbcSMatt Macy *dbp = &db->db; 219eda14cbcSMatt Macy return (err); 220eda14cbcSMatt Macy } 221eda14cbcSMatt Macy 222eda14cbcSMatt Macy int 223eda14cbcSMatt Macy dmu_buf_hold_by_dnode(dnode_t *dn, uint64_t offset, 224a0b956f5SMartin Matuska const void *tag, dmu_buf_t **dbp, int flags) 225eda14cbcSMatt Macy { 226eda14cbcSMatt Macy int err; 227eda14cbcSMatt Macy int db_flags = DB_RF_CANFAIL; 228eda14cbcSMatt Macy 229eda14cbcSMatt Macy if (flags & DMU_READ_NO_PREFETCH) 230eda14cbcSMatt Macy db_flags |= DB_RF_NOPREFETCH; 231eda14cbcSMatt Macy if (flags & DMU_READ_NO_DECRYPT) 232eda14cbcSMatt Macy db_flags |= DB_RF_NO_DECRYPT; 233eda14cbcSMatt Macy 234eda14cbcSMatt Macy err = dmu_buf_hold_noread_by_dnode(dn, offset, tag, dbp); 235eda14cbcSMatt Macy if (err == 0) { 236eda14cbcSMatt Macy dmu_buf_impl_t *db = (dmu_buf_impl_t *)(*dbp); 237eda14cbcSMatt Macy err = dbuf_read(db, NULL, db_flags); 238eda14cbcSMatt Macy if (err != 0) { 239eda14cbcSMatt Macy dbuf_rele(db, tag); 240eda14cbcSMatt Macy *dbp = NULL; 241eda14cbcSMatt Macy } 242eda14cbcSMatt Macy } 243eda14cbcSMatt Macy 244eda14cbcSMatt Macy return (err); 245eda14cbcSMatt Macy } 246eda14cbcSMatt Macy 247eda14cbcSMatt Macy int 248eda14cbcSMatt Macy dmu_buf_hold(objset_t *os, uint64_t object, uint64_t offset, 249a0b956f5SMartin Matuska const void *tag, dmu_buf_t **dbp, int flags) 250eda14cbcSMatt Macy { 251eda14cbcSMatt Macy int err; 252eda14cbcSMatt Macy int db_flags = DB_RF_CANFAIL; 253eda14cbcSMatt Macy 254eda14cbcSMatt Macy if (flags & DMU_READ_NO_PREFETCH) 255eda14cbcSMatt Macy db_flags |= DB_RF_NOPREFETCH; 256eda14cbcSMatt Macy if (flags & DMU_READ_NO_DECRYPT) 257eda14cbcSMatt Macy db_flags |= DB_RF_NO_DECRYPT; 258eda14cbcSMatt Macy 259eda14cbcSMatt Macy err = dmu_buf_hold_noread(os, object, offset, tag, dbp); 260eda14cbcSMatt Macy if (err == 0) { 261eda14cbcSMatt Macy dmu_buf_impl_t *db = (dmu_buf_impl_t *)(*dbp); 262eda14cbcSMatt Macy err = dbuf_read(db, NULL, db_flags); 263eda14cbcSMatt Macy if (err != 0) { 264eda14cbcSMatt Macy dbuf_rele(db, tag); 265eda14cbcSMatt Macy *dbp = NULL; 266eda14cbcSMatt Macy } 267eda14cbcSMatt Macy } 268eda14cbcSMatt Macy 269eda14cbcSMatt Macy return (err); 270eda14cbcSMatt Macy } 271eda14cbcSMatt Macy 272eda14cbcSMatt Macy int 273eda14cbcSMatt Macy dmu_bonus_max(void) 274eda14cbcSMatt Macy { 275eda14cbcSMatt Macy return (DN_OLD_MAX_BONUSLEN); 276eda14cbcSMatt Macy } 277eda14cbcSMatt Macy 278eda14cbcSMatt Macy int 279eda14cbcSMatt Macy dmu_set_bonus(dmu_buf_t *db_fake, int newsize, dmu_tx_t *tx) 280eda14cbcSMatt Macy { 281eda14cbcSMatt Macy dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 282eda14cbcSMatt Macy dnode_t *dn; 283eda14cbcSMatt Macy int error; 284eda14cbcSMatt Macy 285ce4dcb97SMartin Matuska if (newsize < 0 || newsize > db_fake->db_size) 286ce4dcb97SMartin Matuska return (SET_ERROR(EINVAL)); 287ce4dcb97SMartin Matuska 288eda14cbcSMatt Macy DB_DNODE_ENTER(db); 289eda14cbcSMatt Macy dn = DB_DNODE(db); 290eda14cbcSMatt Macy 291eda14cbcSMatt Macy if (dn->dn_bonus != db) { 292eda14cbcSMatt Macy error = SET_ERROR(EINVAL); 293eda14cbcSMatt Macy } else { 294eda14cbcSMatt Macy dnode_setbonuslen(dn, newsize, tx); 295eda14cbcSMatt Macy error = 0; 296eda14cbcSMatt Macy } 297eda14cbcSMatt Macy 298eda14cbcSMatt Macy DB_DNODE_EXIT(db); 299eda14cbcSMatt Macy return (error); 300eda14cbcSMatt Macy } 301eda14cbcSMatt Macy 302eda14cbcSMatt Macy int 303eda14cbcSMatt Macy dmu_set_bonustype(dmu_buf_t *db_fake, dmu_object_type_t type, dmu_tx_t *tx) 304eda14cbcSMatt Macy { 305eda14cbcSMatt Macy dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 306eda14cbcSMatt Macy dnode_t *dn; 307eda14cbcSMatt Macy int error; 308eda14cbcSMatt Macy 309ce4dcb97SMartin Matuska if (!DMU_OT_IS_VALID(type)) 310ce4dcb97SMartin Matuska return (SET_ERROR(EINVAL)); 311ce4dcb97SMartin Matuska 312eda14cbcSMatt Macy DB_DNODE_ENTER(db); 313eda14cbcSMatt Macy dn = DB_DNODE(db); 314eda14cbcSMatt Macy 315ce4dcb97SMartin Matuska if (dn->dn_bonus != db) { 316eda14cbcSMatt Macy error = SET_ERROR(EINVAL); 317eda14cbcSMatt Macy } else { 318eda14cbcSMatt Macy dnode_setbonus_type(dn, type, tx); 319eda14cbcSMatt Macy error = 0; 320eda14cbcSMatt Macy } 321eda14cbcSMatt Macy 322eda14cbcSMatt Macy DB_DNODE_EXIT(db); 323eda14cbcSMatt Macy return (error); 324eda14cbcSMatt Macy } 325eda14cbcSMatt Macy 326eda14cbcSMatt Macy dmu_object_type_t 327eda14cbcSMatt Macy dmu_get_bonustype(dmu_buf_t *db_fake) 328eda14cbcSMatt Macy { 329eda14cbcSMatt Macy dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 330eda14cbcSMatt Macy dmu_object_type_t type; 331eda14cbcSMatt Macy 332eda14cbcSMatt Macy DB_DNODE_ENTER(db); 333ce4dcb97SMartin Matuska type = DB_DNODE(db)->dn_bonustype; 334eda14cbcSMatt Macy DB_DNODE_EXIT(db); 335eda14cbcSMatt Macy 336eda14cbcSMatt Macy return (type); 337eda14cbcSMatt Macy } 338eda14cbcSMatt Macy 339eda14cbcSMatt Macy int 340eda14cbcSMatt Macy dmu_rm_spill(objset_t *os, uint64_t object, dmu_tx_t *tx) 341eda14cbcSMatt Macy { 342eda14cbcSMatt Macy dnode_t *dn; 343eda14cbcSMatt Macy int error; 344eda14cbcSMatt Macy 345eda14cbcSMatt Macy error = dnode_hold(os, object, FTAG, &dn); 346eda14cbcSMatt Macy dbuf_rm_spill(dn, tx); 347eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 348eda14cbcSMatt Macy dnode_rm_spill(dn, tx); 349eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock); 350eda14cbcSMatt Macy dnode_rele(dn, FTAG); 351eda14cbcSMatt Macy return (error); 352eda14cbcSMatt Macy } 353eda14cbcSMatt Macy 354eda14cbcSMatt Macy /* 355eda14cbcSMatt Macy * Lookup and hold the bonus buffer for the provided dnode. If the dnode 356eda14cbcSMatt Macy * has not yet been allocated a new bonus dbuf a will be allocated. 357eda14cbcSMatt Macy * Returns ENOENT, EIO, or 0. 358eda14cbcSMatt Macy */ 359a0b956f5SMartin Matuska int dmu_bonus_hold_by_dnode(dnode_t *dn, const void *tag, dmu_buf_t **dbp, 360eda14cbcSMatt Macy uint32_t flags) 361eda14cbcSMatt Macy { 362eda14cbcSMatt Macy dmu_buf_impl_t *db; 363eda14cbcSMatt Macy int error; 364eda14cbcSMatt Macy uint32_t db_flags = DB_RF_MUST_SUCCEED; 365eda14cbcSMatt Macy 366eda14cbcSMatt Macy if (flags & DMU_READ_NO_PREFETCH) 367eda14cbcSMatt Macy db_flags |= DB_RF_NOPREFETCH; 368eda14cbcSMatt Macy if (flags & DMU_READ_NO_DECRYPT) 369eda14cbcSMatt Macy db_flags |= DB_RF_NO_DECRYPT; 370eda14cbcSMatt Macy 371eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_READER); 372eda14cbcSMatt Macy if (dn->dn_bonus == NULL) { 3732a58b312SMartin Matuska if (!rw_tryupgrade(&dn->dn_struct_rwlock)) { 374eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock); 375eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 3762a58b312SMartin Matuska } 377eda14cbcSMatt Macy if (dn->dn_bonus == NULL) 378eda14cbcSMatt Macy dbuf_create_bonus(dn); 379eda14cbcSMatt Macy } 380eda14cbcSMatt Macy db = dn->dn_bonus; 381eda14cbcSMatt Macy 382eda14cbcSMatt Macy /* as long as the bonus buf is held, the dnode will be held */ 383eda14cbcSMatt Macy if (zfs_refcount_add(&db->db_holds, tag) == 1) { 384eda14cbcSMatt Macy VERIFY(dnode_add_ref(dn, db)); 385eda14cbcSMatt Macy atomic_inc_32(&dn->dn_dbufs_count); 386eda14cbcSMatt Macy } 387eda14cbcSMatt Macy 388eda14cbcSMatt Macy /* 389eda14cbcSMatt Macy * Wait to drop dn_struct_rwlock until after adding the bonus dbuf's 390eda14cbcSMatt Macy * hold and incrementing the dbuf count to ensure that dnode_move() sees 391eda14cbcSMatt Macy * a dnode hold for every dbuf. 392eda14cbcSMatt Macy */ 393eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock); 394eda14cbcSMatt Macy 395eda14cbcSMatt Macy error = dbuf_read(db, NULL, db_flags); 396eda14cbcSMatt Macy if (error) { 397eda14cbcSMatt Macy dnode_evict_bonus(dn); 398eda14cbcSMatt Macy dbuf_rele(db, tag); 399eda14cbcSMatt Macy *dbp = NULL; 400eda14cbcSMatt Macy return (error); 401eda14cbcSMatt Macy } 402eda14cbcSMatt Macy 403eda14cbcSMatt Macy *dbp = &db->db; 404eda14cbcSMatt Macy return (0); 405eda14cbcSMatt Macy } 406eda14cbcSMatt Macy 407eda14cbcSMatt Macy int 408a0b956f5SMartin Matuska dmu_bonus_hold(objset_t *os, uint64_t object, const void *tag, dmu_buf_t **dbp) 409eda14cbcSMatt Macy { 410eda14cbcSMatt Macy dnode_t *dn; 411eda14cbcSMatt Macy int error; 412eda14cbcSMatt Macy 413eda14cbcSMatt Macy error = dnode_hold(os, object, FTAG, &dn); 414eda14cbcSMatt Macy if (error) 415eda14cbcSMatt Macy return (error); 416eda14cbcSMatt Macy 417eda14cbcSMatt Macy error = dmu_bonus_hold_by_dnode(dn, tag, dbp, DMU_READ_NO_PREFETCH); 418eda14cbcSMatt Macy dnode_rele(dn, FTAG); 419eda14cbcSMatt Macy 420eda14cbcSMatt Macy return (error); 421eda14cbcSMatt Macy } 422eda14cbcSMatt Macy 423eda14cbcSMatt Macy /* 424eda14cbcSMatt Macy * returns ENOENT, EIO, or 0. 425eda14cbcSMatt Macy * 426eda14cbcSMatt Macy * This interface will allocate a blank spill dbuf when a spill blk 427eda14cbcSMatt Macy * doesn't already exist on the dnode. 428eda14cbcSMatt Macy * 429eda14cbcSMatt Macy * if you only want to find an already existing spill db, then 430eda14cbcSMatt Macy * dmu_spill_hold_existing() should be used. 431eda14cbcSMatt Macy */ 432eda14cbcSMatt Macy int 433a0b956f5SMartin Matuska dmu_spill_hold_by_dnode(dnode_t *dn, uint32_t flags, const void *tag, 434a0b956f5SMartin Matuska dmu_buf_t **dbp) 435eda14cbcSMatt Macy { 436eda14cbcSMatt Macy dmu_buf_impl_t *db = NULL; 437eda14cbcSMatt Macy int err; 438eda14cbcSMatt Macy 439eda14cbcSMatt Macy if ((flags & DB_RF_HAVESTRUCT) == 0) 440eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_READER); 441eda14cbcSMatt Macy 442eda14cbcSMatt Macy db = dbuf_hold(dn, DMU_SPILL_BLKID, tag); 443eda14cbcSMatt Macy 444eda14cbcSMatt Macy if ((flags & DB_RF_HAVESTRUCT) == 0) 445eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock); 446eda14cbcSMatt Macy 447eda14cbcSMatt Macy if (db == NULL) { 448eda14cbcSMatt Macy *dbp = NULL; 449eda14cbcSMatt Macy return (SET_ERROR(EIO)); 450eda14cbcSMatt Macy } 451eda14cbcSMatt Macy err = dbuf_read(db, NULL, flags); 452eda14cbcSMatt Macy if (err == 0) 453eda14cbcSMatt Macy *dbp = &db->db; 454eda14cbcSMatt Macy else { 455eda14cbcSMatt Macy dbuf_rele(db, tag); 456eda14cbcSMatt Macy *dbp = NULL; 457eda14cbcSMatt Macy } 458eda14cbcSMatt Macy return (err); 459eda14cbcSMatt Macy } 460eda14cbcSMatt Macy 461eda14cbcSMatt Macy int 462a0b956f5SMartin Matuska dmu_spill_hold_existing(dmu_buf_t *bonus, const void *tag, dmu_buf_t **dbp) 463eda14cbcSMatt Macy { 464eda14cbcSMatt Macy dmu_buf_impl_t *db = (dmu_buf_impl_t *)bonus; 465eda14cbcSMatt Macy dnode_t *dn; 466eda14cbcSMatt Macy int err; 467eda14cbcSMatt Macy 468eda14cbcSMatt Macy DB_DNODE_ENTER(db); 469eda14cbcSMatt Macy dn = DB_DNODE(db); 470eda14cbcSMatt Macy 471eda14cbcSMatt Macy if (spa_version(dn->dn_objset->os_spa) < SPA_VERSION_SA) { 472eda14cbcSMatt Macy err = SET_ERROR(EINVAL); 473eda14cbcSMatt Macy } else { 474eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_READER); 475eda14cbcSMatt Macy 476eda14cbcSMatt Macy if (!dn->dn_have_spill) { 477eda14cbcSMatt Macy err = SET_ERROR(ENOENT); 478eda14cbcSMatt Macy } else { 479eda14cbcSMatt Macy err = dmu_spill_hold_by_dnode(dn, 480eda14cbcSMatt Macy DB_RF_HAVESTRUCT | DB_RF_CANFAIL, tag, dbp); 481eda14cbcSMatt Macy } 482eda14cbcSMatt Macy 483eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock); 484eda14cbcSMatt Macy } 485eda14cbcSMatt Macy 486eda14cbcSMatt Macy DB_DNODE_EXIT(db); 487eda14cbcSMatt Macy return (err); 488eda14cbcSMatt Macy } 489eda14cbcSMatt Macy 490eda14cbcSMatt Macy int 491a0b956f5SMartin Matuska dmu_spill_hold_by_bonus(dmu_buf_t *bonus, uint32_t flags, const void *tag, 492eda14cbcSMatt Macy dmu_buf_t **dbp) 493eda14cbcSMatt Macy { 494eda14cbcSMatt Macy dmu_buf_impl_t *db = (dmu_buf_impl_t *)bonus; 495eda14cbcSMatt Macy int err; 496eda14cbcSMatt Macy uint32_t db_flags = DB_RF_CANFAIL; 497eda14cbcSMatt Macy 498eda14cbcSMatt Macy if (flags & DMU_READ_NO_DECRYPT) 499eda14cbcSMatt Macy db_flags |= DB_RF_NO_DECRYPT; 500eda14cbcSMatt Macy 501eda14cbcSMatt Macy DB_DNODE_ENTER(db); 502ce4dcb97SMartin Matuska err = dmu_spill_hold_by_dnode(DB_DNODE(db), db_flags, tag, dbp); 503eda14cbcSMatt Macy DB_DNODE_EXIT(db); 504eda14cbcSMatt Macy 505eda14cbcSMatt Macy return (err); 506eda14cbcSMatt Macy } 507eda14cbcSMatt Macy 508eda14cbcSMatt Macy /* 509eda14cbcSMatt Macy * Note: longer-term, we should modify all of the dmu_buf_*() interfaces 510eda14cbcSMatt Macy * to take a held dnode rather than <os, object> -- the lookup is wasteful, 511eda14cbcSMatt Macy * and can induce severe lock contention when writing to several files 512eda14cbcSMatt Macy * whose dnodes are in the same block. 513eda14cbcSMatt Macy */ 514eda14cbcSMatt Macy int 515eda14cbcSMatt Macy dmu_buf_hold_array_by_dnode(dnode_t *dn, uint64_t offset, uint64_t length, 516a0b956f5SMartin Matuska boolean_t read, const void *tag, int *numbufsp, dmu_buf_t ***dbpp, 517a0b956f5SMartin Matuska uint32_t flags) 518eda14cbcSMatt Macy { 519eda14cbcSMatt Macy dmu_buf_t **dbp; 520f9693befSMartin Matuska zstream_t *zs = NULL; 521eda14cbcSMatt Macy uint64_t blkid, nblks, i; 522eda14cbcSMatt Macy uint32_t dbuf_flags; 523eda14cbcSMatt Macy int err; 5247877fdebSMatt Macy zio_t *zio = NULL; 525f9693befSMartin Matuska boolean_t missed = B_FALSE; 526eda14cbcSMatt Macy 5272a58b312SMartin Matuska ASSERT(!read || length <= DMU_MAX_ACCESS); 528eda14cbcSMatt Macy 529eda14cbcSMatt Macy /* 530eda14cbcSMatt Macy * Note: We directly notify the prefetch code of this read, so that 531eda14cbcSMatt Macy * we can tell it about the multi-block read. dbuf_read() only knows 532eda14cbcSMatt Macy * about the one block it is accessing. 533eda14cbcSMatt Macy */ 534eda14cbcSMatt Macy dbuf_flags = DB_RF_CANFAIL | DB_RF_NEVERWAIT | DB_RF_HAVESTRUCT | 535eda14cbcSMatt Macy DB_RF_NOPREFETCH; 536eda14cbcSMatt Macy 537271171e0SMartin Matuska if ((flags & DMU_READ_NO_DECRYPT) != 0) 538271171e0SMartin Matuska dbuf_flags |= DB_RF_NO_DECRYPT; 539271171e0SMartin Matuska 540eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_READER); 541eda14cbcSMatt Macy if (dn->dn_datablkshift) { 542eda14cbcSMatt Macy int blkshift = dn->dn_datablkshift; 543eda14cbcSMatt Macy nblks = (P2ROUNDUP(offset + length, 1ULL << blkshift) - 544aca928a5SMartin Matuska P2ALIGN_TYPED(offset, 1ULL << blkshift, uint64_t)) 545aca928a5SMartin Matuska >> blkshift; 546eda14cbcSMatt Macy } else { 547eda14cbcSMatt Macy if (offset + length > dn->dn_datablksz) { 548eda14cbcSMatt Macy zfs_panic_recover("zfs: accessing past end of object " 549eda14cbcSMatt Macy "%llx/%llx (size=%u access=%llu+%llu)", 550eda14cbcSMatt Macy (longlong_t)dn->dn_objset-> 551eda14cbcSMatt Macy os_dsl_dataset->ds_object, 552eda14cbcSMatt Macy (longlong_t)dn->dn_object, dn->dn_datablksz, 553eda14cbcSMatt Macy (longlong_t)offset, (longlong_t)length); 554eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock); 555eda14cbcSMatt Macy return (SET_ERROR(EIO)); 556eda14cbcSMatt Macy } 557eda14cbcSMatt Macy nblks = 1; 558eda14cbcSMatt Macy } 559eda14cbcSMatt Macy dbp = kmem_zalloc(sizeof (dmu_buf_t *) * nblks, KM_SLEEP); 560eda14cbcSMatt Macy 5617877fdebSMatt Macy if (read) 5627877fdebSMatt Macy zio = zio_root(dn->dn_objset->os_spa, NULL, NULL, 5637877fdebSMatt Macy ZIO_FLAG_CANFAIL); 564eda14cbcSMatt Macy blkid = dbuf_whichblock(dn, 0, offset); 565315ee00fSMartin Matuska if ((flags & DMU_READ_NO_PREFETCH) == 0) { 566f9693befSMartin Matuska /* 567f9693befSMartin Matuska * Prepare the zfetch before initiating the demand reads, so 568f9693befSMartin Matuska * that if multiple threads block on same indirect block, we 569f9693befSMartin Matuska * base predictions on the original less racy request order. 570f9693befSMartin Matuska */ 57115f0b8c3SMartin Matuska zs = dmu_zfetch_prepare(&dn->dn_zfetch, blkid, nblks, read, 57215f0b8c3SMartin Matuska B_TRUE); 573f9693befSMartin Matuska } 574eda14cbcSMatt Macy for (i = 0; i < nblks; i++) { 575eda14cbcSMatt Macy dmu_buf_impl_t *db = dbuf_hold(dn, blkid + i, tag); 576eda14cbcSMatt Macy if (db == NULL) { 5771719886fSMartin Matuska if (zs) { 5781719886fSMartin Matuska dmu_zfetch_run(&dn->dn_zfetch, zs, missed, 5791719886fSMartin Matuska B_TRUE); 5801719886fSMartin Matuska } 581eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock); 582eda14cbcSMatt Macy dmu_buf_rele_array(dbp, nblks, tag); 5837877fdebSMatt Macy if (read) 584eda14cbcSMatt Macy zio_nowait(zio); 585eda14cbcSMatt Macy return (SET_ERROR(EIO)); 586eda14cbcSMatt Macy } 587eda14cbcSMatt Macy 588f9693befSMartin Matuska /* 589f9693befSMartin Matuska * Initiate async demand data read. 590f9693befSMartin Matuska * We check the db_state after calling dbuf_read() because 591f9693befSMartin Matuska * (1) dbuf_read() may change the state to CACHED due to a 592f9693befSMartin Matuska * hit in the ARC, and (2) on a cache miss, a child will 593f9693befSMartin Matuska * have been added to "zio" but not yet completed, so the 594f9693befSMartin Matuska * state will not yet be CACHED. 595f9693befSMartin Matuska */ 596f9693befSMartin Matuska if (read) { 59715f0b8c3SMartin Matuska if (i == nblks - 1 && blkid + i < dn->dn_maxblkid && 59815f0b8c3SMartin Matuska offset + length < db->db.db_offset + 59915f0b8c3SMartin Matuska db->db.db_size) { 60015f0b8c3SMartin Matuska if (offset <= db->db.db_offset) 60115f0b8c3SMartin Matuska dbuf_flags |= DB_RF_PARTIAL_FIRST; 60215f0b8c3SMartin Matuska else 60315f0b8c3SMartin Matuska dbuf_flags |= DB_RF_PARTIAL_MORE; 60415f0b8c3SMartin Matuska } 605eda14cbcSMatt Macy (void) dbuf_read(db, zio, dbuf_flags); 606f9693befSMartin Matuska if (db->db_state != DB_CACHED) 607f9693befSMartin Matuska missed = B_TRUE; 608f9693befSMartin Matuska } 609eda14cbcSMatt Macy dbp[i] = &db->db; 610eda14cbcSMatt Macy } 611eda14cbcSMatt Macy 6127a7741afSMartin Matuska /* 6137a7741afSMartin Matuska * If we are doing O_DIRECT we still hold the dbufs, even for reads, 6147a7741afSMartin Matuska * but we do not issue any reads here. We do not want to account for 6157a7741afSMartin Matuska * writes in this case. 6167a7741afSMartin Matuska * 6177a7741afSMartin Matuska * O_DIRECT write/read accounting takes place in 6187a7741afSMartin Matuska * dmu_{write/read}_abd(). 6197a7741afSMartin Matuska */ 6207a7741afSMartin Matuska if (!read && ((flags & DMU_DIRECTIO) == 0)) 6217a7741afSMartin Matuska zfs_racct_write(dn->dn_objset->os_spa, length, nblks, flags); 622ba27dd8bSMartin Matuska 623f9693befSMartin Matuska if (zs) 6241719886fSMartin Matuska dmu_zfetch_run(&dn->dn_zfetch, zs, missed, B_TRUE); 625eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock); 626eda14cbcSMatt Macy 6277877fdebSMatt Macy if (read) { 6287877fdebSMatt Macy /* wait for async read i/o */ 629eda14cbcSMatt Macy err = zio_wait(zio); 630eda14cbcSMatt Macy if (err) { 631eda14cbcSMatt Macy dmu_buf_rele_array(dbp, nblks, tag); 632eda14cbcSMatt Macy return (err); 633eda14cbcSMatt Macy } 634eda14cbcSMatt Macy 635eda14cbcSMatt Macy /* wait for other io to complete */ 636eda14cbcSMatt Macy for (i = 0; i < nblks; i++) { 637eda14cbcSMatt Macy dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbp[i]; 638eda14cbcSMatt Macy mutex_enter(&db->db_mtx); 639eda14cbcSMatt Macy while (db->db_state == DB_READ || 640eda14cbcSMatt Macy db->db_state == DB_FILL) 641eda14cbcSMatt Macy cv_wait(&db->db_changed, &db->db_mtx); 642eda14cbcSMatt Macy if (db->db_state == DB_UNCACHED) 643eda14cbcSMatt Macy err = SET_ERROR(EIO); 644eda14cbcSMatt Macy mutex_exit(&db->db_mtx); 645eda14cbcSMatt Macy if (err) { 646eda14cbcSMatt Macy dmu_buf_rele_array(dbp, nblks, tag); 647eda14cbcSMatt Macy return (err); 648eda14cbcSMatt Macy } 649eda14cbcSMatt Macy } 650eda14cbcSMatt Macy } 651eda14cbcSMatt Macy 652eda14cbcSMatt Macy *numbufsp = nblks; 653eda14cbcSMatt Macy *dbpp = dbp; 654eda14cbcSMatt Macy return (0); 655eda14cbcSMatt Macy } 656eda14cbcSMatt Macy 6576ba2210eSMartin Matuska int 658eda14cbcSMatt Macy dmu_buf_hold_array(objset_t *os, uint64_t object, uint64_t offset, 659a0b956f5SMartin Matuska uint64_t length, int read, const void *tag, int *numbufsp, 660a0b956f5SMartin Matuska dmu_buf_t ***dbpp) 661eda14cbcSMatt Macy { 662eda14cbcSMatt Macy dnode_t *dn; 663eda14cbcSMatt Macy int err; 664eda14cbcSMatt Macy 665eda14cbcSMatt Macy err = dnode_hold(os, object, FTAG, &dn); 666eda14cbcSMatt Macy if (err) 667eda14cbcSMatt Macy return (err); 668eda14cbcSMatt Macy 669eda14cbcSMatt Macy err = dmu_buf_hold_array_by_dnode(dn, offset, length, read, tag, 670eda14cbcSMatt Macy numbufsp, dbpp, DMU_READ_PREFETCH); 671eda14cbcSMatt Macy 672eda14cbcSMatt Macy dnode_rele(dn, FTAG); 673eda14cbcSMatt Macy 674eda14cbcSMatt Macy return (err); 675eda14cbcSMatt Macy } 676eda14cbcSMatt Macy 677eda14cbcSMatt Macy int 678eda14cbcSMatt Macy dmu_buf_hold_array_by_bonus(dmu_buf_t *db_fake, uint64_t offset, 679a0b956f5SMartin Matuska uint64_t length, boolean_t read, const void *tag, int *numbufsp, 680eda14cbcSMatt Macy dmu_buf_t ***dbpp) 681eda14cbcSMatt Macy { 682eda14cbcSMatt Macy dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 683eda14cbcSMatt Macy int err; 684eda14cbcSMatt Macy 685eda14cbcSMatt Macy DB_DNODE_ENTER(db); 686ce4dcb97SMartin Matuska err = dmu_buf_hold_array_by_dnode(DB_DNODE(db), offset, length, read, 687ce4dcb97SMartin Matuska tag, numbufsp, dbpp, DMU_READ_PREFETCH); 688eda14cbcSMatt Macy DB_DNODE_EXIT(db); 689eda14cbcSMatt Macy 690eda14cbcSMatt Macy return (err); 691eda14cbcSMatt Macy } 692eda14cbcSMatt Macy 693eda14cbcSMatt Macy void 694a0b956f5SMartin Matuska dmu_buf_rele_array(dmu_buf_t **dbp_fake, int numbufs, const void *tag) 695eda14cbcSMatt Macy { 696eda14cbcSMatt Macy int i; 697eda14cbcSMatt Macy dmu_buf_impl_t **dbp = (dmu_buf_impl_t **)dbp_fake; 698eda14cbcSMatt Macy 699eda14cbcSMatt Macy if (numbufs == 0) 700eda14cbcSMatt Macy return; 701eda14cbcSMatt Macy 702eda14cbcSMatt Macy for (i = 0; i < numbufs; i++) { 703eda14cbcSMatt Macy if (dbp[i]) 704eda14cbcSMatt Macy dbuf_rele(dbp[i], tag); 705eda14cbcSMatt Macy } 706eda14cbcSMatt Macy 707eda14cbcSMatt Macy kmem_free(dbp, sizeof (dmu_buf_t *) * numbufs); 708eda14cbcSMatt Macy } 709eda14cbcSMatt Macy 710eda14cbcSMatt Macy /* 711315ee00fSMartin Matuska * Issue prefetch I/Os for the given blocks. If level is greater than 0, the 712eda14cbcSMatt Macy * indirect blocks prefetched will be those that point to the blocks containing 713315ee00fSMartin Matuska * the data starting at offset, and continuing to offset + len. If the range 714ce4dcb97SMartin Matuska * is too long, prefetch the first dmu_prefetch_max bytes as requested, while 715315ee00fSMartin Matuska * for the rest only a higher level, also fitting within dmu_prefetch_max. It 716315ee00fSMartin Matuska * should primarily help random reads, since for long sequential reads there is 717315ee00fSMartin Matuska * a speculative prefetcher. 718eda14cbcSMatt Macy * 719eda14cbcSMatt Macy * Note that if the indirect blocks above the blocks being prefetched are not 720315ee00fSMartin Matuska * in cache, they will be asynchronously read in. Dnode read by dnode_hold() 721315ee00fSMartin Matuska * is currently synchronous. 722eda14cbcSMatt Macy */ 723eda14cbcSMatt Macy void 724eda14cbcSMatt Macy dmu_prefetch(objset_t *os, uint64_t object, int64_t level, uint64_t offset, 725eda14cbcSMatt Macy uint64_t len, zio_priority_t pri) 726eda14cbcSMatt Macy { 727eda14cbcSMatt Macy dnode_t *dn; 728eda14cbcSMatt Macy 729315ee00fSMartin Matuska if (dmu_prefetch_max == 0 || len == 0) { 730315ee00fSMartin Matuska dmu_prefetch_dnode(os, object, pri); 731eda14cbcSMatt Macy return; 732eda14cbcSMatt Macy } 733eda14cbcSMatt Macy 734315ee00fSMartin Matuska if (dnode_hold(os, object, FTAG, &dn) != 0) 735eda14cbcSMatt Macy return; 736eda14cbcSMatt Macy 737783d3ff6SMartin Matuska dmu_prefetch_by_dnode(dn, level, offset, len, pri); 738783d3ff6SMartin Matuska 739783d3ff6SMartin Matuska dnode_rele(dn, FTAG); 740783d3ff6SMartin Matuska } 741783d3ff6SMartin Matuska 742783d3ff6SMartin Matuska void 743783d3ff6SMartin Matuska dmu_prefetch_by_dnode(dnode_t *dn, int64_t level, uint64_t offset, 744783d3ff6SMartin Matuska uint64_t len, zio_priority_t pri) 745783d3ff6SMartin Matuska { 746783d3ff6SMartin Matuska int64_t level2 = level; 747783d3ff6SMartin Matuska uint64_t start, end, start2, end2; 748783d3ff6SMartin Matuska 749eda14cbcSMatt Macy /* 750315ee00fSMartin Matuska * Depending on len we may do two prefetches: blocks [start, end) at 751315ee00fSMartin Matuska * level, and following blocks [start2, end2) at higher level2. 752eda14cbcSMatt Macy */ 753eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_READER); 754315ee00fSMartin Matuska if (dn->dn_datablkshift != 0) { 755315ee00fSMartin Matuska /* 756315ee00fSMartin Matuska * The object has multiple blocks. Calculate the full range 757315ee00fSMartin Matuska * of blocks [start, end2) and then split it into two parts, 758315ee00fSMartin Matuska * so that the first [start, end) fits into dmu_prefetch_max. 759315ee00fSMartin Matuska */ 760315ee00fSMartin Matuska start = dbuf_whichblock(dn, level, offset); 761315ee00fSMartin Matuska end2 = dbuf_whichblock(dn, level, offset + len - 1) + 1; 762315ee00fSMartin Matuska uint8_t ibs = dn->dn_indblkshift; 763315ee00fSMartin Matuska uint8_t bs = (level == 0) ? dn->dn_datablkshift : ibs; 764315ee00fSMartin Matuska uint_t limit = P2ROUNDUP(dmu_prefetch_max, 1 << bs) >> bs; 765315ee00fSMartin Matuska start2 = end = MIN(end2, start + limit); 766315ee00fSMartin Matuska 767315ee00fSMartin Matuska /* 768315ee00fSMartin Matuska * Find level2 where [start2, end2) fits into dmu_prefetch_max. 769315ee00fSMartin Matuska */ 770315ee00fSMartin Matuska uint8_t ibps = ibs - SPA_BLKPTRSHIFT; 771315ee00fSMartin Matuska limit = P2ROUNDUP(dmu_prefetch_max, 1 << ibs) >> ibs; 772315ee00fSMartin Matuska do { 773315ee00fSMartin Matuska level2++; 774315ee00fSMartin Matuska start2 = P2ROUNDUP(start2, 1 << ibps) >> ibps; 775315ee00fSMartin Matuska end2 = P2ROUNDUP(end2, 1 << ibps) >> ibps; 776315ee00fSMartin Matuska } while (end2 - start2 > limit); 777eda14cbcSMatt Macy } else { 778315ee00fSMartin Matuska /* There is only one block. Prefetch it or nothing. */ 779315ee00fSMartin Matuska start = start2 = end2 = 0; 780315ee00fSMartin Matuska end = start + (level == 0 && offset < dn->dn_datablksz); 781eda14cbcSMatt Macy } 782eda14cbcSMatt Macy 783315ee00fSMartin Matuska for (uint64_t i = start; i < end; i++) 784315ee00fSMartin Matuska dbuf_prefetch(dn, level, i, pri, 0); 785315ee00fSMartin Matuska for (uint64_t i = start2; i < end2; i++) 786315ee00fSMartin Matuska dbuf_prefetch(dn, level2, i, pri, 0); 787eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock); 788eda14cbcSMatt Macy } 789eda14cbcSMatt Macy 790ce4dcb97SMartin Matuska typedef struct { 791ce4dcb97SMartin Matuska kmutex_t dpa_lock; 792ce4dcb97SMartin Matuska kcondvar_t dpa_cv; 793ce4dcb97SMartin Matuska uint64_t dpa_pending_io; 794ce4dcb97SMartin Matuska } dmu_prefetch_arg_t; 795ce4dcb97SMartin Matuska 796ce4dcb97SMartin Matuska static void 797ce4dcb97SMartin Matuska dmu_prefetch_done(void *arg, uint64_t level, uint64_t blkid, boolean_t issued) 798ce4dcb97SMartin Matuska { 799ce4dcb97SMartin Matuska (void) level; (void) blkid; (void)issued; 800ce4dcb97SMartin Matuska dmu_prefetch_arg_t *dpa = arg; 801ce4dcb97SMartin Matuska 802ce4dcb97SMartin Matuska ASSERT0(level); 803ce4dcb97SMartin Matuska 804ce4dcb97SMartin Matuska mutex_enter(&dpa->dpa_lock); 805ce4dcb97SMartin Matuska ASSERT3U(dpa->dpa_pending_io, >, 0); 806ce4dcb97SMartin Matuska if (--dpa->dpa_pending_io == 0) 807ce4dcb97SMartin Matuska cv_broadcast(&dpa->dpa_cv); 808ce4dcb97SMartin Matuska mutex_exit(&dpa->dpa_lock); 809ce4dcb97SMartin Matuska } 810ce4dcb97SMartin Matuska 811ce4dcb97SMartin Matuska static void 812ce4dcb97SMartin Matuska dmu_prefetch_wait_by_dnode(dnode_t *dn, uint64_t offset, uint64_t len) 813ce4dcb97SMartin Matuska { 814ce4dcb97SMartin Matuska dmu_prefetch_arg_t dpa; 815ce4dcb97SMartin Matuska 816ce4dcb97SMartin Matuska mutex_init(&dpa.dpa_lock, NULL, MUTEX_DEFAULT, NULL); 817ce4dcb97SMartin Matuska cv_init(&dpa.dpa_cv, NULL, CV_DEFAULT, NULL); 818ce4dcb97SMartin Matuska 819ce4dcb97SMartin Matuska rw_enter(&dn->dn_struct_rwlock, RW_READER); 820ce4dcb97SMartin Matuska 821ce4dcb97SMartin Matuska uint64_t start = dbuf_whichblock(dn, 0, offset); 822ce4dcb97SMartin Matuska uint64_t end = dbuf_whichblock(dn, 0, offset + len - 1) + 1; 823ce4dcb97SMartin Matuska dpa.dpa_pending_io = end - start; 824ce4dcb97SMartin Matuska 825ce4dcb97SMartin Matuska for (uint64_t blk = start; blk < end; blk++) { 826ce4dcb97SMartin Matuska (void) dbuf_prefetch_impl(dn, 0, blk, ZIO_PRIORITY_ASYNC_READ, 827ce4dcb97SMartin Matuska 0, dmu_prefetch_done, &dpa); 828ce4dcb97SMartin Matuska } 829ce4dcb97SMartin Matuska 830ce4dcb97SMartin Matuska rw_exit(&dn->dn_struct_rwlock); 831ce4dcb97SMartin Matuska 832ce4dcb97SMartin Matuska /* wait for prefetch L0 reads to finish */ 833ce4dcb97SMartin Matuska mutex_enter(&dpa.dpa_lock); 834ce4dcb97SMartin Matuska while (dpa.dpa_pending_io > 0) { 835ce4dcb97SMartin Matuska cv_wait(&dpa.dpa_cv, &dpa.dpa_lock); 836ce4dcb97SMartin Matuska 837ce4dcb97SMartin Matuska } 838ce4dcb97SMartin Matuska mutex_exit(&dpa.dpa_lock); 839ce4dcb97SMartin Matuska 840ce4dcb97SMartin Matuska mutex_destroy(&dpa.dpa_lock); 841ce4dcb97SMartin Matuska cv_destroy(&dpa.dpa_cv); 842ce4dcb97SMartin Matuska } 843ce4dcb97SMartin Matuska 844ce4dcb97SMartin Matuska /* 845ce4dcb97SMartin Matuska * Issue prefetch I/Os for the given L0 block range and wait for the I/O 846ce4dcb97SMartin Matuska * to complete. This does not enforce dmu_prefetch_max and will prefetch 847ce4dcb97SMartin Matuska * the entire range. The blocks are read from disk into the ARC but no 848ce4dcb97SMartin Matuska * decompression occurs (i.e., the dbuf cache is not required). 849ce4dcb97SMartin Matuska */ 850ce4dcb97SMartin Matuska int 851ce4dcb97SMartin Matuska dmu_prefetch_wait(objset_t *os, uint64_t object, uint64_t offset, uint64_t size) 852ce4dcb97SMartin Matuska { 853ce4dcb97SMartin Matuska dnode_t *dn; 854ce4dcb97SMartin Matuska int err = 0; 855ce4dcb97SMartin Matuska 856ce4dcb97SMartin Matuska err = dnode_hold(os, object, FTAG, &dn); 857ce4dcb97SMartin Matuska if (err != 0) 858ce4dcb97SMartin Matuska return (err); 859ce4dcb97SMartin Matuska 860ce4dcb97SMartin Matuska /* 861ce4dcb97SMartin Matuska * Chunk the requests (16 indirects worth) so that we can be interrupted 862ce4dcb97SMartin Matuska */ 863ce4dcb97SMartin Matuska uint64_t chunksize; 864ce4dcb97SMartin Matuska if (dn->dn_indblkshift) { 865ce4dcb97SMartin Matuska uint64_t nbps = bp_span_in_blocks(dn->dn_indblkshift, 1); 866ce4dcb97SMartin Matuska chunksize = (nbps * 16) << dn->dn_datablkshift; 867ce4dcb97SMartin Matuska } else { 868ce4dcb97SMartin Matuska chunksize = dn->dn_datablksz; 869ce4dcb97SMartin Matuska } 870ce4dcb97SMartin Matuska 871ce4dcb97SMartin Matuska while (size > 0) { 872ce4dcb97SMartin Matuska uint64_t mylen = MIN(size, chunksize); 873ce4dcb97SMartin Matuska 874ce4dcb97SMartin Matuska dmu_prefetch_wait_by_dnode(dn, offset, mylen); 875ce4dcb97SMartin Matuska 876ce4dcb97SMartin Matuska offset += mylen; 877ce4dcb97SMartin Matuska size -= mylen; 878ce4dcb97SMartin Matuska 879ce4dcb97SMartin Matuska if (issig()) { 880ce4dcb97SMartin Matuska err = SET_ERROR(EINTR); 881ce4dcb97SMartin Matuska break; 882ce4dcb97SMartin Matuska } 883ce4dcb97SMartin Matuska } 884ce4dcb97SMartin Matuska 885ce4dcb97SMartin Matuska dnode_rele(dn, FTAG); 886ce4dcb97SMartin Matuska 887ce4dcb97SMartin Matuska return (err); 888ce4dcb97SMartin Matuska } 889ce4dcb97SMartin Matuska 890eda14cbcSMatt Macy /* 891315ee00fSMartin Matuska * Issue prefetch I/Os for the given object's dnode. 892315ee00fSMartin Matuska */ 893315ee00fSMartin Matuska void 894315ee00fSMartin Matuska dmu_prefetch_dnode(objset_t *os, uint64_t object, zio_priority_t pri) 895315ee00fSMartin Matuska { 896315ee00fSMartin Matuska if (object == 0 || object >= DN_MAX_OBJECT) 897315ee00fSMartin Matuska return; 898315ee00fSMartin Matuska 899315ee00fSMartin Matuska dnode_t *dn = DMU_META_DNODE(os); 900315ee00fSMartin Matuska rw_enter(&dn->dn_struct_rwlock, RW_READER); 901315ee00fSMartin Matuska uint64_t blkid = dbuf_whichblock(dn, 0, object * sizeof (dnode_phys_t)); 902315ee00fSMartin Matuska dbuf_prefetch(dn, 0, blkid, pri, 0); 903315ee00fSMartin Matuska rw_exit(&dn->dn_struct_rwlock); 904315ee00fSMartin Matuska } 905315ee00fSMartin Matuska 906315ee00fSMartin Matuska /* 907eda14cbcSMatt Macy * Get the next "chunk" of file data to free. We traverse the file from 9087a7741afSMartin Matuska * the end so that the file gets shorter over time (if we crash in the 909eda14cbcSMatt Macy * middle, this will leave us in a better state). We find allocated file 910eda14cbcSMatt Macy * data by simply searching the allocated level 1 indirects. 911eda14cbcSMatt Macy * 912eda14cbcSMatt Macy * On input, *start should be the first offset that does not need to be 913eda14cbcSMatt Macy * freed (e.g. "offset + length"). On return, *start will be the first 914eda14cbcSMatt Macy * offset that should be freed and l1blks is set to the number of level 1 915eda14cbcSMatt Macy * indirect blocks found within the chunk. 916eda14cbcSMatt Macy */ 917eda14cbcSMatt Macy static int 918eda14cbcSMatt Macy get_next_chunk(dnode_t *dn, uint64_t *start, uint64_t minimum, uint64_t *l1blks) 919eda14cbcSMatt Macy { 920eda14cbcSMatt Macy uint64_t blks; 921eda14cbcSMatt Macy uint64_t maxblks = DMU_MAX_ACCESS >> (dn->dn_indblkshift + 1); 922eda14cbcSMatt Macy /* bytes of data covered by a level-1 indirect block */ 923eda14cbcSMatt Macy uint64_t iblkrange = (uint64_t)dn->dn_datablksz * 924eda14cbcSMatt Macy EPB(dn->dn_indblkshift, SPA_BLKPTRSHIFT); 925eda14cbcSMatt Macy 926eda14cbcSMatt Macy ASSERT3U(minimum, <=, *start); 927eda14cbcSMatt Macy 928ce4dcb97SMartin Matuska /* dn_nlevels == 1 means we don't have any L1 blocks */ 929ce4dcb97SMartin Matuska if (dn->dn_nlevels <= 1) { 930ce4dcb97SMartin Matuska *l1blks = 0; 931ce4dcb97SMartin Matuska *start = minimum; 932ce4dcb97SMartin Matuska return (0); 933ce4dcb97SMartin Matuska } 934ce4dcb97SMartin Matuska 935eda14cbcSMatt Macy /* 936eda14cbcSMatt Macy * Check if we can free the entire range assuming that all of the 937eda14cbcSMatt Macy * L1 blocks in this range have data. If we can, we use this 938eda14cbcSMatt Macy * worst case value as an estimate so we can avoid having to look 939eda14cbcSMatt Macy * at the object's actual data. 940eda14cbcSMatt Macy */ 941eda14cbcSMatt Macy uint64_t total_l1blks = 942eda14cbcSMatt Macy (roundup(*start, iblkrange) - (minimum / iblkrange * iblkrange)) / 943eda14cbcSMatt Macy iblkrange; 944eda14cbcSMatt Macy if (total_l1blks <= maxblks) { 945eda14cbcSMatt Macy *l1blks = total_l1blks; 946eda14cbcSMatt Macy *start = minimum; 947eda14cbcSMatt Macy return (0); 948eda14cbcSMatt Macy } 949eda14cbcSMatt Macy ASSERT(ISP2(iblkrange)); 950eda14cbcSMatt Macy 951eda14cbcSMatt Macy for (blks = 0; *start > minimum && blks < maxblks; blks++) { 952eda14cbcSMatt Macy int err; 953eda14cbcSMatt Macy 954eda14cbcSMatt Macy /* 955eda14cbcSMatt Macy * dnode_next_offset(BACKWARDS) will find an allocated L1 956eda14cbcSMatt Macy * indirect block at or before the input offset. We must 957eda14cbcSMatt Macy * decrement *start so that it is at the end of the region 958eda14cbcSMatt Macy * to search. 959eda14cbcSMatt Macy */ 960eda14cbcSMatt Macy (*start)--; 961eda14cbcSMatt Macy 962eda14cbcSMatt Macy err = dnode_next_offset(dn, 963eda14cbcSMatt Macy DNODE_FIND_BACKWARDS, start, 2, 1, 0); 964eda14cbcSMatt Macy 965eda14cbcSMatt Macy /* if there are no indirect blocks before start, we are done */ 966eda14cbcSMatt Macy if (err == ESRCH) { 967eda14cbcSMatt Macy *start = minimum; 968eda14cbcSMatt Macy break; 969eda14cbcSMatt Macy } else if (err != 0) { 970eda14cbcSMatt Macy *l1blks = blks; 971eda14cbcSMatt Macy return (err); 972eda14cbcSMatt Macy } 973eda14cbcSMatt Macy 974eda14cbcSMatt Macy /* set start to the beginning of this L1 indirect */ 975aca928a5SMartin Matuska *start = P2ALIGN_TYPED(*start, iblkrange, uint64_t); 976eda14cbcSMatt Macy } 977eda14cbcSMatt Macy if (*start < minimum) 978eda14cbcSMatt Macy *start = minimum; 979eda14cbcSMatt Macy *l1blks = blks; 980eda14cbcSMatt Macy 981eda14cbcSMatt Macy return (0); 982eda14cbcSMatt Macy } 983eda14cbcSMatt Macy 984eda14cbcSMatt Macy /* 985eda14cbcSMatt Macy * If this objset is of type OST_ZFS return true if vfs's unmounted flag is set, 986eda14cbcSMatt Macy * otherwise return false. 987eda14cbcSMatt Macy * Used below in dmu_free_long_range_impl() to enable abort when unmounting 988eda14cbcSMatt Macy */ 989eda14cbcSMatt Macy static boolean_t 990eda14cbcSMatt Macy dmu_objset_zfs_unmounting(objset_t *os) 991eda14cbcSMatt Macy { 992eda14cbcSMatt Macy #ifdef _KERNEL 993eda14cbcSMatt Macy if (dmu_objset_type(os) == DMU_OST_ZFS) 994eda14cbcSMatt Macy return (zfs_get_vfs_flag_unmounted(os)); 995e92ffd9bSMartin Matuska #else 996e92ffd9bSMartin Matuska (void) os; 997eda14cbcSMatt Macy #endif 998eda14cbcSMatt Macy return (B_FALSE); 999eda14cbcSMatt Macy } 1000eda14cbcSMatt Macy 1001eda14cbcSMatt Macy static int 1002eda14cbcSMatt Macy dmu_free_long_range_impl(objset_t *os, dnode_t *dn, uint64_t offset, 1003eda14cbcSMatt Macy uint64_t length) 1004eda14cbcSMatt Macy { 1005eda14cbcSMatt Macy uint64_t object_size; 1006eda14cbcSMatt Macy int err; 1007eda14cbcSMatt Macy uint64_t dirty_frees_threshold; 1008eda14cbcSMatt Macy dsl_pool_t *dp = dmu_objset_pool(os); 1009eda14cbcSMatt Macy 1010eda14cbcSMatt Macy if (dn == NULL) 1011eda14cbcSMatt Macy return (SET_ERROR(EINVAL)); 1012eda14cbcSMatt Macy 1013eda14cbcSMatt Macy object_size = (dn->dn_maxblkid + 1) * dn->dn_datablksz; 1014eda14cbcSMatt Macy if (offset >= object_size) 1015eda14cbcSMatt Macy return (0); 1016eda14cbcSMatt Macy 1017eda14cbcSMatt Macy if (zfs_per_txg_dirty_frees_percent <= 100) 1018eda14cbcSMatt Macy dirty_frees_threshold = 1019eda14cbcSMatt Macy zfs_per_txg_dirty_frees_percent * zfs_dirty_data_max / 100; 1020eda14cbcSMatt Macy else 1021eda14cbcSMatt Macy dirty_frees_threshold = zfs_dirty_data_max / 20; 1022eda14cbcSMatt Macy 1023eda14cbcSMatt Macy if (length == DMU_OBJECT_END || offset + length > object_size) 1024eda14cbcSMatt Macy length = object_size - offset; 1025eda14cbcSMatt Macy 1026eda14cbcSMatt Macy while (length != 0) { 1027eda14cbcSMatt Macy uint64_t chunk_end, chunk_begin, chunk_len; 1028eda14cbcSMatt Macy uint64_t l1blks; 1029eda14cbcSMatt Macy dmu_tx_t *tx; 1030eda14cbcSMatt Macy 1031eda14cbcSMatt Macy if (dmu_objset_zfs_unmounting(dn->dn_objset)) 1032eda14cbcSMatt Macy return (SET_ERROR(EINTR)); 1033eda14cbcSMatt Macy 1034eda14cbcSMatt Macy chunk_end = chunk_begin = offset + length; 1035eda14cbcSMatt Macy 1036eda14cbcSMatt Macy /* move chunk_begin backwards to the beginning of this chunk */ 1037eda14cbcSMatt Macy err = get_next_chunk(dn, &chunk_begin, offset, &l1blks); 1038eda14cbcSMatt Macy if (err) 1039eda14cbcSMatt Macy return (err); 1040eda14cbcSMatt Macy ASSERT3U(chunk_begin, >=, offset); 1041eda14cbcSMatt Macy ASSERT3U(chunk_begin, <=, chunk_end); 1042eda14cbcSMatt Macy 1043eda14cbcSMatt Macy chunk_len = chunk_end - chunk_begin; 1044eda14cbcSMatt Macy 1045eda14cbcSMatt Macy tx = dmu_tx_create(os); 1046eda14cbcSMatt Macy dmu_tx_hold_free(tx, dn->dn_object, chunk_begin, chunk_len); 1047eda14cbcSMatt Macy 1048eda14cbcSMatt Macy /* 1049eda14cbcSMatt Macy * Mark this transaction as typically resulting in a net 1050eda14cbcSMatt Macy * reduction in space used. 1051eda14cbcSMatt Macy */ 1052eda14cbcSMatt Macy dmu_tx_mark_netfree(tx); 1053eda14cbcSMatt Macy err = dmu_tx_assign(tx, TXG_WAIT); 1054eda14cbcSMatt Macy if (err) { 1055eda14cbcSMatt Macy dmu_tx_abort(tx); 1056eda14cbcSMatt Macy return (err); 1057eda14cbcSMatt Macy } 1058eda14cbcSMatt Macy 1059eda14cbcSMatt Macy uint64_t txg = dmu_tx_get_txg(tx); 1060eda14cbcSMatt Macy 1061eda14cbcSMatt Macy mutex_enter(&dp->dp_lock); 1062eda14cbcSMatt Macy uint64_t long_free_dirty = 1063eda14cbcSMatt Macy dp->dp_long_free_dirty_pertxg[txg & TXG_MASK]; 1064eda14cbcSMatt Macy mutex_exit(&dp->dp_lock); 1065eda14cbcSMatt Macy 1066eda14cbcSMatt Macy /* 1067eda14cbcSMatt Macy * To avoid filling up a TXG with just frees, wait for 1068eda14cbcSMatt Macy * the next TXG to open before freeing more chunks if 1069eda14cbcSMatt Macy * we have reached the threshold of frees. 1070eda14cbcSMatt Macy */ 1071eda14cbcSMatt Macy if (dirty_frees_threshold != 0 && 1072eda14cbcSMatt Macy long_free_dirty >= dirty_frees_threshold) { 1073eda14cbcSMatt Macy DMU_TX_STAT_BUMP(dmu_tx_dirty_frees_delay); 1074eda14cbcSMatt Macy dmu_tx_commit(tx); 1075eda14cbcSMatt Macy txg_wait_open(dp, 0, B_TRUE); 1076eda14cbcSMatt Macy continue; 1077eda14cbcSMatt Macy } 1078eda14cbcSMatt Macy 1079eda14cbcSMatt Macy /* 1080eda14cbcSMatt Macy * In order to prevent unnecessary write throttling, for each 1081eda14cbcSMatt Macy * TXG, we track the cumulative size of L1 blocks being dirtied 1082eda14cbcSMatt Macy * in dnode_free_range() below. We compare this number to a 1083eda14cbcSMatt Macy * tunable threshold, past which we prevent new L1 dirty freeing 1084eda14cbcSMatt Macy * blocks from being added into the open TXG. See 1085eda14cbcSMatt Macy * dmu_free_long_range_impl() for details. The threshold 1086eda14cbcSMatt Macy * prevents write throttle activation due to dirty freeing L1 1087eda14cbcSMatt Macy * blocks taking up a large percentage of zfs_dirty_data_max. 1088eda14cbcSMatt Macy */ 1089eda14cbcSMatt Macy mutex_enter(&dp->dp_lock); 1090eda14cbcSMatt Macy dp->dp_long_free_dirty_pertxg[txg & TXG_MASK] += 1091eda14cbcSMatt Macy l1blks << dn->dn_indblkshift; 1092eda14cbcSMatt Macy mutex_exit(&dp->dp_lock); 1093eda14cbcSMatt Macy DTRACE_PROBE3(free__long__range, 1094eda14cbcSMatt Macy uint64_t, long_free_dirty, uint64_t, chunk_len, 1095eda14cbcSMatt Macy uint64_t, txg); 1096eda14cbcSMatt Macy dnode_free_range(dn, chunk_begin, chunk_len, tx); 1097eda14cbcSMatt Macy 1098eda14cbcSMatt Macy dmu_tx_commit(tx); 1099eda14cbcSMatt Macy 1100eda14cbcSMatt Macy length -= chunk_len; 1101eda14cbcSMatt Macy } 1102eda14cbcSMatt Macy return (0); 1103eda14cbcSMatt Macy } 1104eda14cbcSMatt Macy 1105eda14cbcSMatt Macy int 1106eda14cbcSMatt Macy dmu_free_long_range(objset_t *os, uint64_t object, 1107eda14cbcSMatt Macy uint64_t offset, uint64_t length) 1108eda14cbcSMatt Macy { 1109eda14cbcSMatt Macy dnode_t *dn; 1110eda14cbcSMatt Macy int err; 1111eda14cbcSMatt Macy 1112eda14cbcSMatt Macy err = dnode_hold(os, object, FTAG, &dn); 1113eda14cbcSMatt Macy if (err != 0) 1114eda14cbcSMatt Macy return (err); 1115eda14cbcSMatt Macy err = dmu_free_long_range_impl(os, dn, offset, length); 1116eda14cbcSMatt Macy 1117eda14cbcSMatt Macy /* 1118eda14cbcSMatt Macy * It is important to zero out the maxblkid when freeing the entire 1119eda14cbcSMatt Macy * file, so that (a) subsequent calls to dmu_free_long_range_impl() 1120eda14cbcSMatt Macy * will take the fast path, and (b) dnode_reallocate() can verify 1121eda14cbcSMatt Macy * that the entire file has been freed. 1122eda14cbcSMatt Macy */ 1123eda14cbcSMatt Macy if (err == 0 && offset == 0 && length == DMU_OBJECT_END) 1124eda14cbcSMatt Macy dn->dn_maxblkid = 0; 1125eda14cbcSMatt Macy 1126eda14cbcSMatt Macy dnode_rele(dn, FTAG); 1127eda14cbcSMatt Macy return (err); 1128eda14cbcSMatt Macy } 1129eda14cbcSMatt Macy 1130eda14cbcSMatt Macy int 1131eda14cbcSMatt Macy dmu_free_long_object(objset_t *os, uint64_t object) 1132eda14cbcSMatt Macy { 1133eda14cbcSMatt Macy dmu_tx_t *tx; 1134eda14cbcSMatt Macy int err; 1135eda14cbcSMatt Macy 1136eda14cbcSMatt Macy err = dmu_free_long_range(os, object, 0, DMU_OBJECT_END); 1137eda14cbcSMatt Macy if (err != 0) 1138eda14cbcSMatt Macy return (err); 1139eda14cbcSMatt Macy 1140eda14cbcSMatt Macy tx = dmu_tx_create(os); 1141eda14cbcSMatt Macy dmu_tx_hold_bonus(tx, object); 1142eda14cbcSMatt Macy dmu_tx_hold_free(tx, object, 0, DMU_OBJECT_END); 1143eda14cbcSMatt Macy dmu_tx_mark_netfree(tx); 1144eda14cbcSMatt Macy err = dmu_tx_assign(tx, TXG_WAIT); 1145eda14cbcSMatt Macy if (err == 0) { 1146eda14cbcSMatt Macy err = dmu_object_free(os, object, tx); 1147eda14cbcSMatt Macy dmu_tx_commit(tx); 1148eda14cbcSMatt Macy } else { 1149eda14cbcSMatt Macy dmu_tx_abort(tx); 1150eda14cbcSMatt Macy } 1151eda14cbcSMatt Macy 1152eda14cbcSMatt Macy return (err); 1153eda14cbcSMatt Macy } 1154eda14cbcSMatt Macy 1155eda14cbcSMatt Macy int 1156eda14cbcSMatt Macy dmu_free_range(objset_t *os, uint64_t object, uint64_t offset, 1157eda14cbcSMatt Macy uint64_t size, dmu_tx_t *tx) 1158eda14cbcSMatt Macy { 1159eda14cbcSMatt Macy dnode_t *dn; 1160eda14cbcSMatt Macy int err = dnode_hold(os, object, FTAG, &dn); 1161eda14cbcSMatt Macy if (err) 1162eda14cbcSMatt Macy return (err); 1163eda14cbcSMatt Macy ASSERT(offset < UINT64_MAX); 1164eda14cbcSMatt Macy ASSERT(size == DMU_OBJECT_END || size <= UINT64_MAX - offset); 1165eda14cbcSMatt Macy dnode_free_range(dn, offset, size, tx); 1166eda14cbcSMatt Macy dnode_rele(dn, FTAG); 1167eda14cbcSMatt Macy return (0); 1168eda14cbcSMatt Macy } 1169eda14cbcSMatt Macy 1170eda14cbcSMatt Macy static int 1171eda14cbcSMatt Macy dmu_read_impl(dnode_t *dn, uint64_t offset, uint64_t size, 1172eda14cbcSMatt Macy void *buf, uint32_t flags) 1173eda14cbcSMatt Macy { 1174eda14cbcSMatt Macy dmu_buf_t **dbp; 1175eda14cbcSMatt Macy int numbufs, err = 0; 1176eda14cbcSMatt Macy 1177eda14cbcSMatt Macy /* 1178eda14cbcSMatt Macy * Deal with odd block sizes, where there can't be data past the first 1179eda14cbcSMatt Macy * block. If we ever do the tail block optimization, we will need to 1180eda14cbcSMatt Macy * handle that here as well. 1181eda14cbcSMatt Macy */ 1182eda14cbcSMatt Macy if (dn->dn_maxblkid == 0) { 1183eda14cbcSMatt Macy uint64_t newsz = offset > dn->dn_datablksz ? 0 : 1184eda14cbcSMatt Macy MIN(size, dn->dn_datablksz - offset); 1185da5137abSMartin Matuska memset((char *)buf + newsz, 0, size - newsz); 1186eda14cbcSMatt Macy size = newsz; 1187eda14cbcSMatt Macy } 1188eda14cbcSMatt Macy 11897a7741afSMartin Matuska if (size == 0) 11907a7741afSMartin Matuska return (0); 11917a7741afSMartin Matuska 11927a7741afSMartin Matuska /* Allow Direct I/O when requested and properly aligned */ 11937a7741afSMartin Matuska if ((flags & DMU_DIRECTIO) && zfs_dio_page_aligned(buf) && 11947a7741afSMartin Matuska zfs_dio_aligned(offset, size, PAGESIZE)) { 11957a7741afSMartin Matuska abd_t *data = abd_get_from_buf(buf, size); 11967a7741afSMartin Matuska err = dmu_read_abd(dn, offset, size, data, flags); 11977a7741afSMartin Matuska abd_free(data); 11987a7741afSMartin Matuska return (err); 11997a7741afSMartin Matuska } 12007a7741afSMartin Matuska 1201eda14cbcSMatt Macy while (size > 0) { 1202eda14cbcSMatt Macy uint64_t mylen = MIN(size, DMU_MAX_ACCESS / 2); 1203eda14cbcSMatt Macy int i; 1204eda14cbcSMatt Macy 1205eda14cbcSMatt Macy /* 1206eda14cbcSMatt Macy * NB: we could do this block-at-a-time, but it's nice 1207eda14cbcSMatt Macy * to be reading in parallel. 1208eda14cbcSMatt Macy */ 1209eda14cbcSMatt Macy err = dmu_buf_hold_array_by_dnode(dn, offset, mylen, 1210eda14cbcSMatt Macy TRUE, FTAG, &numbufs, &dbp, flags); 1211eda14cbcSMatt Macy if (err) 1212eda14cbcSMatt Macy break; 1213eda14cbcSMatt Macy 1214eda14cbcSMatt Macy for (i = 0; i < numbufs; i++) { 1215eda14cbcSMatt Macy uint64_t tocpy; 1216eda14cbcSMatt Macy int64_t bufoff; 1217eda14cbcSMatt Macy dmu_buf_t *db = dbp[i]; 1218eda14cbcSMatt Macy 1219eda14cbcSMatt Macy ASSERT(size > 0); 1220eda14cbcSMatt Macy 1221eda14cbcSMatt Macy bufoff = offset - db->db_offset; 1222eda14cbcSMatt Macy tocpy = MIN(db->db_size - bufoff, size); 1223eda14cbcSMatt Macy 1224*17aab35aSMartin Matuska ASSERT(db->db_data != NULL); 1225eda14cbcSMatt Macy (void) memcpy(buf, (char *)db->db_data + bufoff, tocpy); 1226eda14cbcSMatt Macy 1227eda14cbcSMatt Macy offset += tocpy; 1228eda14cbcSMatt Macy size -= tocpy; 1229eda14cbcSMatt Macy buf = (char *)buf + tocpy; 1230eda14cbcSMatt Macy } 1231eda14cbcSMatt Macy dmu_buf_rele_array(dbp, numbufs, FTAG); 1232eda14cbcSMatt Macy } 1233eda14cbcSMatt Macy return (err); 1234eda14cbcSMatt Macy } 1235eda14cbcSMatt Macy 1236eda14cbcSMatt Macy int 1237eda14cbcSMatt Macy dmu_read(objset_t *os, uint64_t object, uint64_t offset, uint64_t size, 1238eda14cbcSMatt Macy void *buf, uint32_t flags) 1239eda14cbcSMatt Macy { 1240eda14cbcSMatt Macy dnode_t *dn; 1241eda14cbcSMatt Macy int err; 1242eda14cbcSMatt Macy 1243eda14cbcSMatt Macy err = dnode_hold(os, object, FTAG, &dn); 1244eda14cbcSMatt Macy if (err != 0) 1245eda14cbcSMatt Macy return (err); 1246eda14cbcSMatt Macy 1247eda14cbcSMatt Macy err = dmu_read_impl(dn, offset, size, buf, flags); 1248eda14cbcSMatt Macy dnode_rele(dn, FTAG); 1249eda14cbcSMatt Macy return (err); 1250eda14cbcSMatt Macy } 1251eda14cbcSMatt Macy 1252eda14cbcSMatt Macy int 1253eda14cbcSMatt Macy dmu_read_by_dnode(dnode_t *dn, uint64_t offset, uint64_t size, void *buf, 1254eda14cbcSMatt Macy uint32_t flags) 1255eda14cbcSMatt Macy { 1256eda14cbcSMatt Macy return (dmu_read_impl(dn, offset, size, buf, flags)); 1257eda14cbcSMatt Macy } 1258eda14cbcSMatt Macy 1259eda14cbcSMatt Macy static void 1260eda14cbcSMatt Macy dmu_write_impl(dmu_buf_t **dbp, int numbufs, uint64_t offset, uint64_t size, 1261eda14cbcSMatt Macy const void *buf, dmu_tx_t *tx) 1262eda14cbcSMatt Macy { 1263eda14cbcSMatt Macy int i; 1264eda14cbcSMatt Macy 1265eda14cbcSMatt Macy for (i = 0; i < numbufs; i++) { 1266eda14cbcSMatt Macy uint64_t tocpy; 1267eda14cbcSMatt Macy int64_t bufoff; 1268eda14cbcSMatt Macy dmu_buf_t *db = dbp[i]; 1269eda14cbcSMatt Macy 1270eda14cbcSMatt Macy ASSERT(size > 0); 1271eda14cbcSMatt Macy 1272eda14cbcSMatt Macy bufoff = offset - db->db_offset; 1273eda14cbcSMatt Macy tocpy = MIN(db->db_size - bufoff, size); 1274eda14cbcSMatt Macy 1275eda14cbcSMatt Macy ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size); 1276eda14cbcSMatt Macy 1277eda14cbcSMatt Macy if (tocpy == db->db_size) 1278188408daSMartin Matuska dmu_buf_will_fill(db, tx, B_FALSE); 1279eda14cbcSMatt Macy else 1280eda14cbcSMatt Macy dmu_buf_will_dirty(db, tx); 1281eda14cbcSMatt Macy 1282*17aab35aSMartin Matuska ASSERT(db->db_data != NULL); 1283eda14cbcSMatt Macy (void) memcpy((char *)db->db_data + bufoff, buf, tocpy); 1284eda14cbcSMatt Macy 1285eda14cbcSMatt Macy if (tocpy == db->db_size) 1286188408daSMartin Matuska dmu_buf_fill_done(db, tx, B_FALSE); 1287eda14cbcSMatt Macy 1288eda14cbcSMatt Macy offset += tocpy; 1289eda14cbcSMatt Macy size -= tocpy; 1290eda14cbcSMatt Macy buf = (char *)buf + tocpy; 1291eda14cbcSMatt Macy } 1292eda14cbcSMatt Macy } 1293eda14cbcSMatt Macy 1294eda14cbcSMatt Macy void 1295eda14cbcSMatt Macy dmu_write(objset_t *os, uint64_t object, uint64_t offset, uint64_t size, 1296eda14cbcSMatt Macy const void *buf, dmu_tx_t *tx) 1297eda14cbcSMatt Macy { 1298eda14cbcSMatt Macy dmu_buf_t **dbp; 1299eda14cbcSMatt Macy int numbufs; 1300eda14cbcSMatt Macy 1301eda14cbcSMatt Macy if (size == 0) 1302eda14cbcSMatt Macy return; 1303eda14cbcSMatt Macy 1304eda14cbcSMatt Macy VERIFY0(dmu_buf_hold_array(os, object, offset, size, 1305eda14cbcSMatt Macy FALSE, FTAG, &numbufs, &dbp)); 1306eda14cbcSMatt Macy dmu_write_impl(dbp, numbufs, offset, size, buf, tx); 1307eda14cbcSMatt Macy dmu_buf_rele_array(dbp, numbufs, FTAG); 1308eda14cbcSMatt Macy } 1309eda14cbcSMatt Macy 1310eda14cbcSMatt Macy /* 13117a7741afSMartin Matuska * This interface is not used internally by ZFS but is provided for 13127a7741afSMartin Matuska * use by Lustre which is built on the DMU interfaces. 1313eda14cbcSMatt Macy */ 13147a7741afSMartin Matuska int 13157a7741afSMartin Matuska dmu_write_by_dnode_flags(dnode_t *dn, uint64_t offset, uint64_t size, 13167a7741afSMartin Matuska const void *buf, dmu_tx_t *tx, uint32_t flags) 1317eda14cbcSMatt Macy { 1318eda14cbcSMatt Macy dmu_buf_t **dbp; 1319eda14cbcSMatt Macy int numbufs; 13207a7741afSMartin Matuska int error; 1321eda14cbcSMatt Macy 1322eda14cbcSMatt Macy if (size == 0) 13237a7741afSMartin Matuska return (0); 13247a7741afSMartin Matuska 13257a7741afSMartin Matuska /* Allow Direct I/O when requested and properly aligned */ 13267a7741afSMartin Matuska if ((flags & DMU_DIRECTIO) && zfs_dio_page_aligned((void *)buf) && 13277a7741afSMartin Matuska zfs_dio_aligned(offset, size, dn->dn_datablksz)) { 13287a7741afSMartin Matuska abd_t *data = abd_get_from_buf((void *)buf, size); 13297a7741afSMartin Matuska error = dmu_write_abd(dn, offset, size, data, DMU_DIRECTIO, tx); 13307a7741afSMartin Matuska abd_free(data); 13317a7741afSMartin Matuska return (error); 13327a7741afSMartin Matuska } 1333eda14cbcSMatt Macy 1334eda14cbcSMatt Macy VERIFY0(dmu_buf_hold_array_by_dnode(dn, offset, size, 1335eda14cbcSMatt Macy FALSE, FTAG, &numbufs, &dbp, DMU_READ_PREFETCH)); 1336eda14cbcSMatt Macy dmu_write_impl(dbp, numbufs, offset, size, buf, tx); 1337eda14cbcSMatt Macy dmu_buf_rele_array(dbp, numbufs, FTAG); 13387a7741afSMartin Matuska return (0); 13397a7741afSMartin Matuska } 13407a7741afSMartin Matuska 13417a7741afSMartin Matuska int 13427a7741afSMartin Matuska dmu_write_by_dnode(dnode_t *dn, uint64_t offset, uint64_t size, 13437a7741afSMartin Matuska const void *buf, dmu_tx_t *tx) 13447a7741afSMartin Matuska { 13457a7741afSMartin Matuska return (dmu_write_by_dnode_flags(dn, offset, size, buf, tx, 0)); 1346eda14cbcSMatt Macy } 1347eda14cbcSMatt Macy 1348eda14cbcSMatt Macy void 1349eda14cbcSMatt Macy dmu_prealloc(objset_t *os, uint64_t object, uint64_t offset, uint64_t size, 1350eda14cbcSMatt Macy dmu_tx_t *tx) 1351eda14cbcSMatt Macy { 1352eda14cbcSMatt Macy dmu_buf_t **dbp; 1353eda14cbcSMatt Macy int numbufs, i; 1354eda14cbcSMatt Macy 1355eda14cbcSMatt Macy if (size == 0) 1356eda14cbcSMatt Macy return; 1357eda14cbcSMatt Macy 1358eda14cbcSMatt Macy VERIFY(0 == dmu_buf_hold_array(os, object, offset, size, 1359eda14cbcSMatt Macy FALSE, FTAG, &numbufs, &dbp)); 1360eda14cbcSMatt Macy 1361eda14cbcSMatt Macy for (i = 0; i < numbufs; i++) { 1362eda14cbcSMatt Macy dmu_buf_t *db = dbp[i]; 1363eda14cbcSMatt Macy 1364eda14cbcSMatt Macy dmu_buf_will_not_fill(db, tx); 1365eda14cbcSMatt Macy } 1366eda14cbcSMatt Macy dmu_buf_rele_array(dbp, numbufs, FTAG); 1367eda14cbcSMatt Macy } 1368eda14cbcSMatt Macy 1369eda14cbcSMatt Macy void 1370eda14cbcSMatt Macy dmu_write_embedded(objset_t *os, uint64_t object, uint64_t offset, 1371eda14cbcSMatt Macy void *data, uint8_t etype, uint8_t comp, int uncompressed_size, 1372eda14cbcSMatt Macy int compressed_size, int byteorder, dmu_tx_t *tx) 1373eda14cbcSMatt Macy { 1374eda14cbcSMatt Macy dmu_buf_t *db; 1375eda14cbcSMatt Macy 1376eda14cbcSMatt Macy ASSERT3U(etype, <, NUM_BP_EMBEDDED_TYPES); 1377eda14cbcSMatt Macy ASSERT3U(comp, <, ZIO_COMPRESS_FUNCTIONS); 1378eda14cbcSMatt Macy VERIFY0(dmu_buf_hold_noread(os, object, offset, 1379eda14cbcSMatt Macy FTAG, &db)); 1380eda14cbcSMatt Macy 1381eda14cbcSMatt Macy dmu_buf_write_embedded(db, 1382eda14cbcSMatt Macy data, (bp_embedded_type_t)etype, (enum zio_compress)comp, 1383eda14cbcSMatt Macy uncompressed_size, compressed_size, byteorder, tx); 1384eda14cbcSMatt Macy 1385eda14cbcSMatt Macy dmu_buf_rele(db, FTAG); 1386eda14cbcSMatt Macy } 1387eda14cbcSMatt Macy 1388eda14cbcSMatt Macy void 1389eda14cbcSMatt Macy dmu_redact(objset_t *os, uint64_t object, uint64_t offset, uint64_t size, 1390eda14cbcSMatt Macy dmu_tx_t *tx) 1391eda14cbcSMatt Macy { 1392eda14cbcSMatt Macy int numbufs, i; 1393eda14cbcSMatt Macy dmu_buf_t **dbp; 1394eda14cbcSMatt Macy 1395eda14cbcSMatt Macy VERIFY0(dmu_buf_hold_array(os, object, offset, size, FALSE, FTAG, 1396eda14cbcSMatt Macy &numbufs, &dbp)); 1397eda14cbcSMatt Macy for (i = 0; i < numbufs; i++) 1398eda14cbcSMatt Macy dmu_buf_redact(dbp[i], tx); 1399eda14cbcSMatt Macy dmu_buf_rele_array(dbp, numbufs, FTAG); 1400eda14cbcSMatt Macy } 1401eda14cbcSMatt Macy 1402eda14cbcSMatt Macy #ifdef _KERNEL 1403eda14cbcSMatt Macy int 1404184c1b94SMartin Matuska dmu_read_uio_dnode(dnode_t *dn, zfs_uio_t *uio, uint64_t size) 1405eda14cbcSMatt Macy { 1406eda14cbcSMatt Macy dmu_buf_t **dbp; 1407eda14cbcSMatt Macy int numbufs, i, err; 1408eda14cbcSMatt Macy 14097a7741afSMartin Matuska if (uio->uio_extflg & UIO_DIRECT) 14107a7741afSMartin Matuska return (dmu_read_uio_direct(dn, uio, size)); 14117a7741afSMartin Matuska 1412eda14cbcSMatt Macy /* 1413eda14cbcSMatt Macy * NB: we could do this block-at-a-time, but it's nice 1414eda14cbcSMatt Macy * to be reading in parallel. 1415eda14cbcSMatt Macy */ 1416184c1b94SMartin Matuska err = dmu_buf_hold_array_by_dnode(dn, zfs_uio_offset(uio), size, 1417eda14cbcSMatt Macy TRUE, FTAG, &numbufs, &dbp, 0); 1418eda14cbcSMatt Macy if (err) 1419eda14cbcSMatt Macy return (err); 1420eda14cbcSMatt Macy 1421eda14cbcSMatt Macy for (i = 0; i < numbufs; i++) { 1422eda14cbcSMatt Macy uint64_t tocpy; 1423eda14cbcSMatt Macy int64_t bufoff; 1424eda14cbcSMatt Macy dmu_buf_t *db = dbp[i]; 1425eda14cbcSMatt Macy 1426eda14cbcSMatt Macy ASSERT(size > 0); 1427eda14cbcSMatt Macy 1428184c1b94SMartin Matuska bufoff = zfs_uio_offset(uio) - db->db_offset; 1429eda14cbcSMatt Macy tocpy = MIN(db->db_size - bufoff, size); 1430eda14cbcSMatt Macy 1431*17aab35aSMartin Matuska ASSERT(db->db_data != NULL); 1432184c1b94SMartin Matuska err = zfs_uio_fault_move((char *)db->db_data + bufoff, tocpy, 1433eda14cbcSMatt Macy UIO_READ, uio); 1434184c1b94SMartin Matuska 1435eda14cbcSMatt Macy if (err) 1436eda14cbcSMatt Macy break; 1437eda14cbcSMatt Macy 1438eda14cbcSMatt Macy size -= tocpy; 1439eda14cbcSMatt Macy } 1440eda14cbcSMatt Macy dmu_buf_rele_array(dbp, numbufs, FTAG); 1441eda14cbcSMatt Macy 1442eda14cbcSMatt Macy return (err); 1443eda14cbcSMatt Macy } 1444eda14cbcSMatt Macy 1445eda14cbcSMatt Macy /* 1446eda14cbcSMatt Macy * Read 'size' bytes into the uio buffer. 1447eda14cbcSMatt Macy * From object zdb->db_object. 1448184c1b94SMartin Matuska * Starting at zfs_uio_offset(uio). 1449eda14cbcSMatt Macy * 1450eda14cbcSMatt Macy * If the caller already has a dbuf in the target object 1451eda14cbcSMatt Macy * (e.g. its bonus buffer), this routine is faster than dmu_read_uio(), 1452eda14cbcSMatt Macy * because we don't have to find the dnode_t for the object. 1453eda14cbcSMatt Macy */ 1454eda14cbcSMatt Macy int 1455184c1b94SMartin Matuska dmu_read_uio_dbuf(dmu_buf_t *zdb, zfs_uio_t *uio, uint64_t size) 1456eda14cbcSMatt Macy { 1457eda14cbcSMatt Macy dmu_buf_impl_t *db = (dmu_buf_impl_t *)zdb; 1458eda14cbcSMatt Macy int err; 1459eda14cbcSMatt Macy 1460eda14cbcSMatt Macy if (size == 0) 1461eda14cbcSMatt Macy return (0); 1462eda14cbcSMatt Macy 1463eda14cbcSMatt Macy DB_DNODE_ENTER(db); 1464ce4dcb97SMartin Matuska err = dmu_read_uio_dnode(DB_DNODE(db), uio, size); 1465eda14cbcSMatt Macy DB_DNODE_EXIT(db); 1466eda14cbcSMatt Macy 1467eda14cbcSMatt Macy return (err); 1468eda14cbcSMatt Macy } 1469eda14cbcSMatt Macy 1470eda14cbcSMatt Macy /* 1471eda14cbcSMatt Macy * Read 'size' bytes into the uio buffer. 1472eda14cbcSMatt Macy * From the specified object 1473184c1b94SMartin Matuska * Starting at offset zfs_uio_offset(uio). 1474eda14cbcSMatt Macy */ 1475eda14cbcSMatt Macy int 1476184c1b94SMartin Matuska dmu_read_uio(objset_t *os, uint64_t object, zfs_uio_t *uio, uint64_t size) 1477eda14cbcSMatt Macy { 1478eda14cbcSMatt Macy dnode_t *dn; 1479eda14cbcSMatt Macy int err; 1480eda14cbcSMatt Macy 1481eda14cbcSMatt Macy if (size == 0) 1482eda14cbcSMatt Macy return (0); 1483eda14cbcSMatt Macy 1484eda14cbcSMatt Macy err = dnode_hold(os, object, FTAG, &dn); 1485eda14cbcSMatt Macy if (err) 1486eda14cbcSMatt Macy return (err); 1487eda14cbcSMatt Macy 1488eda14cbcSMatt Macy err = dmu_read_uio_dnode(dn, uio, size); 1489eda14cbcSMatt Macy 1490eda14cbcSMatt Macy dnode_rele(dn, FTAG); 1491eda14cbcSMatt Macy 1492eda14cbcSMatt Macy return (err); 1493eda14cbcSMatt Macy } 1494eda14cbcSMatt Macy 1495eda14cbcSMatt Macy int 1496184c1b94SMartin Matuska dmu_write_uio_dnode(dnode_t *dn, zfs_uio_t *uio, uint64_t size, dmu_tx_t *tx) 1497eda14cbcSMatt Macy { 1498eda14cbcSMatt Macy dmu_buf_t **dbp; 1499eda14cbcSMatt Macy int numbufs; 1500eda14cbcSMatt Macy int err = 0; 15017a7741afSMartin Matuska uint64_t write_size; 1502eda14cbcSMatt Macy 15037a7741afSMartin Matuska top: 15047a7741afSMartin Matuska write_size = size; 15057a7741afSMartin Matuska 15067a7741afSMartin Matuska /* 15077a7741afSMartin Matuska * We only allow Direct I/O writes to happen if we are block 15087a7741afSMartin Matuska * sized aligned. Otherwise, we pass the write off to the ARC. 15097a7741afSMartin Matuska */ 15107a7741afSMartin Matuska if ((uio->uio_extflg & UIO_DIRECT) && 15117a7741afSMartin Matuska (write_size >= dn->dn_datablksz)) { 15127a7741afSMartin Matuska if (zfs_dio_aligned(zfs_uio_offset(uio), write_size, 15137a7741afSMartin Matuska dn->dn_datablksz)) { 15147a7741afSMartin Matuska return (dmu_write_uio_direct(dn, uio, size, tx)); 15157a7741afSMartin Matuska } else if (write_size > dn->dn_datablksz && 15167a7741afSMartin Matuska zfs_dio_offset_aligned(zfs_uio_offset(uio), 15177a7741afSMartin Matuska dn->dn_datablksz)) { 15187a7741afSMartin Matuska write_size = 15197a7741afSMartin Matuska dn->dn_datablksz * (write_size / dn->dn_datablksz); 15207a7741afSMartin Matuska err = dmu_write_uio_direct(dn, uio, write_size, tx); 15217a7741afSMartin Matuska if (err == 0) { 15227a7741afSMartin Matuska size -= write_size; 15237a7741afSMartin Matuska goto top; 15247a7741afSMartin Matuska } else { 15257a7741afSMartin Matuska return (err); 15267a7741afSMartin Matuska } 15277a7741afSMartin Matuska } else { 15287a7741afSMartin Matuska write_size = 15297a7741afSMartin Matuska P2PHASE(zfs_uio_offset(uio), dn->dn_datablksz); 15307a7741afSMartin Matuska } 15317a7741afSMartin Matuska } 15327a7741afSMartin Matuska 15337a7741afSMartin Matuska err = dmu_buf_hold_array_by_dnode(dn, zfs_uio_offset(uio), write_size, 1534eda14cbcSMatt Macy FALSE, FTAG, &numbufs, &dbp, DMU_READ_PREFETCH); 1535eda14cbcSMatt Macy if (err) 1536eda14cbcSMatt Macy return (err); 1537eda14cbcSMatt Macy 15387a7741afSMartin Matuska for (int i = 0; i < numbufs; i++) { 1539eda14cbcSMatt Macy uint64_t tocpy; 1540eda14cbcSMatt Macy int64_t bufoff; 1541eda14cbcSMatt Macy dmu_buf_t *db = dbp[i]; 1542eda14cbcSMatt Macy 15437a7741afSMartin Matuska ASSERT(write_size > 0); 1544eda14cbcSMatt Macy 1545188408daSMartin Matuska offset_t off = zfs_uio_offset(uio); 1546188408daSMartin Matuska bufoff = off - db->db_offset; 15477a7741afSMartin Matuska tocpy = MIN(db->db_size - bufoff, write_size); 1548eda14cbcSMatt Macy 1549eda14cbcSMatt Macy ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size); 1550eda14cbcSMatt Macy 1551eda14cbcSMatt Macy if (tocpy == db->db_size) 1552188408daSMartin Matuska dmu_buf_will_fill(db, tx, B_TRUE); 1553eda14cbcSMatt Macy else 1554eda14cbcSMatt Macy dmu_buf_will_dirty(db, tx); 1555eda14cbcSMatt Macy 1556*17aab35aSMartin Matuska ASSERT(db->db_data != NULL); 1557184c1b94SMartin Matuska err = zfs_uio_fault_move((char *)db->db_data + bufoff, 1558184c1b94SMartin Matuska tocpy, UIO_WRITE, uio); 1559184c1b94SMartin Matuska 1560188408daSMartin Matuska if (tocpy == db->db_size && dmu_buf_fill_done(db, tx, err)) { 1561188408daSMartin Matuska /* The fill was reverted. Undo any uio progress. */ 1562188408daSMartin Matuska zfs_uio_advance(uio, off - zfs_uio_offset(uio)); 1563188408daSMartin Matuska } 1564eda14cbcSMatt Macy 1565eda14cbcSMatt Macy if (err) 1566eda14cbcSMatt Macy break; 1567eda14cbcSMatt Macy 15687a7741afSMartin Matuska write_size -= tocpy; 1569eda14cbcSMatt Macy size -= tocpy; 1570eda14cbcSMatt Macy } 1571eda14cbcSMatt Macy 15727a7741afSMartin Matuska IMPLY(err == 0, write_size == 0); 15737a7741afSMartin Matuska 1574eda14cbcSMatt Macy dmu_buf_rele_array(dbp, numbufs, FTAG); 15757a7741afSMartin Matuska 15767a7741afSMartin Matuska if ((uio->uio_extflg & UIO_DIRECT) && size > 0) { 15777a7741afSMartin Matuska goto top; 15787a7741afSMartin Matuska } 15797a7741afSMartin Matuska 1580eda14cbcSMatt Macy return (err); 1581eda14cbcSMatt Macy } 1582eda14cbcSMatt Macy 1583eda14cbcSMatt Macy /* 1584eda14cbcSMatt Macy * Write 'size' bytes from the uio buffer. 1585eda14cbcSMatt Macy * To object zdb->db_object. 1586184c1b94SMartin Matuska * Starting at offset zfs_uio_offset(uio). 1587eda14cbcSMatt Macy * 1588eda14cbcSMatt Macy * If the caller already has a dbuf in the target object 1589eda14cbcSMatt Macy * (e.g. its bonus buffer), this routine is faster than dmu_write_uio(), 1590eda14cbcSMatt Macy * because we don't have to find the dnode_t for the object. 1591eda14cbcSMatt Macy */ 1592eda14cbcSMatt Macy int 1593184c1b94SMartin Matuska dmu_write_uio_dbuf(dmu_buf_t *zdb, zfs_uio_t *uio, uint64_t size, 1594eda14cbcSMatt Macy dmu_tx_t *tx) 1595eda14cbcSMatt Macy { 1596eda14cbcSMatt Macy dmu_buf_impl_t *db = (dmu_buf_impl_t *)zdb; 1597eda14cbcSMatt Macy int err; 1598eda14cbcSMatt Macy 1599eda14cbcSMatt Macy if (size == 0) 1600eda14cbcSMatt Macy return (0); 1601eda14cbcSMatt Macy 1602eda14cbcSMatt Macy DB_DNODE_ENTER(db); 1603ce4dcb97SMartin Matuska err = dmu_write_uio_dnode(DB_DNODE(db), uio, size, tx); 1604eda14cbcSMatt Macy DB_DNODE_EXIT(db); 1605eda14cbcSMatt Macy 1606eda14cbcSMatt Macy return (err); 1607eda14cbcSMatt Macy } 1608eda14cbcSMatt Macy 1609eda14cbcSMatt Macy /* 1610eda14cbcSMatt Macy * Write 'size' bytes from the uio buffer. 1611eda14cbcSMatt Macy * To the specified object. 1612184c1b94SMartin Matuska * Starting at offset zfs_uio_offset(uio). 1613eda14cbcSMatt Macy */ 1614eda14cbcSMatt Macy int 1615184c1b94SMartin Matuska dmu_write_uio(objset_t *os, uint64_t object, zfs_uio_t *uio, uint64_t size, 1616eda14cbcSMatt Macy dmu_tx_t *tx) 1617eda14cbcSMatt Macy { 1618eda14cbcSMatt Macy dnode_t *dn; 1619eda14cbcSMatt Macy int err; 1620eda14cbcSMatt Macy 1621eda14cbcSMatt Macy if (size == 0) 1622eda14cbcSMatt Macy return (0); 1623eda14cbcSMatt Macy 1624eda14cbcSMatt Macy err = dnode_hold(os, object, FTAG, &dn); 1625eda14cbcSMatt Macy if (err) 1626eda14cbcSMatt Macy return (err); 1627eda14cbcSMatt Macy 1628eda14cbcSMatt Macy err = dmu_write_uio_dnode(dn, uio, size, tx); 1629eda14cbcSMatt Macy 1630eda14cbcSMatt Macy dnode_rele(dn, FTAG); 1631eda14cbcSMatt Macy 1632eda14cbcSMatt Macy return (err); 1633eda14cbcSMatt Macy } 1634eda14cbcSMatt Macy #endif /* _KERNEL */ 1635eda14cbcSMatt Macy 1636ce4dcb97SMartin Matuska static void 1637ce4dcb97SMartin Matuska dmu_cached_bps(spa_t *spa, blkptr_t *bps, uint_t nbps, 1638ce4dcb97SMartin Matuska uint64_t *l1sz, uint64_t *l2sz) 1639ce4dcb97SMartin Matuska { 1640ce4dcb97SMartin Matuska int cached_flags; 1641ce4dcb97SMartin Matuska 1642ce4dcb97SMartin Matuska if (bps == NULL) 1643ce4dcb97SMartin Matuska return; 1644ce4dcb97SMartin Matuska 1645ce4dcb97SMartin Matuska for (size_t blk_off = 0; blk_off < nbps; blk_off++) { 1646ce4dcb97SMartin Matuska blkptr_t *bp = &bps[blk_off]; 1647ce4dcb97SMartin Matuska 1648ce4dcb97SMartin Matuska if (BP_IS_HOLE(bp)) 1649ce4dcb97SMartin Matuska continue; 1650ce4dcb97SMartin Matuska 1651ce4dcb97SMartin Matuska cached_flags = arc_cached(spa, bp); 1652ce4dcb97SMartin Matuska if (cached_flags == 0) 1653ce4dcb97SMartin Matuska continue; 1654ce4dcb97SMartin Matuska 1655ce4dcb97SMartin Matuska if ((cached_flags & (ARC_CACHED_IN_L1 | ARC_CACHED_IN_L2)) == 1656ce4dcb97SMartin Matuska ARC_CACHED_IN_L2) 1657ce4dcb97SMartin Matuska *l2sz += BP_GET_LSIZE(bp); 1658ce4dcb97SMartin Matuska else 1659ce4dcb97SMartin Matuska *l1sz += BP_GET_LSIZE(bp); 1660ce4dcb97SMartin Matuska } 1661ce4dcb97SMartin Matuska } 1662ce4dcb97SMartin Matuska 1663ce4dcb97SMartin Matuska /* 1664ce4dcb97SMartin Matuska * Estimate DMU object cached size. 1665ce4dcb97SMartin Matuska */ 1666ce4dcb97SMartin Matuska int 1667ce4dcb97SMartin Matuska dmu_object_cached_size(objset_t *os, uint64_t object, 1668ce4dcb97SMartin Matuska uint64_t *l1sz, uint64_t *l2sz) 1669ce4dcb97SMartin Matuska { 1670ce4dcb97SMartin Matuska dnode_t *dn; 1671ce4dcb97SMartin Matuska dmu_object_info_t doi; 1672ce4dcb97SMartin Matuska int err = 0; 1673ce4dcb97SMartin Matuska 1674ce4dcb97SMartin Matuska *l1sz = *l2sz = 0; 1675ce4dcb97SMartin Matuska 1676ce4dcb97SMartin Matuska if (dnode_hold(os, object, FTAG, &dn) != 0) 1677ce4dcb97SMartin Matuska return (0); 1678ce4dcb97SMartin Matuska 1679ce4dcb97SMartin Matuska if (dn->dn_nlevels < 2) { 1680ce4dcb97SMartin Matuska dnode_rele(dn, FTAG); 1681ce4dcb97SMartin Matuska return (0); 1682ce4dcb97SMartin Matuska } 1683ce4dcb97SMartin Matuska 1684ce4dcb97SMartin Matuska dmu_object_info_from_dnode(dn, &doi); 1685ce4dcb97SMartin Matuska 1686ce4dcb97SMartin Matuska for (uint64_t off = 0; off < doi.doi_max_offset; 1687ce4dcb97SMartin Matuska off += dmu_prefetch_max) { 1688ce4dcb97SMartin Matuska /* dbuf_read doesn't prefetch L1 blocks. */ 1689ce4dcb97SMartin Matuska dmu_prefetch_by_dnode(dn, 1, off, 1690ce4dcb97SMartin Matuska dmu_prefetch_max, ZIO_PRIORITY_SYNC_READ); 1691ce4dcb97SMartin Matuska } 1692ce4dcb97SMartin Matuska 1693ce4dcb97SMartin Matuska /* 1694ce4dcb97SMartin Matuska * Hold all valid L1 blocks, asking ARC the status of each BP 1695ce4dcb97SMartin Matuska * contained in each such L1 block. 1696ce4dcb97SMartin Matuska */ 1697ce4dcb97SMartin Matuska uint_t nbps = bp_span_in_blocks(dn->dn_indblkshift, 1); 1698ce4dcb97SMartin Matuska uint64_t l1blks = 1 + (dn->dn_maxblkid / nbps); 1699ce4dcb97SMartin Matuska 1700ce4dcb97SMartin Matuska rw_enter(&dn->dn_struct_rwlock, RW_READER); 1701ce4dcb97SMartin Matuska for (uint64_t blk = 0; blk < l1blks; blk++) { 1702ce4dcb97SMartin Matuska dmu_buf_impl_t *db = NULL; 1703ce4dcb97SMartin Matuska 1704ce4dcb97SMartin Matuska if (issig()) { 1705ce4dcb97SMartin Matuska /* 1706ce4dcb97SMartin Matuska * On interrupt, get out, and bubble up EINTR 1707ce4dcb97SMartin Matuska */ 1708ce4dcb97SMartin Matuska err = EINTR; 1709ce4dcb97SMartin Matuska break; 1710ce4dcb97SMartin Matuska } 1711ce4dcb97SMartin Matuska 1712ce4dcb97SMartin Matuska /* 1713ce4dcb97SMartin Matuska * If we get an i/o error here, the L1 can't be read, 1714ce4dcb97SMartin Matuska * and nothing under it could be cached, so we just 1715ce4dcb97SMartin Matuska * continue. Ignoring the error from dbuf_hold_impl 1716ce4dcb97SMartin Matuska * or from dbuf_read is then a reasonable choice. 1717ce4dcb97SMartin Matuska */ 1718ce4dcb97SMartin Matuska err = dbuf_hold_impl(dn, 1, blk, B_TRUE, B_FALSE, FTAG, &db); 1719ce4dcb97SMartin Matuska if (err != 0) { 1720ce4dcb97SMartin Matuska /* 1721ce4dcb97SMartin Matuska * ignore error and continue 1722ce4dcb97SMartin Matuska */ 1723ce4dcb97SMartin Matuska err = 0; 1724ce4dcb97SMartin Matuska continue; 1725ce4dcb97SMartin Matuska } 1726ce4dcb97SMartin Matuska 1727ce4dcb97SMartin Matuska err = dbuf_read(db, NULL, DB_RF_CANFAIL); 1728ce4dcb97SMartin Matuska if (err == 0) { 1729ce4dcb97SMartin Matuska dmu_cached_bps(dmu_objset_spa(os), db->db.db_data, 1730ce4dcb97SMartin Matuska nbps, l1sz, l2sz); 1731ce4dcb97SMartin Matuska } 1732ce4dcb97SMartin Matuska /* 1733ce4dcb97SMartin Matuska * error may be ignored, and we continue 1734ce4dcb97SMartin Matuska */ 1735ce4dcb97SMartin Matuska err = 0; 1736ce4dcb97SMartin Matuska dbuf_rele(db, FTAG); 1737ce4dcb97SMartin Matuska } 1738ce4dcb97SMartin Matuska rw_exit(&dn->dn_struct_rwlock); 1739ce4dcb97SMartin Matuska 1740ce4dcb97SMartin Matuska dnode_rele(dn, FTAG); 1741ce4dcb97SMartin Matuska return (err); 1742ce4dcb97SMartin Matuska } 1743ce4dcb97SMartin Matuska 1744eda14cbcSMatt Macy /* 1745eda14cbcSMatt Macy * Allocate a loaned anonymous arc buffer. 1746eda14cbcSMatt Macy */ 1747eda14cbcSMatt Macy arc_buf_t * 1748eda14cbcSMatt Macy dmu_request_arcbuf(dmu_buf_t *handle, int size) 1749eda14cbcSMatt Macy { 1750eda14cbcSMatt Macy dmu_buf_impl_t *db = (dmu_buf_impl_t *)handle; 1751eda14cbcSMatt Macy 1752eda14cbcSMatt Macy return (arc_loan_buf(db->db_objset->os_spa, B_FALSE, size)); 1753eda14cbcSMatt Macy } 1754eda14cbcSMatt Macy 1755eda14cbcSMatt Macy /* 1756eda14cbcSMatt Macy * Free a loaned arc buffer. 1757eda14cbcSMatt Macy */ 1758eda14cbcSMatt Macy void 1759eda14cbcSMatt Macy dmu_return_arcbuf(arc_buf_t *buf) 1760eda14cbcSMatt Macy { 1761eda14cbcSMatt Macy arc_return_buf(buf, FTAG); 1762eda14cbcSMatt Macy arc_buf_destroy(buf, FTAG); 1763eda14cbcSMatt Macy } 1764eda14cbcSMatt Macy 1765eda14cbcSMatt Macy /* 17667877fdebSMatt Macy * A "lightweight" write is faster than a regular write (e.g. 17677877fdebSMatt Macy * dmu_write_by_dnode() or dmu_assign_arcbuf_by_dnode()), because it avoids the 17687877fdebSMatt Macy * CPU cost of creating a dmu_buf_impl_t and arc_buf_[hdr_]_t. However, the 17697877fdebSMatt Macy * data can not be read or overwritten until the transaction's txg has been 17707877fdebSMatt Macy * synced. This makes it appropriate for workloads that are known to be 17717877fdebSMatt Macy * (temporarily) write-only, like "zfs receive". 17727877fdebSMatt Macy * 17737877fdebSMatt Macy * A single block is written, starting at the specified offset in bytes. If 17747877fdebSMatt Macy * the call is successful, it returns 0 and the provided abd has been 17757877fdebSMatt Macy * consumed (the caller should not free it). 17767877fdebSMatt Macy */ 17777877fdebSMatt Macy int 17787877fdebSMatt Macy dmu_lightweight_write_by_dnode(dnode_t *dn, uint64_t offset, abd_t *abd, 1779dbd5678dSMartin Matuska const zio_prop_t *zp, zio_flag_t flags, dmu_tx_t *tx) 17807877fdebSMatt Macy { 17817877fdebSMatt Macy dbuf_dirty_record_t *dr = 17827877fdebSMatt Macy dbuf_dirty_lightweight(dn, dbuf_whichblock(dn, 0, offset), tx); 17837877fdebSMatt Macy if (dr == NULL) 17847877fdebSMatt Macy return (SET_ERROR(EIO)); 17857877fdebSMatt Macy dr->dt.dll.dr_abd = abd; 17867877fdebSMatt Macy dr->dt.dll.dr_props = *zp; 17877877fdebSMatt Macy dr->dt.dll.dr_flags = flags; 17887877fdebSMatt Macy return (0); 17897877fdebSMatt Macy } 17907877fdebSMatt Macy 17917877fdebSMatt Macy /* 1792eda14cbcSMatt Macy * When possible directly assign passed loaned arc buffer to a dbuf. 1793eda14cbcSMatt Macy * If this is not possible copy the contents of passed arc buf via 1794eda14cbcSMatt Macy * dmu_write(). 1795eda14cbcSMatt Macy */ 1796eda14cbcSMatt Macy int 1797eda14cbcSMatt Macy dmu_assign_arcbuf_by_dnode(dnode_t *dn, uint64_t offset, arc_buf_t *buf, 1798eda14cbcSMatt Macy dmu_tx_t *tx) 1799eda14cbcSMatt Macy { 1800eda14cbcSMatt Macy dmu_buf_impl_t *db; 1801eda14cbcSMatt Macy objset_t *os = dn->dn_objset; 1802eda14cbcSMatt Macy uint64_t object = dn->dn_object; 1803eda14cbcSMatt Macy uint32_t blksz = (uint32_t)arc_buf_lsize(buf); 1804eda14cbcSMatt Macy uint64_t blkid; 1805eda14cbcSMatt Macy 1806eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_READER); 1807eda14cbcSMatt Macy blkid = dbuf_whichblock(dn, 0, offset); 1808eda14cbcSMatt Macy db = dbuf_hold(dn, blkid, FTAG); 18095fb307d2SMartin Matuska rw_exit(&dn->dn_struct_rwlock); 1810eda14cbcSMatt Macy if (db == NULL) 1811eda14cbcSMatt Macy return (SET_ERROR(EIO)); 1812eda14cbcSMatt Macy 1813eda14cbcSMatt Macy /* 18147877fdebSMatt Macy * We can only assign if the offset is aligned and the arc buf is the 18157877fdebSMatt Macy * same size as the dbuf. 1816eda14cbcSMatt Macy */ 1817eda14cbcSMatt Macy if (offset == db->db.db_offset && blksz == db->db.db_size) { 18187a7741afSMartin Matuska zfs_racct_write(os->os_spa, blksz, 1, 0); 1819eda14cbcSMatt Macy dbuf_assign_arcbuf(db, buf, tx); 1820eda14cbcSMatt Macy dbuf_rele(db, FTAG); 1821eda14cbcSMatt Macy } else { 1822eda14cbcSMatt Macy /* compressed bufs must always be assignable to their dbuf */ 1823eda14cbcSMatt Macy ASSERT3U(arc_get_compression(buf), ==, ZIO_COMPRESS_OFF); 1824eda14cbcSMatt Macy ASSERT(!(buf->b_flags & ARC_BUF_FLAG_COMPRESSED)); 1825eda14cbcSMatt Macy 1826eda14cbcSMatt Macy dbuf_rele(db, FTAG); 1827eda14cbcSMatt Macy dmu_write(os, object, offset, blksz, buf->b_data, tx); 1828eda14cbcSMatt Macy dmu_return_arcbuf(buf); 1829eda14cbcSMatt Macy } 1830eda14cbcSMatt Macy 1831eda14cbcSMatt Macy return (0); 1832eda14cbcSMatt Macy } 1833eda14cbcSMatt Macy 1834eda14cbcSMatt Macy int 1835eda14cbcSMatt Macy dmu_assign_arcbuf_by_dbuf(dmu_buf_t *handle, uint64_t offset, arc_buf_t *buf, 1836eda14cbcSMatt Macy dmu_tx_t *tx) 1837eda14cbcSMatt Macy { 1838eda14cbcSMatt Macy int err; 1839ce4dcb97SMartin Matuska dmu_buf_impl_t *db = (dmu_buf_impl_t *)handle; 1840eda14cbcSMatt Macy 1841ce4dcb97SMartin Matuska DB_DNODE_ENTER(db); 1842ce4dcb97SMartin Matuska err = dmu_assign_arcbuf_by_dnode(DB_DNODE(db), offset, buf, tx); 1843ce4dcb97SMartin Matuska DB_DNODE_EXIT(db); 1844eda14cbcSMatt Macy 1845eda14cbcSMatt Macy return (err); 1846eda14cbcSMatt Macy } 1847eda14cbcSMatt Macy 18487a7741afSMartin Matuska void 1849eda14cbcSMatt Macy dmu_sync_ready(zio_t *zio, arc_buf_t *buf, void *varg) 1850eda14cbcSMatt Macy { 1851e92ffd9bSMartin Matuska (void) buf; 1852eda14cbcSMatt Macy dmu_sync_arg_t *dsa = varg; 1853eda14cbcSMatt Macy 1854eda14cbcSMatt Macy if (zio->io_error == 0) { 18557a7741afSMartin Matuska dbuf_dirty_record_t *dr = dsa->dsa_dr; 18567a7741afSMartin Matuska blkptr_t *bp = zio->io_bp; 18577a7741afSMartin Matuska 1858eda14cbcSMatt Macy if (BP_IS_HOLE(bp)) { 18597a7741afSMartin Matuska dmu_buf_t *db = NULL; 18607a7741afSMartin Matuska if (dr) 18617a7741afSMartin Matuska db = &(dr->dr_dbuf->db); 18627a7741afSMartin Matuska else 18637a7741afSMartin Matuska db = dsa->dsa_zgd->zgd_db; 1864eda14cbcSMatt Macy /* 1865eda14cbcSMatt Macy * A block of zeros may compress to a hole, but the 1866eda14cbcSMatt Macy * block size still needs to be known for replay. 1867eda14cbcSMatt Macy */ 1868eda14cbcSMatt Macy BP_SET_LSIZE(bp, db->db_size); 1869eda14cbcSMatt Macy } else if (!BP_IS_EMBEDDED(bp)) { 1870eda14cbcSMatt Macy ASSERT(BP_GET_LEVEL(bp) == 0); 1871eda14cbcSMatt Macy BP_SET_FILL(bp, 1); 1872eda14cbcSMatt Macy } 1873eda14cbcSMatt Macy } 1874eda14cbcSMatt Macy } 1875eda14cbcSMatt Macy 1876eda14cbcSMatt Macy static void 1877eda14cbcSMatt Macy dmu_sync_late_arrival_ready(zio_t *zio) 1878eda14cbcSMatt Macy { 1879eda14cbcSMatt Macy dmu_sync_ready(zio, NULL, zio->io_private); 1880eda14cbcSMatt Macy } 1881eda14cbcSMatt Macy 18827a7741afSMartin Matuska void 1883eda14cbcSMatt Macy dmu_sync_done(zio_t *zio, arc_buf_t *buf, void *varg) 1884eda14cbcSMatt Macy { 1885e92ffd9bSMartin Matuska (void) buf; 1886eda14cbcSMatt Macy dmu_sync_arg_t *dsa = varg; 1887eda14cbcSMatt Macy dbuf_dirty_record_t *dr = dsa->dsa_dr; 1888eda14cbcSMatt Macy dmu_buf_impl_t *db = dr->dr_dbuf; 1889eda14cbcSMatt Macy zgd_t *zgd = dsa->dsa_zgd; 1890eda14cbcSMatt Macy 1891eda14cbcSMatt Macy /* 1892eda14cbcSMatt Macy * Record the vdev(s) backing this blkptr so they can be flushed after 1893eda14cbcSMatt Macy * the writes for the lwb have completed. 1894eda14cbcSMatt Macy */ 18957a7741afSMartin Matuska if (zgd && zio->io_error == 0) { 1896eda14cbcSMatt Macy zil_lwb_add_block(zgd->zgd_lwb, zgd->zgd_bp); 1897eda14cbcSMatt Macy } 1898eda14cbcSMatt Macy 1899eda14cbcSMatt Macy mutex_enter(&db->db_mtx); 1900eda14cbcSMatt Macy ASSERT(dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC); 1901eda14cbcSMatt Macy if (zio->io_error == 0) { 19025c65a0a9SMartin Matuska ASSERT0(dr->dt.dl.dr_has_raw_params); 1903eda14cbcSMatt Macy dr->dt.dl.dr_nopwrite = !!(zio->io_flags & ZIO_FLAG_NOPWRITE); 1904eda14cbcSMatt Macy if (dr->dt.dl.dr_nopwrite) { 1905eda14cbcSMatt Macy blkptr_t *bp = zio->io_bp; 1906eda14cbcSMatt Macy blkptr_t *bp_orig = &zio->io_bp_orig; 1907eda14cbcSMatt Macy uint8_t chksum = BP_GET_CHECKSUM(bp_orig); 1908eda14cbcSMatt Macy 1909eda14cbcSMatt Macy ASSERT(BP_EQUAL(bp, bp_orig)); 1910eda14cbcSMatt Macy VERIFY(BP_EQUAL(bp, db->db_blkptr)); 1911eda14cbcSMatt Macy ASSERT(zio->io_prop.zp_compress != ZIO_COMPRESS_OFF); 1912eda14cbcSMatt Macy VERIFY(zio_checksum_table[chksum].ci_flags & 1913eda14cbcSMatt Macy ZCHECKSUM_FLAG_NOPWRITE); 1914eda14cbcSMatt Macy } 1915eda14cbcSMatt Macy dr->dt.dl.dr_overridden_by = *zio->io_bp; 1916eda14cbcSMatt Macy dr->dt.dl.dr_override_state = DR_OVERRIDDEN; 1917eda14cbcSMatt Macy dr->dt.dl.dr_copies = zio->io_prop.zp_copies; 1918eda14cbcSMatt Macy 1919eda14cbcSMatt Macy /* 1920eda14cbcSMatt Macy * Old style holes are filled with all zeros, whereas 1921eda14cbcSMatt Macy * new-style holes maintain their lsize, type, level, 1922eda14cbcSMatt Macy * and birth time (see zio_write_compress). While we 1923eda14cbcSMatt Macy * need to reset the BP_SET_LSIZE() call that happened 1924eda14cbcSMatt Macy * in dmu_sync_ready for old style holes, we do *not* 1925eda14cbcSMatt Macy * want to wipe out the information contained in new 1926eda14cbcSMatt Macy * style holes. Thus, only zero out the block pointer if 1927eda14cbcSMatt Macy * it's an old style hole. 1928eda14cbcSMatt Macy */ 1929eda14cbcSMatt Macy if (BP_IS_HOLE(&dr->dt.dl.dr_overridden_by) && 1930783d3ff6SMartin Matuska BP_GET_LOGICAL_BIRTH(&dr->dt.dl.dr_overridden_by) == 0) 1931eda14cbcSMatt Macy BP_ZERO(&dr->dt.dl.dr_overridden_by); 1932eda14cbcSMatt Macy } else { 1933eda14cbcSMatt Macy dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 1934eda14cbcSMatt Macy } 19357a7741afSMartin Matuska 1936eda14cbcSMatt Macy cv_broadcast(&db->db_changed); 1937eda14cbcSMatt Macy mutex_exit(&db->db_mtx); 1938eda14cbcSMatt Macy 19397a7741afSMartin Matuska if (dsa->dsa_done) 1940eda14cbcSMatt Macy dsa->dsa_done(dsa->dsa_zgd, zio->io_error); 1941eda14cbcSMatt Macy 1942eda14cbcSMatt Macy kmem_free(dsa, sizeof (*dsa)); 1943eda14cbcSMatt Macy } 1944eda14cbcSMatt Macy 1945eda14cbcSMatt Macy static void 1946eda14cbcSMatt Macy dmu_sync_late_arrival_done(zio_t *zio) 1947eda14cbcSMatt Macy { 1948eda14cbcSMatt Macy blkptr_t *bp = zio->io_bp; 1949eda14cbcSMatt Macy dmu_sync_arg_t *dsa = zio->io_private; 1950eda14cbcSMatt Macy zgd_t *zgd = dsa->dsa_zgd; 1951eda14cbcSMatt Macy 1952eda14cbcSMatt Macy if (zio->io_error == 0) { 1953eda14cbcSMatt Macy /* 1954eda14cbcSMatt Macy * Record the vdev(s) backing this blkptr so they can be 1955eda14cbcSMatt Macy * flushed after the writes for the lwb have completed. 1956eda14cbcSMatt Macy */ 1957eda14cbcSMatt Macy zil_lwb_add_block(zgd->zgd_lwb, zgd->zgd_bp); 1958eda14cbcSMatt Macy 1959eda14cbcSMatt Macy if (!BP_IS_HOLE(bp)) { 1960eda14cbcSMatt Macy blkptr_t *bp_orig __maybe_unused = &zio->io_bp_orig; 1961eda14cbcSMatt Macy ASSERT(!(zio->io_flags & ZIO_FLAG_NOPWRITE)); 1962eda14cbcSMatt Macy ASSERT(BP_IS_HOLE(bp_orig) || !BP_EQUAL(bp, bp_orig)); 1963783d3ff6SMartin Matuska ASSERT(BP_GET_LOGICAL_BIRTH(zio->io_bp) == zio->io_txg); 1964eda14cbcSMatt Macy ASSERT(zio->io_txg > spa_syncing_txg(zio->io_spa)); 1965eda14cbcSMatt Macy zio_free(zio->io_spa, zio->io_txg, zio->io_bp); 1966eda14cbcSMatt Macy } 1967eda14cbcSMatt Macy } 1968eda14cbcSMatt Macy 1969eda14cbcSMatt Macy dmu_tx_commit(dsa->dsa_tx); 1970eda14cbcSMatt Macy 1971eda14cbcSMatt Macy dsa->dsa_done(dsa->dsa_zgd, zio->io_error); 1972eda14cbcSMatt Macy 1973184c1b94SMartin Matuska abd_free(zio->io_abd); 1974eda14cbcSMatt Macy kmem_free(dsa, sizeof (*dsa)); 1975eda14cbcSMatt Macy } 1976eda14cbcSMatt Macy 1977eda14cbcSMatt Macy static int 1978eda14cbcSMatt Macy dmu_sync_late_arrival(zio_t *pio, objset_t *os, dmu_sync_cb_t *done, zgd_t *zgd, 1979eda14cbcSMatt Macy zio_prop_t *zp, zbookmark_phys_t *zb) 1980eda14cbcSMatt Macy { 1981eda14cbcSMatt Macy dmu_sync_arg_t *dsa; 1982eda14cbcSMatt Macy dmu_tx_t *tx; 1983315ee00fSMartin Matuska int error; 1984315ee00fSMartin Matuska 1985315ee00fSMartin Matuska error = dbuf_read((dmu_buf_impl_t *)zgd->zgd_db, NULL, 1986315ee00fSMartin Matuska DB_RF_CANFAIL | DB_RF_NOPREFETCH); 1987315ee00fSMartin Matuska if (error != 0) 1988315ee00fSMartin Matuska return (error); 1989eda14cbcSMatt Macy 1990eda14cbcSMatt Macy tx = dmu_tx_create(os); 1991eda14cbcSMatt Macy dmu_tx_hold_space(tx, zgd->zgd_db->db_size); 1992315ee00fSMartin Matuska /* 1993315ee00fSMartin Matuska * This transaction does not produce any dirty data or log blocks, so 1994315ee00fSMartin Matuska * it should not be throttled. All other cases wait for TXG sync, by 1995315ee00fSMartin Matuska * which time the log block we are writing will be obsolete, so we can 1996315ee00fSMartin Matuska * skip waiting and just return error here instead. 1997315ee00fSMartin Matuska */ 1998315ee00fSMartin Matuska if (dmu_tx_assign(tx, TXG_NOWAIT | TXG_NOTHROTTLE) != 0) { 1999eda14cbcSMatt Macy dmu_tx_abort(tx); 2000eda14cbcSMatt Macy /* Make zl_get_data do txg_waited_synced() */ 2001eda14cbcSMatt Macy return (SET_ERROR(EIO)); 2002eda14cbcSMatt Macy } 2003eda14cbcSMatt Macy 2004eda14cbcSMatt Macy /* 2005eda14cbcSMatt Macy * In order to prevent the zgd's lwb from being free'd prior to 2006eda14cbcSMatt Macy * dmu_sync_late_arrival_done() being called, we have to ensure 2007eda14cbcSMatt Macy * the lwb's "max txg" takes this tx's txg into account. 2008eda14cbcSMatt Macy */ 2009eda14cbcSMatt Macy zil_lwb_add_txg(zgd->zgd_lwb, dmu_tx_get_txg(tx)); 2010eda14cbcSMatt Macy 2011eda14cbcSMatt Macy dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP); 2012eda14cbcSMatt Macy dsa->dsa_dr = NULL; 2013eda14cbcSMatt Macy dsa->dsa_done = done; 2014eda14cbcSMatt Macy dsa->dsa_zgd = zgd; 2015eda14cbcSMatt Macy dsa->dsa_tx = tx; 2016eda14cbcSMatt Macy 2017eda14cbcSMatt Macy /* 2018eda14cbcSMatt Macy * Since we are currently syncing this txg, it's nontrivial to 2019eda14cbcSMatt Macy * determine what BP to nopwrite against, so we disable nopwrite. 2020eda14cbcSMatt Macy * 2021eda14cbcSMatt Macy * When syncing, the db_blkptr is initially the BP of the previous 2022eda14cbcSMatt Macy * txg. We can not nopwrite against it because it will be changed 2023eda14cbcSMatt Macy * (this is similar to the non-late-arrival case where the dbuf is 2024eda14cbcSMatt Macy * dirty in a future txg). 2025eda14cbcSMatt Macy * 2026eda14cbcSMatt Macy * Then dbuf_write_ready() sets bp_blkptr to the location we will write. 2027eda14cbcSMatt Macy * We can not nopwrite against it because although the BP will not 2028eda14cbcSMatt Macy * (typically) be changed, the data has not yet been persisted to this 2029eda14cbcSMatt Macy * location. 2030eda14cbcSMatt Macy * 2031eda14cbcSMatt Macy * Finally, when dbuf_write_done() is called, it is theoretically 2032eda14cbcSMatt Macy * possible to always nopwrite, because the data that was written in 2033eda14cbcSMatt Macy * this txg is the same data that we are trying to write. However we 2034eda14cbcSMatt Macy * would need to check that this dbuf is not dirty in any future 2035eda14cbcSMatt Macy * txg's (as we do in the normal dmu_sync() path). For simplicity, we 2036eda14cbcSMatt Macy * don't nopwrite in this case. 2037eda14cbcSMatt Macy */ 2038eda14cbcSMatt Macy zp->zp_nopwrite = B_FALSE; 2039eda14cbcSMatt Macy 2040eda14cbcSMatt Macy zio_nowait(zio_write(pio, os->os_spa, dmu_tx_get_txg(tx), zgd->zgd_bp, 2041eda14cbcSMatt Macy abd_get_from_buf(zgd->zgd_db->db_data, zgd->zgd_db->db_size), 2042eda14cbcSMatt Macy zgd->zgd_db->db_size, zgd->zgd_db->db_size, zp, 2043b7198dcfSMartin Matuska dmu_sync_late_arrival_ready, NULL, dmu_sync_late_arrival_done, 2044eda14cbcSMatt Macy dsa, ZIO_PRIORITY_SYNC_WRITE, ZIO_FLAG_CANFAIL, zb)); 2045eda14cbcSMatt Macy 2046eda14cbcSMatt Macy return (0); 2047eda14cbcSMatt Macy } 2048eda14cbcSMatt Macy 2049eda14cbcSMatt Macy /* 2050eda14cbcSMatt Macy * Intent log support: sync the block associated with db to disk. 2051eda14cbcSMatt Macy * N.B. and XXX: the caller is responsible for making sure that the 2052eda14cbcSMatt Macy * data isn't changing while dmu_sync() is writing it. 2053eda14cbcSMatt Macy * 2054eda14cbcSMatt Macy * Return values: 2055eda14cbcSMatt Macy * 2056eda14cbcSMatt Macy * EEXIST: this txg has already been synced, so there's nothing to do. 2057eda14cbcSMatt Macy * The caller should not log the write. 2058eda14cbcSMatt Macy * 2059eda14cbcSMatt Macy * ENOENT: the block was dbuf_free_range()'d, so there's nothing to do. 2060eda14cbcSMatt Macy * The caller should not log the write. 2061eda14cbcSMatt Macy * 2062eda14cbcSMatt Macy * EALREADY: this block is already in the process of being synced. 2063eda14cbcSMatt Macy * The caller should track its progress (somehow). 2064eda14cbcSMatt Macy * 2065eda14cbcSMatt Macy * EIO: could not do the I/O. 2066eda14cbcSMatt Macy * The caller should do a txg_wait_synced(). 2067eda14cbcSMatt Macy * 2068eda14cbcSMatt Macy * 0: the I/O has been initiated. 2069eda14cbcSMatt Macy * The caller should log this blkptr in the done callback. 2070eda14cbcSMatt Macy * It is possible that the I/O will fail, in which case 2071eda14cbcSMatt Macy * the error will be reported to the done callback and 2072eda14cbcSMatt Macy * propagated to pio from zio_done(). 2073eda14cbcSMatt Macy */ 2074eda14cbcSMatt Macy int 2075eda14cbcSMatt Macy dmu_sync(zio_t *pio, uint64_t txg, dmu_sync_cb_t *done, zgd_t *zgd) 2076eda14cbcSMatt Macy { 2077eda14cbcSMatt Macy dmu_buf_impl_t *db = (dmu_buf_impl_t *)zgd->zgd_db; 2078eda14cbcSMatt Macy objset_t *os = db->db_objset; 2079eda14cbcSMatt Macy dsl_dataset_t *ds = os->os_dsl_dataset; 2080eda14cbcSMatt Macy dbuf_dirty_record_t *dr, *dr_next; 2081eda14cbcSMatt Macy dmu_sync_arg_t *dsa; 2082eda14cbcSMatt Macy zbookmark_phys_t zb; 2083eda14cbcSMatt Macy zio_prop_t zp; 2084eda14cbcSMatt Macy 2085eda14cbcSMatt Macy ASSERT(pio != NULL); 2086eda14cbcSMatt Macy ASSERT(txg != 0); 2087eda14cbcSMatt Macy 2088eda14cbcSMatt Macy SET_BOOKMARK(&zb, ds->ds_object, 2089eda14cbcSMatt Macy db->db.db_object, db->db_level, db->db_blkid); 2090eda14cbcSMatt Macy 2091eda14cbcSMatt Macy DB_DNODE_ENTER(db); 2092ce4dcb97SMartin Matuska dmu_write_policy(os, DB_DNODE(db), db->db_level, WP_DMU_SYNC, &zp); 2093eda14cbcSMatt Macy DB_DNODE_EXIT(db); 2094eda14cbcSMatt Macy 2095eda14cbcSMatt Macy /* 2096eda14cbcSMatt Macy * If we're frozen (running ziltest), we always need to generate a bp. 2097eda14cbcSMatt Macy */ 2098eda14cbcSMatt Macy if (txg > spa_freeze_txg(os->os_spa)) 2099eda14cbcSMatt Macy return (dmu_sync_late_arrival(pio, os, done, zgd, &zp, &zb)); 2100eda14cbcSMatt Macy 2101eda14cbcSMatt Macy /* 2102eda14cbcSMatt Macy * Grabbing db_mtx now provides a barrier between dbuf_sync_leaf() 2103eda14cbcSMatt Macy * and us. If we determine that this txg is not yet syncing, 2104eda14cbcSMatt Macy * but it begins to sync a moment later, that's OK because the 2105eda14cbcSMatt Macy * sync thread will block in dbuf_sync_leaf() until we drop db_mtx. 2106eda14cbcSMatt Macy */ 2107eda14cbcSMatt Macy mutex_enter(&db->db_mtx); 2108eda14cbcSMatt Macy 2109eda14cbcSMatt Macy if (txg <= spa_last_synced_txg(os->os_spa)) { 2110eda14cbcSMatt Macy /* 2111eda14cbcSMatt Macy * This txg has already synced. There's nothing to do. 2112eda14cbcSMatt Macy */ 2113eda14cbcSMatt Macy mutex_exit(&db->db_mtx); 2114eda14cbcSMatt Macy return (SET_ERROR(EEXIST)); 2115eda14cbcSMatt Macy } 2116eda14cbcSMatt Macy 2117eda14cbcSMatt Macy if (txg <= spa_syncing_txg(os->os_spa)) { 2118eda14cbcSMatt Macy /* 2119eda14cbcSMatt Macy * This txg is currently syncing, so we can't mess with 2120eda14cbcSMatt Macy * the dirty record anymore; just write a new log block. 2121eda14cbcSMatt Macy */ 2122eda14cbcSMatt Macy mutex_exit(&db->db_mtx); 2123eda14cbcSMatt Macy return (dmu_sync_late_arrival(pio, os, done, zgd, &zp, &zb)); 2124eda14cbcSMatt Macy } 2125eda14cbcSMatt Macy 2126eda14cbcSMatt Macy dr = dbuf_find_dirty_eq(db, txg); 2127eda14cbcSMatt Macy 2128eda14cbcSMatt Macy if (dr == NULL) { 2129eda14cbcSMatt Macy /* 2130eda14cbcSMatt Macy * There's no dr for this dbuf, so it must have been freed. 2131eda14cbcSMatt Macy * There's no need to log writes to freed blocks, so we're done. 2132eda14cbcSMatt Macy */ 2133eda14cbcSMatt Macy mutex_exit(&db->db_mtx); 2134eda14cbcSMatt Macy return (SET_ERROR(ENOENT)); 2135eda14cbcSMatt Macy } 2136eda14cbcSMatt Macy 2137eda14cbcSMatt Macy dr_next = list_next(&db->db_dirty_records, dr); 2138eda14cbcSMatt Macy ASSERT(dr_next == NULL || dr_next->dr_txg < txg); 2139eda14cbcSMatt Macy 2140eda14cbcSMatt Macy if (db->db_blkptr != NULL) { 2141eda14cbcSMatt Macy /* 2142eda14cbcSMatt Macy * We need to fill in zgd_bp with the current blkptr so that 2143eda14cbcSMatt Macy * the nopwrite code can check if we're writing the same 2144eda14cbcSMatt Macy * data that's already on disk. We can only nopwrite if we 2145eda14cbcSMatt Macy * are sure that after making the copy, db_blkptr will not 2146eda14cbcSMatt Macy * change until our i/o completes. We ensure this by 2147eda14cbcSMatt Macy * holding the db_mtx, and only allowing nopwrite if the 2148eda14cbcSMatt Macy * block is not already dirty (see below). This is verified 2149eda14cbcSMatt Macy * by dmu_sync_done(), which VERIFYs that the db_blkptr has 2150eda14cbcSMatt Macy * not changed. 2151eda14cbcSMatt Macy */ 2152eda14cbcSMatt Macy *zgd->zgd_bp = *db->db_blkptr; 2153eda14cbcSMatt Macy } 2154eda14cbcSMatt Macy 2155eda14cbcSMatt Macy /* 2156eda14cbcSMatt Macy * Assume the on-disk data is X, the current syncing data (in 2157eda14cbcSMatt Macy * txg - 1) is Y, and the current in-memory data is Z (currently 2158eda14cbcSMatt Macy * in dmu_sync). 2159eda14cbcSMatt Macy * 2160eda14cbcSMatt Macy * We usually want to perform a nopwrite if X and Z are the 2161eda14cbcSMatt Macy * same. However, if Y is different (i.e. the BP is going to 2162eda14cbcSMatt Macy * change before this write takes effect), then a nopwrite will 2163eda14cbcSMatt Macy * be incorrect - we would override with X, which could have 2164eda14cbcSMatt Macy * been freed when Y was written. 2165eda14cbcSMatt Macy * 2166eda14cbcSMatt Macy * (Note that this is not a concern when we are nop-writing from 2167eda14cbcSMatt Macy * syncing context, because X and Y must be identical, because 2168eda14cbcSMatt Macy * all previous txgs have been synced.) 2169eda14cbcSMatt Macy * 2170eda14cbcSMatt Macy * Therefore, we disable nopwrite if the current BP could change 2171eda14cbcSMatt Macy * before this TXG. There are two ways it could change: by 2172eda14cbcSMatt Macy * being dirty (dr_next is non-NULL), or by being freed 2173eda14cbcSMatt Macy * (dnode_block_freed()). This behavior is verified by 2174eda14cbcSMatt Macy * zio_done(), which VERIFYs that the override BP is identical 2175eda14cbcSMatt Macy * to the on-disk BP. 2176eda14cbcSMatt Macy */ 2177ce4dcb97SMartin Matuska if (dr_next != NULL) { 2178ce4dcb97SMartin Matuska zp.zp_nopwrite = B_FALSE; 2179ce4dcb97SMartin Matuska } else { 2180eda14cbcSMatt Macy DB_DNODE_ENTER(db); 2181ce4dcb97SMartin Matuska if (dnode_block_freed(DB_DNODE(db), db->db_blkid)) 2182eda14cbcSMatt Macy zp.zp_nopwrite = B_FALSE; 2183eda14cbcSMatt Macy DB_DNODE_EXIT(db); 2184ce4dcb97SMartin Matuska } 2185eda14cbcSMatt Macy 2186eda14cbcSMatt Macy ASSERT(dr->dr_txg == txg); 2187eda14cbcSMatt Macy if (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC || 2188eda14cbcSMatt Macy dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { 2189eda14cbcSMatt Macy /* 2190eda14cbcSMatt Macy * We have already issued a sync write for this buffer, 2191eda14cbcSMatt Macy * or this buffer has already been synced. It could not 2192eda14cbcSMatt Macy * have been dirtied since, or we would have cleared the state. 2193eda14cbcSMatt Macy */ 2194eda14cbcSMatt Macy mutex_exit(&db->db_mtx); 2195eda14cbcSMatt Macy return (SET_ERROR(EALREADY)); 2196eda14cbcSMatt Macy } 2197eda14cbcSMatt Macy 21985c65a0a9SMartin Matuska ASSERT0(dr->dt.dl.dr_has_raw_params); 2199eda14cbcSMatt Macy ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); 2200eda14cbcSMatt Macy dr->dt.dl.dr_override_state = DR_IN_DMU_SYNC; 2201eda14cbcSMatt Macy mutex_exit(&db->db_mtx); 2202eda14cbcSMatt Macy 2203eda14cbcSMatt Macy dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP); 2204eda14cbcSMatt Macy dsa->dsa_dr = dr; 2205eda14cbcSMatt Macy dsa->dsa_done = done; 2206eda14cbcSMatt Macy dsa->dsa_zgd = zgd; 2207eda14cbcSMatt Macy dsa->dsa_tx = NULL; 2208eda14cbcSMatt Macy 220915f0b8c3SMartin Matuska zio_nowait(arc_write(pio, os->os_spa, txg, zgd->zgd_bp, 22107a7741afSMartin Matuska dr->dt.dl.dr_data, !DBUF_IS_CACHEABLE(db), 22117a7741afSMartin Matuska dbuf_is_l2cacheable(db, NULL), &zp, dmu_sync_ready, NULL, 22127a7741afSMartin Matuska dmu_sync_done, dsa, ZIO_PRIORITY_SYNC_WRITE, ZIO_FLAG_CANFAIL, 22137a7741afSMartin Matuska &zb)); 2214eda14cbcSMatt Macy 2215eda14cbcSMatt Macy return (0); 2216eda14cbcSMatt Macy } 2217eda14cbcSMatt Macy 2218eda14cbcSMatt Macy int 2219eda14cbcSMatt Macy dmu_object_set_nlevels(objset_t *os, uint64_t object, int nlevels, dmu_tx_t *tx) 2220eda14cbcSMatt Macy { 2221eda14cbcSMatt Macy dnode_t *dn; 2222eda14cbcSMatt Macy int err; 2223eda14cbcSMatt Macy 2224eda14cbcSMatt Macy err = dnode_hold(os, object, FTAG, &dn); 2225eda14cbcSMatt Macy if (err) 2226eda14cbcSMatt Macy return (err); 2227eda14cbcSMatt Macy err = dnode_set_nlevels(dn, nlevels, tx); 2228eda14cbcSMatt Macy dnode_rele(dn, FTAG); 2229eda14cbcSMatt Macy return (err); 2230eda14cbcSMatt Macy } 2231eda14cbcSMatt Macy 2232eda14cbcSMatt Macy int 2233eda14cbcSMatt Macy dmu_object_set_blocksize(objset_t *os, uint64_t object, uint64_t size, int ibs, 2234eda14cbcSMatt Macy dmu_tx_t *tx) 2235eda14cbcSMatt Macy { 2236eda14cbcSMatt Macy dnode_t *dn; 2237eda14cbcSMatt Macy int err; 2238eda14cbcSMatt Macy 2239eda14cbcSMatt Macy err = dnode_hold(os, object, FTAG, &dn); 2240eda14cbcSMatt Macy if (err) 2241eda14cbcSMatt Macy return (err); 2242eda14cbcSMatt Macy err = dnode_set_blksz(dn, size, ibs, tx); 2243eda14cbcSMatt Macy dnode_rele(dn, FTAG); 2244eda14cbcSMatt Macy return (err); 2245eda14cbcSMatt Macy } 2246eda14cbcSMatt Macy 2247eda14cbcSMatt Macy int 2248eda14cbcSMatt Macy dmu_object_set_maxblkid(objset_t *os, uint64_t object, uint64_t maxblkid, 2249eda14cbcSMatt Macy dmu_tx_t *tx) 2250eda14cbcSMatt Macy { 2251eda14cbcSMatt Macy dnode_t *dn; 2252eda14cbcSMatt Macy int err; 2253eda14cbcSMatt Macy 2254eda14cbcSMatt Macy err = dnode_hold(os, object, FTAG, &dn); 2255eda14cbcSMatt Macy if (err) 2256eda14cbcSMatt Macy return (err); 2257eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 2258eda14cbcSMatt Macy dnode_new_blkid(dn, maxblkid, tx, B_FALSE, B_TRUE); 2259eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock); 2260eda14cbcSMatt Macy dnode_rele(dn, FTAG); 2261eda14cbcSMatt Macy return (0); 2262eda14cbcSMatt Macy } 2263eda14cbcSMatt Macy 2264eda14cbcSMatt Macy void 2265eda14cbcSMatt Macy dmu_object_set_checksum(objset_t *os, uint64_t object, uint8_t checksum, 2266eda14cbcSMatt Macy dmu_tx_t *tx) 2267eda14cbcSMatt Macy { 2268eda14cbcSMatt Macy dnode_t *dn; 2269eda14cbcSMatt Macy 2270eda14cbcSMatt Macy /* 2271eda14cbcSMatt Macy * Send streams include each object's checksum function. This 2272eda14cbcSMatt Macy * check ensures that the receiving system can understand the 2273eda14cbcSMatt Macy * checksum function transmitted. 2274eda14cbcSMatt Macy */ 2275eda14cbcSMatt Macy ASSERT3U(checksum, <, ZIO_CHECKSUM_LEGACY_FUNCTIONS); 2276eda14cbcSMatt Macy 2277eda14cbcSMatt Macy VERIFY0(dnode_hold(os, object, FTAG, &dn)); 2278eda14cbcSMatt Macy ASSERT3U(checksum, <, ZIO_CHECKSUM_FUNCTIONS); 2279eda14cbcSMatt Macy dn->dn_checksum = checksum; 2280eda14cbcSMatt Macy dnode_setdirty(dn, tx); 2281eda14cbcSMatt Macy dnode_rele(dn, FTAG); 2282eda14cbcSMatt Macy } 2283eda14cbcSMatt Macy 2284eda14cbcSMatt Macy void 2285eda14cbcSMatt Macy dmu_object_set_compress(objset_t *os, uint64_t object, uint8_t compress, 2286eda14cbcSMatt Macy dmu_tx_t *tx) 2287eda14cbcSMatt Macy { 2288eda14cbcSMatt Macy dnode_t *dn; 2289eda14cbcSMatt Macy 2290eda14cbcSMatt Macy /* 2291eda14cbcSMatt Macy * Send streams include each object's compression function. This 2292eda14cbcSMatt Macy * check ensures that the receiving system can understand the 2293eda14cbcSMatt Macy * compression function transmitted. 2294eda14cbcSMatt Macy */ 2295eda14cbcSMatt Macy ASSERT3U(compress, <, ZIO_COMPRESS_LEGACY_FUNCTIONS); 2296eda14cbcSMatt Macy 2297eda14cbcSMatt Macy VERIFY0(dnode_hold(os, object, FTAG, &dn)); 2298eda14cbcSMatt Macy dn->dn_compress = compress; 2299eda14cbcSMatt Macy dnode_setdirty(dn, tx); 2300eda14cbcSMatt Macy dnode_rele(dn, FTAG); 2301eda14cbcSMatt Macy } 2302eda14cbcSMatt Macy 2303eda14cbcSMatt Macy /* 2304eda14cbcSMatt Macy * When the "redundant_metadata" property is set to "most", only indirect 2305eda14cbcSMatt Macy * blocks of this level and higher will have an additional ditto block. 2306eda14cbcSMatt Macy */ 2307e92ffd9bSMartin Matuska static const int zfs_redundant_metadata_most_ditto_level = 2; 2308eda14cbcSMatt Macy 2309eda14cbcSMatt Macy void 2310eda14cbcSMatt Macy dmu_write_policy(objset_t *os, dnode_t *dn, int level, int wp, zio_prop_t *zp) 2311eda14cbcSMatt Macy { 2312eda14cbcSMatt Macy dmu_object_type_t type = dn ? dn->dn_type : DMU_OT_OBJSET; 2313eda14cbcSMatt Macy boolean_t ismd = (level > 0 || DMU_OT_IS_METADATA(type) || 2314eda14cbcSMatt Macy (wp & WP_SPILL)); 2315eda14cbcSMatt Macy enum zio_checksum checksum = os->os_checksum; 2316eda14cbcSMatt Macy enum zio_compress compress = os->os_compress; 2317eda14cbcSMatt Macy uint8_t complevel = os->os_complevel; 2318eda14cbcSMatt Macy enum zio_checksum dedup_checksum = os->os_dedup_checksum; 2319eda14cbcSMatt Macy boolean_t dedup = B_FALSE; 2320eda14cbcSMatt Macy boolean_t nopwrite = B_FALSE; 2321eda14cbcSMatt Macy boolean_t dedup_verify = os->os_dedup_verify; 2322eda14cbcSMatt Macy boolean_t encrypt = B_FALSE; 2323eda14cbcSMatt Macy int copies = os->os_copies; 2324eda14cbcSMatt Macy 2325eda14cbcSMatt Macy /* 2326eda14cbcSMatt Macy * We maintain different write policies for each of the following 2327eda14cbcSMatt Macy * types of data: 2328eda14cbcSMatt Macy * 1. metadata 2329eda14cbcSMatt Macy * 2. preallocated blocks (i.e. level-0 blocks of a dump device) 2330eda14cbcSMatt Macy * 3. all other level 0 blocks 2331eda14cbcSMatt Macy */ 2332eda14cbcSMatt Macy if (ismd) { 2333eda14cbcSMatt Macy /* 2334eda14cbcSMatt Macy * XXX -- we should design a compression algorithm 2335eda14cbcSMatt Macy * that specializes in arrays of bps. 2336eda14cbcSMatt Macy */ 2337eda14cbcSMatt Macy compress = zio_compress_select(os->os_spa, 2338eda14cbcSMatt Macy ZIO_COMPRESS_ON, ZIO_COMPRESS_ON); 2339eda14cbcSMatt Macy 2340eda14cbcSMatt Macy /* 2341eda14cbcSMatt Macy * Metadata always gets checksummed. If the data 2342eda14cbcSMatt Macy * checksum is multi-bit correctable, and it's not a 2343eda14cbcSMatt Macy * ZBT-style checksum, then it's suitable for metadata 2344eda14cbcSMatt Macy * as well. Otherwise, the metadata checksum defaults 2345eda14cbcSMatt Macy * to fletcher4. 2346eda14cbcSMatt Macy */ 2347eda14cbcSMatt Macy if (!(zio_checksum_table[checksum].ci_flags & 2348eda14cbcSMatt Macy ZCHECKSUM_FLAG_METADATA) || 2349eda14cbcSMatt Macy (zio_checksum_table[checksum].ci_flags & 2350eda14cbcSMatt Macy ZCHECKSUM_FLAG_EMBEDDED)) 2351eda14cbcSMatt Macy checksum = ZIO_CHECKSUM_FLETCHER_4; 2352eda14cbcSMatt Macy 2353dbd5678dSMartin Matuska switch (os->os_redundant_metadata) { 2354dbd5678dSMartin Matuska case ZFS_REDUNDANT_METADATA_ALL: 2355eda14cbcSMatt Macy copies++; 2356dbd5678dSMartin Matuska break; 2357dbd5678dSMartin Matuska case ZFS_REDUNDANT_METADATA_MOST: 2358dbd5678dSMartin Matuska if (level >= zfs_redundant_metadata_most_ditto_level || 2359dbd5678dSMartin Matuska DMU_OT_IS_METADATA(type) || (wp & WP_SPILL)) 2360dbd5678dSMartin Matuska copies++; 2361dbd5678dSMartin Matuska break; 2362dbd5678dSMartin Matuska case ZFS_REDUNDANT_METADATA_SOME: 2363dbd5678dSMartin Matuska if (DMU_OT_IS_CRITICAL(type)) 2364dbd5678dSMartin Matuska copies++; 2365dbd5678dSMartin Matuska break; 2366dbd5678dSMartin Matuska case ZFS_REDUNDANT_METADATA_NONE: 2367dbd5678dSMartin Matuska break; 2368dbd5678dSMartin Matuska } 2369e2df9bb4SMartin Matuska 2370e2df9bb4SMartin Matuska if (dmu_ddt_copies > 0) { 2371e2df9bb4SMartin Matuska /* 2372e2df9bb4SMartin Matuska * If this tuneable is set, and this is a write for a 2373e2df9bb4SMartin Matuska * dedup entry store (zap or log), then we treat it 2374e2df9bb4SMartin Matuska * something like ZFS_REDUNDANT_METADATA_MOST on a 2375e2df9bb4SMartin Matuska * regular dataset: this many copies, and one more for 2376e2df9bb4SMartin Matuska * "higher" indirect blocks. This specific exception is 2377e2df9bb4SMartin Matuska * necessary because dedup objects are stored in the 2378e2df9bb4SMartin Matuska * MOS, which always has the highest possible copies. 2379e2df9bb4SMartin Matuska */ 2380e2df9bb4SMartin Matuska dmu_object_type_t stype = 2381e2df9bb4SMartin Matuska dn ? dn->dn_storage_type : DMU_OT_NONE; 2382e2df9bb4SMartin Matuska if (stype == DMU_OT_NONE) 2383e2df9bb4SMartin Matuska stype = type; 2384e2df9bb4SMartin Matuska if (stype == DMU_OT_DDT_ZAP) { 2385e2df9bb4SMartin Matuska copies = dmu_ddt_copies; 2386e2df9bb4SMartin Matuska if (level >= 2387e2df9bb4SMartin Matuska zfs_redundant_metadata_most_ditto_level) 2388e2df9bb4SMartin Matuska copies++; 2389e2df9bb4SMartin Matuska } 2390e2df9bb4SMartin Matuska } 2391eda14cbcSMatt Macy } else if (wp & WP_NOFILL) { 2392eda14cbcSMatt Macy ASSERT(level == 0); 2393eda14cbcSMatt Macy 2394eda14cbcSMatt Macy /* 2395eda14cbcSMatt Macy * If we're writing preallocated blocks, we aren't actually 2396eda14cbcSMatt Macy * writing them so don't set any policy properties. These 2397eda14cbcSMatt Macy * blocks are currently only used by an external subsystem 2398eda14cbcSMatt Macy * outside of zfs (i.e. dump) and not written by the zio 2399eda14cbcSMatt Macy * pipeline. 2400eda14cbcSMatt Macy */ 2401eda14cbcSMatt Macy compress = ZIO_COMPRESS_OFF; 2402eda14cbcSMatt Macy checksum = ZIO_CHECKSUM_OFF; 2403eda14cbcSMatt Macy } else { 2404eda14cbcSMatt Macy compress = zio_compress_select(os->os_spa, dn->dn_compress, 2405eda14cbcSMatt Macy compress); 2406eda14cbcSMatt Macy complevel = zio_complevel_select(os->os_spa, compress, 2407eda14cbcSMatt Macy complevel, complevel); 2408eda14cbcSMatt Macy 2409eda14cbcSMatt Macy checksum = (dedup_checksum == ZIO_CHECKSUM_OFF) ? 2410eda14cbcSMatt Macy zio_checksum_select(dn->dn_checksum, checksum) : 2411eda14cbcSMatt Macy dedup_checksum; 2412eda14cbcSMatt Macy 2413eda14cbcSMatt Macy /* 2414eda14cbcSMatt Macy * Determine dedup setting. If we are in dmu_sync(), 2415eda14cbcSMatt Macy * we won't actually dedup now because that's all 2416eda14cbcSMatt Macy * done in syncing context; but we do want to use the 2417eda14cbcSMatt Macy * dedup checksum. If the checksum is not strong 2418eda14cbcSMatt Macy * enough to ensure unique signatures, force 2419eda14cbcSMatt Macy * dedup_verify. 2420eda14cbcSMatt Macy */ 2421eda14cbcSMatt Macy if (dedup_checksum != ZIO_CHECKSUM_OFF) { 2422eda14cbcSMatt Macy dedup = (wp & WP_DMU_SYNC) ? B_FALSE : B_TRUE; 2423eda14cbcSMatt Macy if (!(zio_checksum_table[checksum].ci_flags & 2424eda14cbcSMatt Macy ZCHECKSUM_FLAG_DEDUP)) 2425eda14cbcSMatt Macy dedup_verify = B_TRUE; 2426eda14cbcSMatt Macy } 2427eda14cbcSMatt Macy 2428eda14cbcSMatt Macy /* 2429eda14cbcSMatt Macy * Enable nopwrite if we have secure enough checksum 2430eda14cbcSMatt Macy * algorithm (see comment in zio_nop_write) and 2431eda14cbcSMatt Macy * compression is enabled. We don't enable nopwrite if 2432eda14cbcSMatt Macy * dedup is enabled as the two features are mutually 2433eda14cbcSMatt Macy * exclusive. 2434eda14cbcSMatt Macy */ 2435eda14cbcSMatt Macy nopwrite = (!dedup && (zio_checksum_table[checksum].ci_flags & 2436eda14cbcSMatt Macy ZCHECKSUM_FLAG_NOPWRITE) && 2437eda14cbcSMatt Macy compress != ZIO_COMPRESS_OFF && zfs_nopwrite_enabled); 2438eda14cbcSMatt Macy } 2439eda14cbcSMatt Macy 2440eda14cbcSMatt Macy /* 2441eda14cbcSMatt Macy * All objects in an encrypted objset are protected from modification 2442eda14cbcSMatt Macy * via a MAC. Encrypted objects store their IV and salt in the last DVA 2443eda14cbcSMatt Macy * in the bp, so we cannot use all copies. Encrypted objects are also 2444eda14cbcSMatt Macy * not subject to nopwrite since writing the same data will still 2445eda14cbcSMatt Macy * result in a new ciphertext. Only encrypted blocks can be dedup'd 2446eda14cbcSMatt Macy * to avoid ambiguity in the dedup code since the DDT does not store 2447eda14cbcSMatt Macy * object types. 2448eda14cbcSMatt Macy */ 2449eda14cbcSMatt Macy if (os->os_encrypted && (wp & WP_NOFILL) == 0) { 2450eda14cbcSMatt Macy encrypt = B_TRUE; 2451eda14cbcSMatt Macy 2452eda14cbcSMatt Macy if (DMU_OT_IS_ENCRYPTED(type)) { 2453eda14cbcSMatt Macy copies = MIN(copies, SPA_DVAS_PER_BP - 1); 2454eda14cbcSMatt Macy nopwrite = B_FALSE; 2455eda14cbcSMatt Macy } else { 2456eda14cbcSMatt Macy dedup = B_FALSE; 2457eda14cbcSMatt Macy } 2458eda14cbcSMatt Macy 2459eda14cbcSMatt Macy if (level <= 0 && 2460eda14cbcSMatt Macy (type == DMU_OT_DNODE || type == DMU_OT_OBJSET)) { 2461eda14cbcSMatt Macy compress = ZIO_COMPRESS_EMPTY; 2462eda14cbcSMatt Macy } 2463eda14cbcSMatt Macy } 2464eda14cbcSMatt Macy 2465eda14cbcSMatt Macy zp->zp_compress = compress; 2466eda14cbcSMatt Macy zp->zp_complevel = complevel; 2467eda14cbcSMatt Macy zp->zp_checksum = checksum; 2468eda14cbcSMatt Macy zp->zp_type = (wp & WP_SPILL) ? dn->dn_bonustype : type; 2469eda14cbcSMatt Macy zp->zp_level = level; 2470eda14cbcSMatt Macy zp->zp_copies = MIN(copies, spa_max_replication(os->os_spa)); 2471eda14cbcSMatt Macy zp->zp_dedup = dedup; 2472eda14cbcSMatt Macy zp->zp_dedup_verify = dedup && dedup_verify; 2473eda14cbcSMatt Macy zp->zp_nopwrite = nopwrite; 2474eda14cbcSMatt Macy zp->zp_encrypt = encrypt; 2475eda14cbcSMatt Macy zp->zp_byteorder = ZFS_HOST_BYTEORDER; 24767a7741afSMartin Matuska zp->zp_direct_write = (wp & WP_DIRECT_WR) ? B_TRUE : B_FALSE; 2477da5137abSMartin Matuska memset(zp->zp_salt, 0, ZIO_DATA_SALT_LEN); 2478da5137abSMartin Matuska memset(zp->zp_iv, 0, ZIO_DATA_IV_LEN); 2479da5137abSMartin Matuska memset(zp->zp_mac, 0, ZIO_DATA_MAC_LEN); 2480eda14cbcSMatt Macy zp->zp_zpl_smallblk = DMU_OT_IS_FILE(zp->zp_type) ? 2481eda14cbcSMatt Macy os->os_zpl_special_smallblock : 0; 2482ce4dcb97SMartin Matuska zp->zp_storage_type = dn ? dn->dn_storage_type : DMU_OT_NONE; 2483eda14cbcSMatt Macy 2484eda14cbcSMatt Macy ASSERT3U(zp->zp_compress, !=, ZIO_COMPRESS_INHERIT); 2485eda14cbcSMatt Macy } 2486eda14cbcSMatt Macy 2487eda14cbcSMatt Macy /* 24882a58b312SMartin Matuska * Reports the location of data and holes in an object. In order to 24892a58b312SMartin Matuska * accurately report holes all dirty data must be synced to disk. This 24902a58b312SMartin Matuska * causes extremely poor performance when seeking for holes in a dirty file. 24912a58b312SMartin Matuska * As a compromise, only provide hole data when the dnode is clean. When 24922a58b312SMartin Matuska * a dnode is dirty report the dnode as having no holes by returning EBUSY 24932a58b312SMartin Matuska * which is always safe to do. 2494eda14cbcSMatt Macy */ 2495eda14cbcSMatt Macy int 2496eda14cbcSMatt Macy dmu_offset_next(objset_t *os, uint64_t object, boolean_t hole, uint64_t *off) 2497eda14cbcSMatt Macy { 2498eda14cbcSMatt Macy dnode_t *dn; 24992a58b312SMartin Matuska int restarted = 0, err; 2500eda14cbcSMatt Macy 250181b22a98SMartin Matuska restart: 2502eda14cbcSMatt Macy err = dnode_hold(os, object, FTAG, &dn); 2503eda14cbcSMatt Macy if (err) 2504eda14cbcSMatt Macy return (err); 2505eda14cbcSMatt Macy 250681b22a98SMartin Matuska rw_enter(&dn->dn_struct_rwlock, RW_READER); 2507eda14cbcSMatt Macy 250881b22a98SMartin Matuska if (dnode_is_dirty(dn)) { 2509eda14cbcSMatt Macy /* 251081b22a98SMartin Matuska * If the zfs_dmu_offset_next_sync module option is enabled 25112a58b312SMartin Matuska * then hole reporting has been requested. Dirty dnodes 25122a58b312SMartin Matuska * must be synced to disk to accurately report holes. 251381b22a98SMartin Matuska * 25142a58b312SMartin Matuska * Provided a RL_READER rangelock spanning 0-UINT64_MAX is 25152a58b312SMartin Matuska * held by the caller only a single restart will be required. 25162a58b312SMartin Matuska * We tolerate callers which do not hold the rangelock by 25172a58b312SMartin Matuska * returning EBUSY and not reporting holes after one restart. 2518eda14cbcSMatt Macy */ 251981b22a98SMartin Matuska if (zfs_dmu_offset_next_sync) { 252081b22a98SMartin Matuska rw_exit(&dn->dn_struct_rwlock); 2521eda14cbcSMatt Macy dnode_rele(dn, FTAG); 25222a58b312SMartin Matuska 25232a58b312SMartin Matuska if (restarted) 25242a58b312SMartin Matuska return (SET_ERROR(EBUSY)); 25252a58b312SMartin Matuska 2526eda14cbcSMatt Macy txg_wait_synced(dmu_objset_pool(os), 0); 25272a58b312SMartin Matuska restarted = 1; 252881b22a98SMartin Matuska goto restart; 2529eda14cbcSMatt Macy } 2530eda14cbcSMatt Macy 2531eda14cbcSMatt Macy err = SET_ERROR(EBUSY); 253281b22a98SMartin Matuska } else { 253381b22a98SMartin Matuska err = dnode_next_offset(dn, DNODE_FIND_HAVELOCK | 253481b22a98SMartin Matuska (hole ? DNODE_FIND_HOLE : 0), off, 1, 1, 0); 253581b22a98SMartin Matuska } 2536eda14cbcSMatt Macy 253781b22a98SMartin Matuska rw_exit(&dn->dn_struct_rwlock); 2538eda14cbcSMatt Macy dnode_rele(dn, FTAG); 2539eda14cbcSMatt Macy 2540eda14cbcSMatt Macy return (err); 2541eda14cbcSMatt Macy } 2542eda14cbcSMatt Macy 25432a58b312SMartin Matuska int 25442a58b312SMartin Matuska dmu_read_l0_bps(objset_t *os, uint64_t object, uint64_t offset, uint64_t length, 2545e639e0d2SMartin Matuska blkptr_t *bps, size_t *nbpsp) 25462a58b312SMartin Matuska { 25472a58b312SMartin Matuska dmu_buf_t **dbp, *dbuf; 25482a58b312SMartin Matuska dmu_buf_impl_t *db; 25492a58b312SMartin Matuska blkptr_t *bp; 25502a58b312SMartin Matuska int error, numbufs; 25512a58b312SMartin Matuska 25522a58b312SMartin Matuska error = dmu_buf_hold_array(os, object, offset, length, FALSE, FTAG, 25532a58b312SMartin Matuska &numbufs, &dbp); 25542a58b312SMartin Matuska if (error != 0) { 25552a58b312SMartin Matuska if (error == ESRCH) { 25562a58b312SMartin Matuska error = SET_ERROR(ENXIO); 25572a58b312SMartin Matuska } 25582a58b312SMartin Matuska return (error); 25592a58b312SMartin Matuska } 25602a58b312SMartin Matuska 25612a58b312SMartin Matuska ASSERT3U(numbufs, <=, *nbpsp); 25622a58b312SMartin Matuska 25632a58b312SMartin Matuska for (int i = 0; i < numbufs; i++) { 25642a58b312SMartin Matuska dbuf = dbp[i]; 25652a58b312SMartin Matuska db = (dmu_buf_impl_t *)dbuf; 25662a58b312SMartin Matuska 25672a58b312SMartin Matuska mutex_enter(&db->db_mtx); 25682a58b312SMartin Matuska 25692a58b312SMartin Matuska if (!list_is_empty(&db->db_dirty_records)) { 25702a58b312SMartin Matuska dbuf_dirty_record_t *dr; 25712a58b312SMartin Matuska 25722a58b312SMartin Matuska dr = list_head(&db->db_dirty_records); 25732a58b312SMartin Matuska if (dr->dt.dl.dr_brtwrite) { 25742a58b312SMartin Matuska /* 25752a58b312SMartin Matuska * This is very special case where we clone a 25762a58b312SMartin Matuska * block and in the same transaction group we 25772a58b312SMartin Matuska * read its BP (most likely to clone the clone). 25782a58b312SMartin Matuska */ 25792a58b312SMartin Matuska bp = &dr->dt.dl.dr_overridden_by; 25802a58b312SMartin Matuska } else { 25812a58b312SMartin Matuska /* 25822a58b312SMartin Matuska * The block was modified in the same 25832a58b312SMartin Matuska * transaction group. 25842a58b312SMartin Matuska */ 25852a58b312SMartin Matuska mutex_exit(&db->db_mtx); 25862a58b312SMartin Matuska error = SET_ERROR(EAGAIN); 25872a58b312SMartin Matuska goto out; 25882a58b312SMartin Matuska } 25892a58b312SMartin Matuska } else { 25902a58b312SMartin Matuska bp = db->db_blkptr; 25912a58b312SMartin Matuska } 25922a58b312SMartin Matuska 25932a58b312SMartin Matuska mutex_exit(&db->db_mtx); 25942a58b312SMartin Matuska 25952a58b312SMartin Matuska if (bp == NULL) { 25962a58b312SMartin Matuska /* 2597783d3ff6SMartin Matuska * The file size was increased, but the block was never 2598783d3ff6SMartin Matuska * written, otherwise we would either have the block 2599783d3ff6SMartin Matuska * pointer or the dirty record and would not get here. 2600783d3ff6SMartin Matuska * It is effectively a hole, so report it as such. 26012a58b312SMartin Matuska */ 2602783d3ff6SMartin Matuska BP_ZERO(&bps[i]); 2603783d3ff6SMartin Matuska continue; 26042a58b312SMartin Matuska } 26052a58b312SMartin Matuska /* 26062a58b312SMartin Matuska * Make sure we clone only data blocks. 26072a58b312SMartin Matuska */ 26082a58b312SMartin Matuska if (BP_IS_METADATA(bp) && !BP_IS_HOLE(bp)) { 26092a58b312SMartin Matuska error = SET_ERROR(EINVAL); 26102a58b312SMartin Matuska goto out; 26112a58b312SMartin Matuska } 26122a58b312SMartin Matuska 26133494f7c0SMartin Matuska /* 26143494f7c0SMartin Matuska * If the block was allocated in transaction group that is not 26153494f7c0SMartin Matuska * yet synced, we could clone it, but we couldn't write this 26163494f7c0SMartin Matuska * operation into ZIL, or it may be impossible to replay, since 26173494f7c0SMartin Matuska * the block may appear not yet allocated at that point. 26183494f7c0SMartin Matuska */ 2619783d3ff6SMartin Matuska if (BP_GET_BIRTH(bp) > spa_freeze_txg(os->os_spa)) { 26203494f7c0SMartin Matuska error = SET_ERROR(EINVAL); 26213494f7c0SMartin Matuska goto out; 26223494f7c0SMartin Matuska } 2623783d3ff6SMartin Matuska if (BP_GET_BIRTH(bp) > spa_last_synced_txg(os->os_spa)) { 26243494f7c0SMartin Matuska error = SET_ERROR(EAGAIN); 26253494f7c0SMartin Matuska goto out; 26263494f7c0SMartin Matuska } 26273494f7c0SMartin Matuska 26282a58b312SMartin Matuska bps[i] = *bp; 26292a58b312SMartin Matuska } 26302a58b312SMartin Matuska 26312a58b312SMartin Matuska *nbpsp = numbufs; 26322a58b312SMartin Matuska out: 26332a58b312SMartin Matuska dmu_buf_rele_array(dbp, numbufs, FTAG); 26342a58b312SMartin Matuska 26352a58b312SMartin Matuska return (error); 26362a58b312SMartin Matuska } 26372a58b312SMartin Matuska 2638e639e0d2SMartin Matuska int 26392a58b312SMartin Matuska dmu_brt_clone(objset_t *os, uint64_t object, uint64_t offset, uint64_t length, 2640525fe93dSMartin Matuska dmu_tx_t *tx, const blkptr_t *bps, size_t nbps) 26412a58b312SMartin Matuska { 26422a58b312SMartin Matuska spa_t *spa; 26432a58b312SMartin Matuska dmu_buf_t **dbp, *dbuf; 26442a58b312SMartin Matuska dmu_buf_impl_t *db; 26452a58b312SMartin Matuska struct dirty_leaf *dl; 26462a58b312SMartin Matuska dbuf_dirty_record_t *dr; 26472a58b312SMartin Matuska const blkptr_t *bp; 2648e639e0d2SMartin Matuska int error = 0, i, numbufs; 26492a58b312SMartin Matuska 26502a58b312SMartin Matuska spa = os->os_spa; 26512a58b312SMartin Matuska 26522a58b312SMartin Matuska VERIFY0(dmu_buf_hold_array(os, object, offset, length, FALSE, FTAG, 26532a58b312SMartin Matuska &numbufs, &dbp)); 26542a58b312SMartin Matuska ASSERT3U(nbps, ==, numbufs); 26552a58b312SMartin Matuska 2656e639e0d2SMartin Matuska /* 2657e639e0d2SMartin Matuska * Before we start cloning make sure that the dbufs sizes match new BPs 2658e639e0d2SMartin Matuska * sizes. If they don't, that's a no-go, as we are not able to shrink 2659e639e0d2SMartin Matuska * dbufs. 2660e639e0d2SMartin Matuska */ 2661e639e0d2SMartin Matuska for (i = 0; i < numbufs; i++) { 26622a58b312SMartin Matuska dbuf = dbp[i]; 26632a58b312SMartin Matuska db = (dmu_buf_impl_t *)dbuf; 26642a58b312SMartin Matuska bp = &bps[i]; 26652a58b312SMartin Matuska 26665c65a0a9SMartin Matuska ASSERT3U(db->db.db_object, !=, DMU_META_DNODE_OBJECT); 26672a58b312SMartin Matuska ASSERT0(db->db_level); 26682a58b312SMartin Matuska ASSERT(db->db_blkid != DMU_BONUS_BLKID); 2669e639e0d2SMartin Matuska ASSERT(db->db_blkid != DMU_SPILL_BLKID); 26702a58b312SMartin Matuska 2671e639e0d2SMartin Matuska if (!BP_IS_HOLE(bp) && BP_GET_LSIZE(bp) != dbuf->db_size) { 2672e639e0d2SMartin Matuska error = SET_ERROR(EXDEV); 2673e639e0d2SMartin Matuska goto out; 2674e639e0d2SMartin Matuska } 26752a58b312SMartin Matuska } 26762a58b312SMartin Matuska 2677e639e0d2SMartin Matuska for (i = 0; i < numbufs; i++) { 2678e639e0d2SMartin Matuska dbuf = dbp[i]; 2679e639e0d2SMartin Matuska db = (dmu_buf_impl_t *)dbuf; 2680e639e0d2SMartin Matuska bp = &bps[i]; 26812a58b312SMartin Matuska 26827a7741afSMartin Matuska dmu_buf_will_clone_or_dio(dbuf, tx); 26832a58b312SMartin Matuska 26842a58b312SMartin Matuska mutex_enter(&db->db_mtx); 26852a58b312SMartin Matuska 26862a58b312SMartin Matuska dr = list_head(&db->db_dirty_records); 26872a58b312SMartin Matuska VERIFY(dr != NULL); 26882a58b312SMartin Matuska ASSERT3U(dr->dr_txg, ==, tx->tx_txg); 26892a58b312SMartin Matuska dl = &dr->dt.dl; 26905c65a0a9SMartin Matuska ASSERT0(dl->dr_has_raw_params); 26912a58b312SMartin Matuska dl->dr_overridden_by = *bp; 2692783d3ff6SMartin Matuska if (!BP_IS_HOLE(bp) || BP_GET_LOGICAL_BIRTH(bp) != 0) { 2693783d3ff6SMartin Matuska if (!BP_IS_EMBEDDED(bp)) { 2694783d3ff6SMartin Matuska BP_SET_BIRTH(&dl->dr_overridden_by, dr->dr_txg, 2695783d3ff6SMartin Matuska BP_GET_BIRTH(bp)); 2696783d3ff6SMartin Matuska } else { 2697783d3ff6SMartin Matuska BP_SET_LOGICAL_BIRTH(&dl->dr_overridden_by, 2698783d3ff6SMartin Matuska dr->dr_txg); 2699783d3ff6SMartin Matuska } 2700783d3ff6SMartin Matuska } 27012a58b312SMartin Matuska dl->dr_brtwrite = B_TRUE; 27022a58b312SMartin Matuska dl->dr_override_state = DR_OVERRIDDEN; 27032a58b312SMartin Matuska 27042a58b312SMartin Matuska mutex_exit(&db->db_mtx); 27052a58b312SMartin Matuska 27062a58b312SMartin Matuska /* 27072a58b312SMartin Matuska * When data in embedded into BP there is no need to create 27082a58b312SMartin Matuska * BRT entry as there is no data block. Just copy the BP as 27092a58b312SMartin Matuska * it contains the data. 27102a58b312SMartin Matuska */ 2711525fe93dSMartin Matuska if (!BP_IS_HOLE(bp) && !BP_IS_EMBEDDED(bp)) { 27122a58b312SMartin Matuska brt_pending_add(spa, bp, tx); 27132a58b312SMartin Matuska } 27142a58b312SMartin Matuska } 2715e639e0d2SMartin Matuska out: 27162a58b312SMartin Matuska dmu_buf_rele_array(dbp, numbufs, FTAG); 2717e639e0d2SMartin Matuska 2718e639e0d2SMartin Matuska return (error); 27192a58b312SMartin Matuska } 27202a58b312SMartin Matuska 2721eda14cbcSMatt Macy void 2722eda14cbcSMatt Macy __dmu_object_info_from_dnode(dnode_t *dn, dmu_object_info_t *doi) 2723eda14cbcSMatt Macy { 2724eda14cbcSMatt Macy dnode_phys_t *dnp = dn->dn_phys; 2725eda14cbcSMatt Macy 2726eda14cbcSMatt Macy doi->doi_data_block_size = dn->dn_datablksz; 2727eda14cbcSMatt Macy doi->doi_metadata_block_size = dn->dn_indblkshift ? 2728eda14cbcSMatt Macy 1ULL << dn->dn_indblkshift : 0; 2729eda14cbcSMatt Macy doi->doi_type = dn->dn_type; 2730eda14cbcSMatt Macy doi->doi_bonus_type = dn->dn_bonustype; 2731eda14cbcSMatt Macy doi->doi_bonus_size = dn->dn_bonuslen; 2732eda14cbcSMatt Macy doi->doi_dnodesize = dn->dn_num_slots << DNODE_SHIFT; 2733eda14cbcSMatt Macy doi->doi_indirection = dn->dn_nlevels; 2734eda14cbcSMatt Macy doi->doi_checksum = dn->dn_checksum; 2735eda14cbcSMatt Macy doi->doi_compress = dn->dn_compress; 2736eda14cbcSMatt Macy doi->doi_nblkptr = dn->dn_nblkptr; 2737eda14cbcSMatt Macy doi->doi_physical_blocks_512 = (DN_USED_BYTES(dnp) + 256) >> 9; 2738eda14cbcSMatt Macy doi->doi_max_offset = (dn->dn_maxblkid + 1) * dn->dn_datablksz; 2739eda14cbcSMatt Macy doi->doi_fill_count = 0; 2740eda14cbcSMatt Macy for (int i = 0; i < dnp->dn_nblkptr; i++) 2741eda14cbcSMatt Macy doi->doi_fill_count += BP_GET_FILL(&dnp->dn_blkptr[i]); 2742eda14cbcSMatt Macy } 2743eda14cbcSMatt Macy 2744eda14cbcSMatt Macy void 2745eda14cbcSMatt Macy dmu_object_info_from_dnode(dnode_t *dn, dmu_object_info_t *doi) 2746eda14cbcSMatt Macy { 2747eda14cbcSMatt Macy rw_enter(&dn->dn_struct_rwlock, RW_READER); 2748eda14cbcSMatt Macy mutex_enter(&dn->dn_mtx); 2749eda14cbcSMatt Macy 2750eda14cbcSMatt Macy __dmu_object_info_from_dnode(dn, doi); 2751eda14cbcSMatt Macy 2752eda14cbcSMatt Macy mutex_exit(&dn->dn_mtx); 2753eda14cbcSMatt Macy rw_exit(&dn->dn_struct_rwlock); 2754eda14cbcSMatt Macy } 2755eda14cbcSMatt Macy 2756eda14cbcSMatt Macy /* 2757eda14cbcSMatt Macy * Get information on a DMU object. 2758eda14cbcSMatt Macy * If doi is NULL, just indicates whether the object exists. 2759eda14cbcSMatt Macy */ 2760eda14cbcSMatt Macy int 2761eda14cbcSMatt Macy dmu_object_info(objset_t *os, uint64_t object, dmu_object_info_t *doi) 2762eda14cbcSMatt Macy { 2763eda14cbcSMatt Macy dnode_t *dn; 2764eda14cbcSMatt Macy int err = dnode_hold(os, object, FTAG, &dn); 2765eda14cbcSMatt Macy 2766eda14cbcSMatt Macy if (err) 2767eda14cbcSMatt Macy return (err); 2768eda14cbcSMatt Macy 2769eda14cbcSMatt Macy if (doi != NULL) 2770eda14cbcSMatt Macy dmu_object_info_from_dnode(dn, doi); 2771eda14cbcSMatt Macy 2772eda14cbcSMatt Macy dnode_rele(dn, FTAG); 2773eda14cbcSMatt Macy return (0); 2774eda14cbcSMatt Macy } 2775eda14cbcSMatt Macy 2776eda14cbcSMatt Macy /* 2777eda14cbcSMatt Macy * As above, but faster; can be used when you have a held dbuf in hand. 2778eda14cbcSMatt Macy */ 2779eda14cbcSMatt Macy void 2780eda14cbcSMatt Macy dmu_object_info_from_db(dmu_buf_t *db_fake, dmu_object_info_t *doi) 2781eda14cbcSMatt Macy { 2782eda14cbcSMatt Macy dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2783eda14cbcSMatt Macy 2784eda14cbcSMatt Macy DB_DNODE_ENTER(db); 2785eda14cbcSMatt Macy dmu_object_info_from_dnode(DB_DNODE(db), doi); 2786eda14cbcSMatt Macy DB_DNODE_EXIT(db); 2787eda14cbcSMatt Macy } 2788eda14cbcSMatt Macy 2789eda14cbcSMatt Macy /* 2790eda14cbcSMatt Macy * Faster still when you only care about the size. 2791eda14cbcSMatt Macy */ 2792eda14cbcSMatt Macy void 2793eda14cbcSMatt Macy dmu_object_size_from_db(dmu_buf_t *db_fake, uint32_t *blksize, 2794eda14cbcSMatt Macy u_longlong_t *nblk512) 2795eda14cbcSMatt Macy { 2796eda14cbcSMatt Macy dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2797eda14cbcSMatt Macy dnode_t *dn; 2798eda14cbcSMatt Macy 2799eda14cbcSMatt Macy DB_DNODE_ENTER(db); 2800eda14cbcSMatt Macy dn = DB_DNODE(db); 2801eda14cbcSMatt Macy 2802eda14cbcSMatt Macy *blksize = dn->dn_datablksz; 2803eda14cbcSMatt Macy /* add in number of slots used for the dnode itself */ 2804eda14cbcSMatt Macy *nblk512 = ((DN_USED_BYTES(dn->dn_phys) + SPA_MINBLOCKSIZE/2) >> 2805eda14cbcSMatt Macy SPA_MINBLOCKSHIFT) + dn->dn_num_slots; 2806eda14cbcSMatt Macy DB_DNODE_EXIT(db); 2807eda14cbcSMatt Macy } 2808eda14cbcSMatt Macy 2809eda14cbcSMatt Macy void 2810eda14cbcSMatt Macy dmu_object_dnsize_from_db(dmu_buf_t *db_fake, int *dnsize) 2811eda14cbcSMatt Macy { 2812eda14cbcSMatt Macy dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2813eda14cbcSMatt Macy 2814eda14cbcSMatt Macy DB_DNODE_ENTER(db); 2815ce4dcb97SMartin Matuska *dnsize = DB_DNODE(db)->dn_num_slots << DNODE_SHIFT; 2816eda14cbcSMatt Macy DB_DNODE_EXIT(db); 2817eda14cbcSMatt Macy } 2818eda14cbcSMatt Macy 2819eda14cbcSMatt Macy void 2820eda14cbcSMatt Macy byteswap_uint64_array(void *vbuf, size_t size) 2821eda14cbcSMatt Macy { 2822eda14cbcSMatt Macy uint64_t *buf = vbuf; 2823eda14cbcSMatt Macy size_t count = size >> 3; 2824eda14cbcSMatt Macy int i; 2825eda14cbcSMatt Macy 2826eda14cbcSMatt Macy ASSERT((size & 7) == 0); 2827eda14cbcSMatt Macy 2828eda14cbcSMatt Macy for (i = 0; i < count; i++) 2829eda14cbcSMatt Macy buf[i] = BSWAP_64(buf[i]); 2830eda14cbcSMatt Macy } 2831eda14cbcSMatt Macy 2832eda14cbcSMatt Macy void 2833eda14cbcSMatt Macy byteswap_uint32_array(void *vbuf, size_t size) 2834eda14cbcSMatt Macy { 2835eda14cbcSMatt Macy uint32_t *buf = vbuf; 2836eda14cbcSMatt Macy size_t count = size >> 2; 2837eda14cbcSMatt Macy int i; 2838eda14cbcSMatt Macy 2839eda14cbcSMatt Macy ASSERT((size & 3) == 0); 2840eda14cbcSMatt Macy 2841eda14cbcSMatt Macy for (i = 0; i < count; i++) 2842eda14cbcSMatt Macy buf[i] = BSWAP_32(buf[i]); 2843eda14cbcSMatt Macy } 2844eda14cbcSMatt Macy 2845eda14cbcSMatt Macy void 2846eda14cbcSMatt Macy byteswap_uint16_array(void *vbuf, size_t size) 2847eda14cbcSMatt Macy { 2848eda14cbcSMatt Macy uint16_t *buf = vbuf; 2849eda14cbcSMatt Macy size_t count = size >> 1; 2850eda14cbcSMatt Macy int i; 2851eda14cbcSMatt Macy 2852eda14cbcSMatt Macy ASSERT((size & 1) == 0); 2853eda14cbcSMatt Macy 2854eda14cbcSMatt Macy for (i = 0; i < count; i++) 2855eda14cbcSMatt Macy buf[i] = BSWAP_16(buf[i]); 2856eda14cbcSMatt Macy } 2857eda14cbcSMatt Macy 2858eda14cbcSMatt Macy void 2859eda14cbcSMatt Macy byteswap_uint8_array(void *vbuf, size_t size) 2860eda14cbcSMatt Macy { 2861e92ffd9bSMartin Matuska (void) vbuf, (void) size; 2862eda14cbcSMatt Macy } 2863eda14cbcSMatt Macy 2864eda14cbcSMatt Macy void 2865eda14cbcSMatt Macy dmu_init(void) 2866eda14cbcSMatt Macy { 2867eda14cbcSMatt Macy abd_init(); 2868eda14cbcSMatt Macy zfs_dbgmsg_init(); 2869eda14cbcSMatt Macy sa_cache_init(); 2870eda14cbcSMatt Macy dmu_objset_init(); 2871eda14cbcSMatt Macy dnode_init(); 2872eda14cbcSMatt Macy zfetch_init(); 2873eda14cbcSMatt Macy dmu_tx_init(); 2874eda14cbcSMatt Macy l2arc_init(); 2875eda14cbcSMatt Macy arc_init(); 2876eda14cbcSMatt Macy dbuf_init(); 2877eda14cbcSMatt Macy } 2878eda14cbcSMatt Macy 2879eda14cbcSMatt Macy void 2880eda14cbcSMatt Macy dmu_fini(void) 2881eda14cbcSMatt Macy { 2882eda14cbcSMatt Macy arc_fini(); /* arc depends on l2arc, so arc must go first */ 2883eda14cbcSMatt Macy l2arc_fini(); 2884eda14cbcSMatt Macy dmu_tx_fini(); 2885eda14cbcSMatt Macy zfetch_fini(); 2886eda14cbcSMatt Macy dbuf_fini(); 2887eda14cbcSMatt Macy dnode_fini(); 2888eda14cbcSMatt Macy dmu_objset_fini(); 2889eda14cbcSMatt Macy sa_cache_fini(); 2890eda14cbcSMatt Macy zfs_dbgmsg_fini(); 2891eda14cbcSMatt Macy abd_fini(); 2892eda14cbcSMatt Macy } 2893eda14cbcSMatt Macy 2894eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_bonus_hold); 2895eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_bonus_hold_by_dnode); 2896eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_buf_hold_array_by_bonus); 2897eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_buf_rele_array); 2898eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_prefetch); 2899783d3ff6SMartin Matuska EXPORT_SYMBOL(dmu_prefetch_by_dnode); 2900783d3ff6SMartin Matuska EXPORT_SYMBOL(dmu_prefetch_dnode); 2901eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_free_range); 2902eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_free_long_range); 2903eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_free_long_object); 2904eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_read); 2905eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_read_by_dnode); 29067a7741afSMartin Matuska EXPORT_SYMBOL(dmu_read_uio); 29077a7741afSMartin Matuska EXPORT_SYMBOL(dmu_read_uio_dbuf); 29087a7741afSMartin Matuska EXPORT_SYMBOL(dmu_read_uio_dnode); 2909eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_write); 2910eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_write_by_dnode); 29117a7741afSMartin Matuska EXPORT_SYMBOL(dmu_write_by_dnode_flags); 29127a7741afSMartin Matuska EXPORT_SYMBOL(dmu_write_uio); 29137a7741afSMartin Matuska EXPORT_SYMBOL(dmu_write_uio_dbuf); 29147a7741afSMartin Matuska EXPORT_SYMBOL(dmu_write_uio_dnode); 2915eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_prealloc); 2916eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_object_info); 2917eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_object_info_from_dnode); 2918eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_object_info_from_db); 2919eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_object_size_from_db); 2920eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_object_dnsize_from_db); 2921eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_object_set_nlevels); 2922eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_object_set_blocksize); 2923eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_object_set_maxblkid); 2924eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_object_set_checksum); 2925eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_object_set_compress); 2926ac0bf12eSMatt Macy EXPORT_SYMBOL(dmu_offset_next); 2927eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_write_policy); 2928eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_sync); 2929eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_request_arcbuf); 2930eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_return_arcbuf); 2931eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_assign_arcbuf_by_dnode); 2932eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_assign_arcbuf_by_dbuf); 2933eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_buf_hold); 2934eda14cbcSMatt Macy EXPORT_SYMBOL(dmu_ot); 2935eda14cbcSMatt Macy 2936eda14cbcSMatt Macy ZFS_MODULE_PARAM(zfs, zfs_, nopwrite_enabled, INT, ZMOD_RW, 2937eda14cbcSMatt Macy "Enable NOP writes"); 2938eda14cbcSMatt Macy 2939dbd5678dSMartin Matuska ZFS_MODULE_PARAM(zfs, zfs_, per_txg_dirty_frees_percent, UINT, ZMOD_RW, 2940eda14cbcSMatt Macy "Percentage of dirtied blocks from frees in one TXG"); 2941eda14cbcSMatt Macy 2942eda14cbcSMatt Macy ZFS_MODULE_PARAM(zfs, zfs_, dmu_offset_next_sync, INT, ZMOD_RW, 2943eda14cbcSMatt Macy "Enable forcing txg sync to find holes"); 2944eda14cbcSMatt Macy 2945be181ee2SMartin Matuska ZFS_MODULE_PARAM(zfs, , dmu_prefetch_max, UINT, ZMOD_RW, 2946eda14cbcSMatt Macy "Limit one prefetch call to this size"); 2947e2df9bb4SMartin Matuska 2948e2df9bb4SMartin Matuska ZFS_MODULE_PARAM(zfs, , dmu_ddt_copies, UINT, ZMOD_RW, 2949e2df9bb4SMartin Matuska "Override copies= for dedup objects"); 2950