1 /* 2 * Copyright (c) 2020 iXsystems, Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 */ 27 28 #include <sys/types.h> 29 #include <sys/param.h> 30 #include <sys/dmu.h> 31 #include <sys/dmu_impl.h> 32 #include <sys/dmu_tx.h> 33 #include <sys/dbuf.h> 34 #include <sys/dnode.h> 35 #include <sys/zfs_context.h> 36 #include <sys/dmu_objset.h> 37 #include <sys/dmu_traverse.h> 38 #include <sys/dsl_dataset.h> 39 #include <sys/dsl_dir.h> 40 #include <sys/dsl_pool.h> 41 #include <sys/dsl_synctask.h> 42 #include <sys/dsl_prop.h> 43 #include <sys/dmu_zfetch.h> 44 #include <sys/zfs_ioctl.h> 45 #include <sys/zap.h> 46 #include <sys/zio_checksum.h> 47 #include <sys/zio_compress.h> 48 #include <sys/sa.h> 49 #include <sys/zfeature.h> 50 #include <sys/abd.h> 51 #include <sys/zfs_rlock.h> 52 #include <sys/racct.h> 53 #include <sys/vm.h> 54 #include <sys/zfs_znode.h> 55 #include <sys/zfs_vnops.h> 56 57 #include <sys/ccompat.h> 58 59 #ifndef IDX_TO_OFF 60 #define IDX_TO_OFF(idx) (((vm_ooffset_t)(idx)) << PAGE_SHIFT) 61 #endif 62 63 #define VM_ALLOC_BUSY_FLAGS VM_ALLOC_SBUSY | VM_ALLOC_IGN_SBUSY 64 65 int 66 dmu_write_pages(objset_t *os, uint64_t object, uint64_t offset, uint64_t size, 67 vm_page_t *ma, dmu_tx_t *tx) 68 { 69 dmu_buf_t **dbp; 70 struct sf_buf *sf; 71 int numbufs, i; 72 int err; 73 74 if (size == 0) 75 return (0); 76 77 err = dmu_buf_hold_array(os, object, offset, size, 78 FALSE, FTAG, &numbufs, &dbp); 79 if (err) 80 return (err); 81 82 for (i = 0; i < numbufs; i++) { 83 int tocpy, copied, thiscpy; 84 int bufoff; 85 dmu_buf_t *db = dbp[i]; 86 caddr_t va; 87 88 ASSERT3U(size, >, 0); 89 ASSERT3U(db->db_size, >=, PAGESIZE); 90 91 bufoff = offset - db->db_offset; 92 tocpy = (int)MIN(db->db_size - bufoff, size); 93 94 ASSERT(i == 0 || i == numbufs-1 || tocpy == db->db_size); 95 96 if (tocpy == db->db_size) 97 dmu_buf_will_fill(db, tx, B_FALSE); 98 else 99 dmu_buf_will_dirty(db, tx); 100 101 for (copied = 0; copied < tocpy; copied += PAGESIZE) { 102 ASSERT3U(ptoa((*ma)->pindex), ==, 103 db->db_offset + bufoff); 104 thiscpy = MIN(PAGESIZE, tocpy - copied); 105 va = zfs_map_page(*ma, &sf); 106 ASSERT(db->db_data != NULL); 107 memcpy((char *)db->db_data + bufoff, va, thiscpy); 108 zfs_unmap_page(sf); 109 ma += 1; 110 bufoff += PAGESIZE; 111 } 112 113 if (tocpy == db->db_size) 114 dmu_buf_fill_done(db, tx, B_FALSE); 115 116 offset += tocpy; 117 size -= tocpy; 118 } 119 dmu_buf_rele_array(dbp, numbufs, FTAG); 120 return (err); 121 } 122 123 int 124 dmu_read_pages(objset_t *os, uint64_t object, vm_page_t *ma, int count, 125 int *rbehind, int *rahead, int last_size) 126 { 127 struct sf_buf *sf; 128 vm_object_t vmobj; 129 vm_page_t m; 130 dmu_buf_t **dbp; 131 dmu_buf_t *db; 132 caddr_t va; 133 int numbufs, i; 134 int bufoff, pgoff, tocpy; 135 int mi, di; 136 int err; 137 138 ASSERT3U(ma[0]->pindex + count - 1, ==, ma[count - 1]->pindex); 139 ASSERT3S(last_size, <=, PAGE_SIZE); 140 141 err = dmu_buf_hold_array(os, object, IDX_TO_OFF(ma[0]->pindex), 142 IDX_TO_OFF(count - 1) + last_size, TRUE, FTAG, &numbufs, &dbp); 143 if (err != 0) 144 return (err); 145 146 #ifdef ZFS_DEBUG 147 IMPLY(last_size < PAGE_SIZE, *rahead == 0); 148 if (dbp[0]->db_offset != 0 || numbufs > 1) { 149 for (i = 0; i < numbufs; i++) { 150 ASSERT(ISP2(dbp[i]->db_size)); 151 ASSERT3U((dbp[i]->db_offset % dbp[i]->db_size), ==, 0); 152 ASSERT3U(dbp[i]->db_size, ==, dbp[0]->db_size); 153 } 154 } 155 #endif 156 157 vmobj = ma[0]->object; 158 159 db = dbp[0]; 160 for (i = 0; i < *rbehind; i++) { 161 m = vm_page_grab_unlocked(vmobj, ma[0]->pindex - 1 - i, 162 VM_ALLOC_NORMAL | VM_ALLOC_NOWAIT | VM_ALLOC_BUSY_FLAGS); 163 if (m == NULL) 164 break; 165 if (!vm_page_none_valid(m)) { 166 ASSERT3U(m->valid, ==, VM_PAGE_BITS_ALL); 167 vm_page_sunbusy(m); 168 break; 169 } 170 ASSERT3U(m->dirty, ==, 0); 171 ASSERT(!pmap_page_is_write_mapped(m)); 172 173 ASSERT3U(db->db_size, >, PAGE_SIZE); 174 bufoff = IDX_TO_OFF(m->pindex) % db->db_size; 175 va = zfs_map_page(m, &sf); 176 ASSERT(db->db_data != NULL); 177 memcpy(va, (char *)db->db_data + bufoff, PAGESIZE); 178 zfs_unmap_page(sf); 179 vm_page_valid(m); 180 if ((m->busy_lock & VPB_BIT_WAITERS) != 0) 181 vm_page_activate(m); 182 else 183 vm_page_deactivate(m); 184 vm_page_sunbusy(m); 185 } 186 *rbehind = i; 187 188 bufoff = IDX_TO_OFF(ma[0]->pindex) % db->db_size; 189 pgoff = 0; 190 for (mi = 0, di = 0; mi < count && di < numbufs; ) { 191 if (pgoff == 0) { 192 m = ma[mi]; 193 if (m != bogus_page) { 194 vm_page_assert_xbusied(m); 195 ASSERT(vm_page_none_valid(m)); 196 ASSERT3U(m->dirty, ==, 0); 197 ASSERT(!pmap_page_is_write_mapped(m)); 198 va = zfs_map_page(m, &sf); 199 } 200 } 201 if (bufoff == 0) 202 db = dbp[di]; 203 204 if (m != bogus_page) { 205 ASSERT3U(IDX_TO_OFF(m->pindex) + pgoff, ==, 206 db->db_offset + bufoff); 207 } 208 209 /* 210 * We do not need to clamp the copy size by the file 211 * size as the last block is zero-filled beyond the 212 * end of file anyway. 213 */ 214 tocpy = MIN(db->db_size - bufoff, PAGESIZE - pgoff); 215 ASSERT3S(tocpy, >=, 0); 216 if (m != bogus_page) { 217 ASSERT(db->db_data != NULL); 218 memcpy(va + pgoff, (char *)db->db_data + bufoff, tocpy); 219 } 220 221 pgoff += tocpy; 222 ASSERT3S(pgoff, >=, 0); 223 ASSERT3S(pgoff, <=, PAGESIZE); 224 if (pgoff == PAGESIZE) { 225 if (m != bogus_page) { 226 zfs_unmap_page(sf); 227 vm_page_valid(m); 228 } 229 ASSERT3S(mi, <, count); 230 mi++; 231 pgoff = 0; 232 } 233 234 bufoff += tocpy; 235 ASSERT3S(bufoff, >=, 0); 236 ASSERT3S(bufoff, <=, db->db_size); 237 if (bufoff == db->db_size) { 238 ASSERT3S(di, <, numbufs); 239 di++; 240 bufoff = 0; 241 } 242 } 243 244 #ifdef ZFS_DEBUG 245 /* 246 * Three possibilities: 247 * - last requested page ends at a buffer boundary and , thus, 248 * all pages and buffers have been iterated; 249 * - all requested pages are filled, but the last buffer 250 * has not been exhausted; 251 * the read-ahead is possible only in this case; 252 * - all buffers have been read, but the last page has not been 253 * fully filled; 254 * this is only possible if the file has only a single buffer 255 * with a size that is not a multiple of the page size. 256 */ 257 if (mi == count) { 258 ASSERT3S(di, >=, numbufs - 1); 259 IMPLY(*rahead != 0, di == numbufs - 1); 260 IMPLY(*rahead != 0, bufoff != 0); 261 ASSERT0(pgoff); 262 } 263 if (di == numbufs) { 264 ASSERT3S(mi, >=, count - 1); 265 ASSERT0(*rahead); 266 IMPLY(pgoff == 0, mi == count); 267 if (pgoff != 0) { 268 ASSERT3S(mi, ==, count - 1); 269 ASSERT3U((dbp[0]->db_size & PAGE_MASK), !=, 0); 270 } 271 } 272 #endif 273 if (pgoff != 0) { 274 ASSERT3P(m, !=, bogus_page); 275 memset(va + pgoff, 0, PAGESIZE - pgoff); 276 zfs_unmap_page(sf); 277 vm_page_valid(m); 278 } 279 280 for (i = 0; i < *rahead; i++) { 281 m = vm_page_grab_unlocked(vmobj, ma[count - 1]->pindex + 1 + i, 282 VM_ALLOC_NORMAL | VM_ALLOC_NOWAIT | VM_ALLOC_BUSY_FLAGS); 283 if (m == NULL) 284 break; 285 if (!vm_page_none_valid(m)) { 286 ASSERT3U(m->valid, ==, VM_PAGE_BITS_ALL); 287 vm_page_sunbusy(m); 288 break; 289 } 290 ASSERT3U(m->dirty, ==, 0); 291 ASSERT(!pmap_page_is_write_mapped(m)); 292 293 ASSERT3U(db->db_size, >, PAGE_SIZE); 294 bufoff = IDX_TO_OFF(m->pindex) % db->db_size; 295 tocpy = MIN(db->db_size - bufoff, PAGESIZE); 296 va = zfs_map_page(m, &sf); 297 ASSERT(db->db_data != NULL); 298 memcpy(va, (char *)db->db_data + bufoff, tocpy); 299 if (tocpy < PAGESIZE) { 300 ASSERT3S(i, ==, *rahead - 1); 301 ASSERT3U((db->db_size & PAGE_MASK), !=, 0); 302 memset(va + tocpy, 0, PAGESIZE - tocpy); 303 } 304 zfs_unmap_page(sf); 305 vm_page_valid(m); 306 if ((m->busy_lock & VPB_BIT_WAITERS) != 0) 307 vm_page_activate(m); 308 else 309 vm_page_deactivate(m); 310 vm_page_sunbusy(m); 311 } 312 *rahead = i; 313 314 dmu_buf_rele_array(dbp, numbufs, FTAG); 315 return (0); 316 } 317