1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 27 /* All Rights Reserved */ 28 29 /* 30 * University Copyright- Copyright (c) 1982, 1986, 1988 31 * The Regents of the University of California 32 * All Rights Reserved 33 * 34 * University Acknowledgment- Portions of this document are derived from 35 * software developed by the University of California, Berkeley, and its 36 * contributors. 37 */ 38 39 /* 40 * $FreeBSD$ 41 */ 42 43 #include <sys/param.h> 44 #include <sys/uio_impl.h> 45 #include <sys/vnode.h> 46 #include <sys/zfs_znode.h> 47 #include <sys/byteorder.h> 48 #include <sys/lock.h> 49 #include <sys/vm.h> 50 #include <vm/vm_map.h> 51 52 static void 53 zfs_freeuio(struct uio *uio) 54 { 55 #if __FreeBSD_version > 1500013 56 freeuio(uio); 57 #else 58 free(uio, M_IOV); 59 #endif 60 } 61 62 int 63 zfs_uiomove(void *cp, size_t n, zfs_uio_rw_t dir, zfs_uio_t *uio) 64 { 65 ASSERT3U(zfs_uio_rw(uio), ==, dir); 66 return (uiomove(cp, (int)n, GET_UIO_STRUCT(uio))); 67 } 68 69 /* 70 * same as zfs_uiomove() but doesn't modify uio structure. 71 * return in cbytes how many bytes were copied. 72 */ 73 int 74 zfs_uiocopy(void *p, size_t n, zfs_uio_rw_t rw, zfs_uio_t *uio, size_t *cbytes) 75 { 76 struct iovec small_iovec[1]; 77 struct uio small_uio_clone; 78 struct uio *uio_clone; 79 int error; 80 81 ASSERT3U(zfs_uio_rw(uio), ==, rw); 82 if (zfs_uio_iovcnt(uio) == 1) { 83 small_uio_clone = *(GET_UIO_STRUCT(uio)); 84 small_iovec[0] = *(GET_UIO_STRUCT(uio)->uio_iov); 85 small_uio_clone.uio_iov = small_iovec; 86 uio_clone = &small_uio_clone; 87 } else { 88 uio_clone = cloneuio(GET_UIO_STRUCT(uio)); 89 } 90 91 error = vn_io_fault_uiomove(p, n, uio_clone); 92 *cbytes = zfs_uio_resid(uio) - uio_clone->uio_resid; 93 if (uio_clone != &small_uio_clone) 94 zfs_freeuio(uio_clone); 95 return (error); 96 } 97 98 /* 99 * Drop the next n chars out of *uiop. 100 */ 101 void 102 zfs_uioskip(zfs_uio_t *uio, size_t n) 103 { 104 zfs_uio_seg_t segflg; 105 106 /* For the full compatibility with illumos. */ 107 if (n > zfs_uio_resid(uio)) 108 return; 109 110 segflg = zfs_uio_segflg(uio); 111 zfs_uio_segflg(uio) = UIO_NOCOPY; 112 zfs_uiomove(NULL, n, zfs_uio_rw(uio), uio); 113 zfs_uio_segflg(uio) = segflg; 114 } 115 116 int 117 zfs_uio_fault_move(void *p, size_t n, zfs_uio_rw_t dir, zfs_uio_t *uio) 118 { 119 ASSERT3U(zfs_uio_rw(uio), ==, dir); 120 return (vn_io_fault_uiomove(p, n, GET_UIO_STRUCT(uio))); 121 } 122 123 /* 124 * Check if the uio is page-aligned in memory. 125 */ 126 boolean_t 127 zfs_uio_page_aligned(zfs_uio_t *uio) 128 { 129 const struct iovec *iov = GET_UIO_STRUCT(uio)->uio_iov; 130 131 for (int i = zfs_uio_iovcnt(uio); i > 0; iov++, i--) { 132 uintptr_t addr = (uintptr_t)iov->iov_base; 133 size_t size = iov->iov_len; 134 if ((addr & (PAGE_SIZE - 1)) || (size & (PAGE_SIZE - 1))) { 135 return (B_FALSE); 136 } 137 } 138 139 return (B_TRUE); 140 } 141 142 static void 143 zfs_uio_set_pages_to_stable(zfs_uio_t *uio) 144 { 145 ASSERT3P(uio->uio_dio.pages, !=, NULL); 146 ASSERT3S(uio->uio_dio.npages, >, 0); 147 148 for (int i = 0; i < uio->uio_dio.npages; i++) { 149 vm_page_t page = uio->uio_dio.pages[i]; 150 ASSERT3P(page, !=, NULL); 151 152 MPASS(page == PHYS_TO_VM_PAGE(VM_PAGE_TO_PHYS(page))); 153 vm_page_busy_acquire(page, VM_ALLOC_SBUSY); 154 pmap_remove_write(page); 155 } 156 } 157 158 static void 159 zfs_uio_release_stable_pages(zfs_uio_t *uio) 160 { 161 ASSERT3P(uio->uio_dio.pages, !=, NULL); 162 for (int i = 0; i < uio->uio_dio.npages; i++) { 163 vm_page_t page = uio->uio_dio.pages[i]; 164 165 ASSERT3P(page, !=, NULL); 166 vm_page_sunbusy(page); 167 } 168 } 169 170 /* 171 * If the operation is marked as read, then we are stating the pages will be 172 * written to and must be given write access. 173 */ 174 static int 175 zfs_uio_hold_pages(unsigned long start, size_t len, int nr_pages, 176 zfs_uio_rw_t rw, vm_page_t *pages) 177 { 178 vm_map_t map; 179 vm_prot_t prot; 180 int count; 181 182 map = &curthread->td_proc->p_vmspace->vm_map; 183 ASSERT3S(len, >, 0); 184 185 prot = rw == UIO_READ ? (VM_PROT_READ | VM_PROT_WRITE) : VM_PROT_READ; 186 count = vm_fault_quick_hold_pages(map, start, len, prot, pages, 187 nr_pages); 188 189 return (count); 190 } 191 192 void 193 zfs_uio_free_dio_pages(zfs_uio_t *uio, zfs_uio_rw_t rw) 194 { 195 ASSERT(uio->uio_extflg & UIO_DIRECT); 196 ASSERT3P(uio->uio_dio.pages, !=, NULL); 197 ASSERT(zfs_uio_rw(uio) == rw); 198 199 if (rw == UIO_WRITE) 200 zfs_uio_release_stable_pages(uio); 201 202 vm_page_unhold_pages(&uio->uio_dio.pages[0], 203 uio->uio_dio.npages); 204 205 kmem_free(uio->uio_dio.pages, 206 uio->uio_dio.npages * sizeof (vm_page_t)); 207 } 208 209 static int 210 zfs_uio_get_user_pages(unsigned long start, int nr_pages, 211 size_t len, zfs_uio_rw_t rw, vm_page_t *pages) 212 { 213 int count; 214 215 count = zfs_uio_hold_pages(start, len, nr_pages, rw, pages); 216 217 if (count != nr_pages) { 218 if (count > 0) 219 vm_page_unhold_pages(pages, count); 220 return (0); 221 } 222 223 ASSERT3S(count, ==, nr_pages); 224 225 return (count); 226 } 227 228 static int 229 zfs_uio_iov_step(struct iovec v, zfs_uio_t *uio, int *numpages) 230 { 231 unsigned long addr = (unsigned long)(v.iov_base); 232 size_t len = v.iov_len; 233 int n = DIV_ROUND_UP(len, PAGE_SIZE); 234 235 int res = zfs_uio_get_user_pages( 236 P2ALIGN_TYPED(addr, PAGE_SIZE, unsigned long), n, len, 237 zfs_uio_rw(uio), &uio->uio_dio.pages[uio->uio_dio.npages]); 238 239 if (res != n) 240 return (SET_ERROR(EFAULT)); 241 242 ASSERT3U(len, ==, res * PAGE_SIZE); 243 *numpages = res; 244 return (0); 245 } 246 247 static int 248 zfs_uio_get_dio_pages_impl(zfs_uio_t *uio) 249 { 250 const struct iovec *iovp = GET_UIO_STRUCT(uio)->uio_iov; 251 size_t len = zfs_uio_resid(uio); 252 253 for (int i = 0; i < zfs_uio_iovcnt(uio); i++) { 254 struct iovec iov; 255 int numpages = 0; 256 257 if (iovp->iov_len == 0) { 258 iovp++; 259 continue; 260 } 261 iov.iov_len = MIN(len, iovp->iov_len); 262 iov.iov_base = iovp->iov_base; 263 int error = zfs_uio_iov_step(iov, uio, &numpages); 264 265 if (error) 266 return (error); 267 268 uio->uio_dio.npages += numpages; 269 len -= iov.iov_len; 270 iovp++; 271 } 272 273 ASSERT0(len); 274 275 return (0); 276 } 277 278 /* 279 * This function holds user pages into the kernel. In the event that the user 280 * pages are not successfully held an error value is returned. 281 * 282 * On success, 0 is returned. 283 */ 284 int 285 zfs_uio_get_dio_pages_alloc(zfs_uio_t *uio, zfs_uio_rw_t rw) 286 { 287 int error = 0; 288 int npages = DIV_ROUND_UP(zfs_uio_resid(uio), PAGE_SIZE); 289 size_t size = npages * sizeof (vm_page_t); 290 291 ASSERT(zfs_uio_rw(uio) == rw); 292 293 uio->uio_dio.pages = kmem_alloc(size, KM_SLEEP); 294 295 error = zfs_uio_get_dio_pages_impl(uio); 296 297 if (error) { 298 vm_page_unhold_pages(&uio->uio_dio.pages[0], 299 uio->uio_dio.npages); 300 kmem_free(uio->uio_dio.pages, size); 301 return (error); 302 } 303 304 ASSERT3S(uio->uio_dio.npages, >, 0); 305 306 /* 307 * Since we will be writing the user pages we must make sure that 308 * they are stable. That way the contents of the pages can not change 309 * while we are doing: compression, checksumming, encryption, parity 310 * calculations or deduplication. 311 */ 312 if (zfs_uio_rw(uio) == UIO_WRITE) 313 zfs_uio_set_pages_to_stable(uio); 314 315 uio->uio_extflg |= UIO_DIRECT; 316 317 return (0); 318 } 319