xref: /netbsd-src/sys/miscfs/genfs/genfs_io.c (revision bdc22b2e01993381dcefeff2bc9b56ca75a4235c)
1 /*	$NetBSD: genfs_io.c,v 1.72 2018/05/28 21:04:38 chs Exp $	*/
2 
3 /*
4  * Copyright (c) 1982, 1986, 1989, 1993
5  *	The Regents of the University of California.  All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  * 3. Neither the name of the University nor the names of its contributors
16  *    may be used to endorse or promote products derived from this software
17  *    without specific prior written permission.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29  * SUCH DAMAGE.
30  *
31  */
32 
33 #include <sys/cdefs.h>
34 __KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.72 2018/05/28 21:04:38 chs Exp $");
35 
36 #include <sys/param.h>
37 #include <sys/systm.h>
38 #include <sys/proc.h>
39 #include <sys/kernel.h>
40 #include <sys/mount.h>
41 #include <sys/vnode.h>
42 #include <sys/kmem.h>
43 #include <sys/kauth.h>
44 #include <sys/fstrans.h>
45 #include <sys/buf.h>
46 
47 #include <miscfs/genfs/genfs.h>
48 #include <miscfs/genfs/genfs_node.h>
49 #include <miscfs/specfs/specdev.h>
50 
51 #include <uvm/uvm.h>
52 #include <uvm/uvm_pager.h>
53 
54 static int genfs_do_directio(struct vmspace *, vaddr_t, size_t, struct vnode *,
55     off_t, enum uio_rw);
56 static void genfs_dio_iodone(struct buf *);
57 
58 static int genfs_getpages_read(struct vnode *, struct vm_page **, int, off_t,
59     off_t, bool, bool, bool, bool);
60 static int genfs_do_io(struct vnode *, off_t, vaddr_t, size_t, int, enum uio_rw,
61     void (*)(struct buf *));
62 static void genfs_rel_pages(struct vm_page **, unsigned int);
63 static void genfs_markdirty(struct vnode *);
64 
65 int genfs_maxdio = MAXPHYS;
66 
67 static void
68 genfs_rel_pages(struct vm_page **pgs, unsigned int npages)
69 {
70 	unsigned int i;
71 
72 	for (i = 0; i < npages; i++) {
73 		struct vm_page *pg = pgs[i];
74 
75 		if (pg == NULL || pg == PGO_DONTCARE)
76 			continue;
77 		KASSERT(uvm_page_locked_p(pg));
78 		if (pg->flags & PG_FAKE) {
79 			pg->flags |= PG_RELEASED;
80 		}
81 	}
82 	mutex_enter(&uvm_pageqlock);
83 	uvm_page_unbusy(pgs, npages);
84 	mutex_exit(&uvm_pageqlock);
85 }
86 
87 static void
88 genfs_markdirty(struct vnode *vp)
89 {
90 	struct genfs_node * const gp = VTOG(vp);
91 
92 	KASSERT(mutex_owned(vp->v_interlock));
93 	gp->g_dirtygen++;
94 	if ((vp->v_iflag & VI_ONWORKLST) == 0) {
95 		vn_syncer_add_to_worklist(vp, filedelay);
96 	}
97 	if ((vp->v_iflag & (VI_WRMAP|VI_WRMAPDIRTY)) == VI_WRMAP) {
98 		vp->v_iflag |= VI_WRMAPDIRTY;
99 	}
100 }
101 
102 /*
103  * generic VM getpages routine.
104  * Return PG_BUSY pages for the given range,
105  * reading from backing store if necessary.
106  */
107 
108 int
109 genfs_getpages(void *v)
110 {
111 	struct vop_getpages_args /* {
112 		struct vnode *a_vp;
113 		voff_t a_offset;
114 		struct vm_page **a_m;
115 		int *a_count;
116 		int a_centeridx;
117 		vm_prot_t a_access_type;
118 		int a_advice;
119 		int a_flags;
120 	} */ * const ap = v;
121 
122 	off_t diskeof, memeof;
123 	int i, error, npages;
124 	const int flags = ap->a_flags;
125 	struct vnode * const vp = ap->a_vp;
126 	struct uvm_object * const uobj = &vp->v_uobj;
127 	const bool async = (flags & PGO_SYNCIO) == 0;
128 	const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0;
129 	const bool overwrite = (flags & PGO_OVERWRITE) != 0;
130 	const bool blockalloc = memwrite && (flags & PGO_NOBLOCKALLOC) == 0;
131 	const bool glocked = (flags & PGO_GLOCKHELD) != 0;
132 	bool holds_wapbl = false;
133 	struct mount *trans_mount = NULL;
134 	UVMHIST_FUNC("genfs_getpages"); UVMHIST_CALLED(ubchist);
135 
136 	UVMHIST_LOG(ubchist, "vp %#jx off 0x%jx/%jx count %jd",
137 	    (uintptr_t)vp, ap->a_offset >> 32, ap->a_offset, *ap->a_count);
138 
139 	KASSERT(vp->v_type == VREG || vp->v_type == VDIR ||
140 	    vp->v_type == VLNK || vp->v_type == VBLK);
141 
142 	error = vdead_check(vp, VDEAD_NOWAIT);
143 	if (error) {
144 		if ((flags & PGO_LOCKED) == 0)
145 			mutex_exit(uobj->vmobjlock);
146 		return error;
147 	}
148 
149 startover:
150 	error = 0;
151 	const voff_t origvsize = vp->v_size;
152 	const off_t origoffset = ap->a_offset;
153 	const int orignpages = *ap->a_count;
154 
155 	GOP_SIZE(vp, origvsize, &diskeof, 0);
156 	if (flags & PGO_PASTEOF) {
157 		off_t newsize;
158 #if defined(DIAGNOSTIC)
159 		off_t writeeof;
160 #endif /* defined(DIAGNOSTIC) */
161 
162 		newsize = MAX(origvsize,
163 		    origoffset + (orignpages << PAGE_SHIFT));
164 		GOP_SIZE(vp, newsize, &memeof, GOP_SIZE_MEM);
165 #if defined(DIAGNOSTIC)
166 		GOP_SIZE(vp, vp->v_writesize, &writeeof, GOP_SIZE_MEM);
167 		if (newsize > round_page(writeeof)) {
168 			panic("%s: past eof: %" PRId64 " vs. %" PRId64,
169 			    __func__, newsize, round_page(writeeof));
170 		}
171 #endif /* defined(DIAGNOSTIC) */
172 	} else {
173 		GOP_SIZE(vp, origvsize, &memeof, GOP_SIZE_MEM);
174 	}
175 	KASSERT(ap->a_centeridx >= 0 || ap->a_centeridx <= orignpages);
176 	KASSERT((origoffset & (PAGE_SIZE - 1)) == 0 && origoffset >= 0);
177 	KASSERT(orignpages > 0);
178 
179 	/*
180 	 * Bounds-check the request.
181 	 */
182 
183 	if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= memeof) {
184 		if ((flags & PGO_LOCKED) == 0) {
185 			mutex_exit(uobj->vmobjlock);
186 		}
187 		UVMHIST_LOG(ubchist, "off 0x%jx count %jd goes past EOF 0x%jx",
188 		    origoffset, *ap->a_count, memeof,0);
189 		error = EINVAL;
190 		goto out_err;
191 	}
192 
193 	/* uobj is locked */
194 
195 	if ((flags & PGO_NOTIMESTAMP) == 0 &&
196 	    (vp->v_type != VBLK ||
197 	    (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) {
198 		int updflags = 0;
199 
200 		if ((vp->v_mount->mnt_flag & MNT_NOATIME) == 0) {
201 			updflags = GOP_UPDATE_ACCESSED;
202 		}
203 		if (memwrite) {
204 			updflags |= GOP_UPDATE_MODIFIED;
205 		}
206 		if (updflags != 0) {
207 			GOP_MARKUPDATE(vp, updflags);
208 		}
209 	}
210 
211 	/*
212 	 * For PGO_LOCKED requests, just return whatever's in memory.
213 	 */
214 
215 	if (flags & PGO_LOCKED) {
216 		int nfound;
217 		struct vm_page *pg;
218 
219 		KASSERT(!glocked);
220 		npages = *ap->a_count;
221 #if defined(DEBUG)
222 		for (i = 0; i < npages; i++) {
223 			pg = ap->a_m[i];
224 			KASSERT(pg == NULL || pg == PGO_DONTCARE);
225 		}
226 #endif /* defined(DEBUG) */
227 		nfound = uvn_findpages(uobj, origoffset, &npages,
228 		    ap->a_m, UFP_NOWAIT|UFP_NOALLOC|(memwrite ? UFP_NORDONLY : 0));
229 		KASSERT(npages == *ap->a_count);
230 		if (nfound == 0) {
231 			error = EBUSY;
232 			goto out_err;
233 		}
234 		if (!genfs_node_rdtrylock(vp)) {
235 			genfs_rel_pages(ap->a_m, npages);
236 
237 			/*
238 			 * restore the array.
239 			 */
240 
241 			for (i = 0; i < npages; i++) {
242 				pg = ap->a_m[i];
243 
244 				if (pg != NULL && pg != PGO_DONTCARE) {
245 					ap->a_m[i] = NULL;
246 				}
247 				KASSERT(ap->a_m[i] == NULL ||
248 				    ap->a_m[i] == PGO_DONTCARE);
249 			}
250 		} else {
251 			genfs_node_unlock(vp);
252 		}
253 		error = (ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0);
254 		if (error == 0 && memwrite) {
255 			genfs_markdirty(vp);
256 		}
257 		goto out_err;
258 	}
259 	mutex_exit(uobj->vmobjlock);
260 
261 	/*
262 	 * find the requested pages and make some simple checks.
263 	 * leave space in the page array for a whole block.
264 	 */
265 
266 	const int fs_bshift = (vp->v_type != VBLK) ?
267 	    vp->v_mount->mnt_fs_bshift : DEV_BSHIFT;
268 	const int fs_bsize = 1 << fs_bshift;
269 #define	blk_mask	(fs_bsize - 1)
270 #define	trunc_blk(x)	((x) & ~blk_mask)
271 #define	round_blk(x)	(((x) + blk_mask) & ~blk_mask)
272 
273 	const int orignmempages = MIN(orignpages,
274 	    round_page(memeof - origoffset) >> PAGE_SHIFT);
275 	npages = orignmempages;
276 	const off_t startoffset = trunc_blk(origoffset);
277 	const off_t endoffset = MIN(
278 	    round_page(round_blk(origoffset + (npages << PAGE_SHIFT))),
279 	    round_page(memeof));
280 	const int ridx = (origoffset - startoffset) >> PAGE_SHIFT;
281 
282 	const int pgs_size = sizeof(struct vm_page *) *
283 	    ((endoffset - startoffset) >> PAGE_SHIFT);
284 	struct vm_page **pgs, *pgs_onstack[UBC_MAX_PAGES];
285 
286 	if (pgs_size > sizeof(pgs_onstack)) {
287 		pgs = kmem_zalloc(pgs_size, async ? KM_NOSLEEP : KM_SLEEP);
288 		if (pgs == NULL) {
289 			pgs = pgs_onstack;
290 			error = ENOMEM;
291 			goto out_err;
292 		}
293 	} else {
294 		pgs = pgs_onstack;
295 		(void)memset(pgs, 0, pgs_size);
296 	}
297 
298 	UVMHIST_LOG(ubchist, "ridx %jd npages %jd startoff %jd endoff %jd",
299 	    ridx, npages, startoffset, endoffset);
300 
301 	if (trans_mount == NULL) {
302 		trans_mount = vp->v_mount;
303 		fstrans_start(trans_mount);
304 		/*
305 		 * check if this vnode is still valid.
306 		 */
307 		mutex_enter(vp->v_interlock);
308 		error = vdead_check(vp, 0);
309 		mutex_exit(vp->v_interlock);
310 		if (error)
311 			goto out_err_free;
312 		/*
313 		 * XXX: This assumes that we come here only via
314 		 * the mmio path
315 		 */
316 		if (blockalloc && vp->v_mount->mnt_wapbl) {
317 			error = WAPBL_BEGIN(trans_mount);
318 			if (error)
319 				goto out_err_free;
320 			holds_wapbl = true;
321 		}
322 	}
323 
324 	/*
325 	 * hold g_glock to prevent a race with truncate.
326 	 *
327 	 * check if our idea of v_size is still valid.
328 	 */
329 
330 	KASSERT(!glocked || genfs_node_wrlocked(vp));
331 	if (!glocked) {
332 		if (blockalloc) {
333 			genfs_node_wrlock(vp);
334 		} else {
335 			genfs_node_rdlock(vp);
336 		}
337 	}
338 	mutex_enter(uobj->vmobjlock);
339 	if (vp->v_size < origvsize) {
340 		if (!glocked) {
341 			genfs_node_unlock(vp);
342 		}
343 		if (pgs != pgs_onstack)
344 			kmem_free(pgs, pgs_size);
345 		goto startover;
346 	}
347 
348 	if (uvn_findpages(uobj, origoffset, &npages, &pgs[ridx],
349 	    async ? UFP_NOWAIT : UFP_ALL) != orignmempages) {
350 		if (!glocked) {
351 			genfs_node_unlock(vp);
352 		}
353 		KASSERT(async != 0);
354 		genfs_rel_pages(&pgs[ridx], orignmempages);
355 		mutex_exit(uobj->vmobjlock);
356 		error = EBUSY;
357 		goto out_err_free;
358 	}
359 
360 	/*
361 	 * if the pages are already resident, just return them.
362 	 */
363 
364 	for (i = 0; i < npages; i++) {
365 		struct vm_page *pg = pgs[ridx + i];
366 
367 		if ((pg->flags & PG_FAKE) ||
368 		    (blockalloc && (pg->flags & PG_RDONLY))) {
369 			break;
370 		}
371 	}
372 	if (i == npages) {
373 		if (!glocked) {
374 			genfs_node_unlock(vp);
375 		}
376 		UVMHIST_LOG(ubchist, "returning cached pages", 0,0,0,0);
377 		npages += ridx;
378 		goto out;
379 	}
380 
381 	/*
382 	 * if PGO_OVERWRITE is set, don't bother reading the pages.
383 	 */
384 
385 	if (overwrite) {
386 		if (!glocked) {
387 			genfs_node_unlock(vp);
388 		}
389 		UVMHIST_LOG(ubchist, "PGO_OVERWRITE",0,0,0,0);
390 
391 		for (i = 0; i < npages; i++) {
392 			struct vm_page *pg = pgs[ridx + i];
393 
394 			pg->flags &= ~(PG_RDONLY|PG_CLEAN);
395 		}
396 		npages += ridx;
397 		goto out;
398 	}
399 
400 	/*
401 	 * the page wasn't resident and we're not overwriting,
402 	 * so we're going to have to do some i/o.
403 	 * find any additional pages needed to cover the expanded range.
404 	 */
405 
406 	npages = (endoffset - startoffset) >> PAGE_SHIFT;
407 	if (startoffset != origoffset || npages != orignmempages) {
408 		int npgs;
409 
410 		/*
411 		 * we need to avoid deadlocks caused by locking
412 		 * additional pages at lower offsets than pages we
413 		 * already have locked.  unlock them all and start over.
414 		 */
415 
416 		genfs_rel_pages(&pgs[ridx], orignmempages);
417 		memset(pgs, 0, pgs_size);
418 
419 		UVMHIST_LOG(ubchist, "reset npages start 0x%jx end 0x%jx",
420 		    startoffset, endoffset, 0,0);
421 		npgs = npages;
422 		if (uvn_findpages(uobj, startoffset, &npgs, pgs,
423 		    async ? UFP_NOWAIT : UFP_ALL) != npages) {
424 			if (!glocked) {
425 				genfs_node_unlock(vp);
426 			}
427 			KASSERT(async != 0);
428 			genfs_rel_pages(pgs, npages);
429 			mutex_exit(uobj->vmobjlock);
430 			error = EBUSY;
431 			goto out_err_free;
432 		}
433 	}
434 
435 	mutex_exit(uobj->vmobjlock);
436 	error = genfs_getpages_read(vp, pgs, npages, startoffset, diskeof,
437 	    async, memwrite, blockalloc, glocked);
438 	if (!glocked) {
439 		genfs_node_unlock(vp);
440 	}
441 	if (error == 0 && async)
442 		goto out_err_free;
443 	mutex_enter(uobj->vmobjlock);
444 
445 	/*
446 	 * we're almost done!  release the pages...
447 	 * for errors, we free the pages.
448 	 * otherwise we activate them and mark them as valid and clean.
449 	 * also, unbusy pages that were not actually requested.
450 	 */
451 
452 	if (error) {
453 		genfs_rel_pages(pgs, npages);
454 		mutex_exit(uobj->vmobjlock);
455 		UVMHIST_LOG(ubchist, "returning error %jd", error,0,0,0);
456 		goto out_err_free;
457 	}
458 
459 out:
460 	UVMHIST_LOG(ubchist, "succeeding, npages %jd", npages,0,0,0);
461 	error = 0;
462 	mutex_enter(&uvm_pageqlock);
463 	for (i = 0; i < npages; i++) {
464 		struct vm_page *pg = pgs[i];
465 		if (pg == NULL) {
466 			continue;
467 		}
468 		UVMHIST_LOG(ubchist, "examining pg %#jx flags 0x%jx",
469 		    (uintptr_t)pg, pg->flags, 0,0);
470 		if (pg->flags & PG_FAKE && !overwrite) {
471 			pg->flags &= ~(PG_FAKE);
472 			pmap_clear_modify(pgs[i]);
473 		}
474 		KASSERT(!memwrite || !blockalloc || (pg->flags & PG_RDONLY) == 0);
475 		if (i < ridx || i >= ridx + orignmempages || async) {
476 			UVMHIST_LOG(ubchist, "unbusy pg %#jx offset 0x%jx",
477 			    (uintptr_t)pg, pg->offset,0,0);
478 			if (pg->flags & PG_WANTED) {
479 				wakeup(pg);
480 			}
481 			if (pg->flags & PG_FAKE) {
482 				KASSERT(overwrite);
483 				uvm_pagezero(pg);
484 			}
485 			if (pg->flags & PG_RELEASED) {
486 				uvm_pagefree(pg);
487 				continue;
488 			}
489 			uvm_pageenqueue(pg);
490 			pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE);
491 			UVM_PAGE_OWN(pg, NULL);
492 		}
493 	}
494 	mutex_exit(&uvm_pageqlock);
495 	if (memwrite) {
496 		genfs_markdirty(vp);
497 	}
498 	mutex_exit(uobj->vmobjlock);
499 	if (ap->a_m != NULL) {
500 		memcpy(ap->a_m, &pgs[ridx],
501 		    orignmempages * sizeof(struct vm_page *));
502 	}
503 
504 out_err_free:
505 	if (pgs != NULL && pgs != pgs_onstack)
506 		kmem_free(pgs, pgs_size);
507 out_err:
508 	if (trans_mount != NULL) {
509 		if (holds_wapbl)
510 			WAPBL_END(trans_mount);
511 		fstrans_done(trans_mount);
512 	}
513 	return error;
514 }
515 
516 /*
517  * genfs_getpages_read: Read the pages in with VOP_BMAP/VOP_STRATEGY.
518  *
519  * "glocked" (which is currently not actually used) tells us not whether
520  * the genfs_node is locked on entry (it always is) but whether it was
521  * locked on entry to genfs_getpages.
522  */
523 static int
524 genfs_getpages_read(struct vnode *vp, struct vm_page **pgs, int npages,
525     off_t startoffset, off_t diskeof,
526     bool async, bool memwrite, bool blockalloc, bool glocked)
527 {
528 	struct uvm_object * const uobj = &vp->v_uobj;
529 	const int fs_bshift = (vp->v_type != VBLK) ?
530 	    vp->v_mount->mnt_fs_bshift : DEV_BSHIFT;
531 	const int dev_bshift = (vp->v_type != VBLK) ?
532 	    vp->v_mount->mnt_dev_bshift : DEV_BSHIFT;
533 	kauth_cred_t const cred = curlwp->l_cred;		/* XXXUBC curlwp */
534 	size_t bytes, iobytes, tailstart, tailbytes, totalbytes, skipbytes;
535 	vaddr_t kva;
536 	struct buf *bp, *mbp;
537 	bool sawhole = false;
538 	int i;
539 	int error = 0;
540 
541 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist);
542 
543 	/*
544 	 * read the desired page(s).
545 	 */
546 
547 	totalbytes = npages << PAGE_SHIFT;
548 	bytes = MIN(totalbytes, MAX(diskeof - startoffset, 0));
549 	tailbytes = totalbytes - bytes;
550 	skipbytes = 0;
551 
552 	kva = uvm_pagermapin(pgs, npages,
553 	    UVMPAGER_MAPIN_READ | (async ? 0 : UVMPAGER_MAPIN_WAITOK));
554 	if (kva == 0)
555 		return EBUSY;
556 
557 	mbp = getiobuf(vp, true);
558 	mbp->b_bufsize = totalbytes;
559 	mbp->b_data = (void *)kva;
560 	mbp->b_resid = mbp->b_bcount = bytes;
561 	mbp->b_cflags = BC_BUSY;
562 	if (async) {
563 		mbp->b_flags = B_READ | B_ASYNC;
564 		mbp->b_iodone = uvm_aio_biodone;
565 	} else {
566 		mbp->b_flags = B_READ;
567 		mbp->b_iodone = NULL;
568 	}
569 	if (async)
570 		BIO_SETPRIO(mbp, BPRIO_TIMELIMITED);
571 	else
572 		BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL);
573 
574 	/*
575 	 * if EOF is in the middle of the range, zero the part past EOF.
576 	 * skip over pages which are not PG_FAKE since in that case they have
577 	 * valid data that we need to preserve.
578 	 */
579 
580 	tailstart = bytes;
581 	while (tailbytes > 0) {
582 		const int len = PAGE_SIZE - (tailstart & PAGE_MASK);
583 
584 		KASSERT(len <= tailbytes);
585 		if ((pgs[tailstart >> PAGE_SHIFT]->flags & PG_FAKE) != 0) {
586 			memset((void *)(kva + tailstart), 0, len);
587 			UVMHIST_LOG(ubchist, "tailbytes %#jx 0x%jx 0x%jx",
588 			    (uintptr_t)kva, tailstart, len, 0);
589 		}
590 		tailstart += len;
591 		tailbytes -= len;
592 	}
593 
594 	/*
595 	 * now loop over the pages, reading as needed.
596 	 */
597 
598 	bp = NULL;
599 	off_t offset;
600 	for (offset = startoffset;
601 	    bytes > 0;
602 	    offset += iobytes, bytes -= iobytes) {
603 		int run;
604 		daddr_t lbn, blkno;
605 		int pidx;
606 		struct vnode *devvp;
607 
608 		/*
609 		 * skip pages which don't need to be read.
610 		 */
611 
612 		pidx = (offset - startoffset) >> PAGE_SHIFT;
613 		while ((pgs[pidx]->flags & PG_FAKE) == 0) {
614 			size_t b;
615 
616 			KASSERT((offset & (PAGE_SIZE - 1)) == 0);
617 			if ((pgs[pidx]->flags & PG_RDONLY)) {
618 				sawhole = true;
619 			}
620 			b = MIN(PAGE_SIZE, bytes);
621 			offset += b;
622 			bytes -= b;
623 			skipbytes += b;
624 			pidx++;
625 			UVMHIST_LOG(ubchist, "skipping, new offset 0x%jx",
626 			    offset, 0,0,0);
627 			if (bytes == 0) {
628 				goto loopdone;
629 			}
630 		}
631 
632 		/*
633 		 * bmap the file to find out the blkno to read from and
634 		 * how much we can read in one i/o.  if bmap returns an error,
635 		 * skip the rest of the top-level i/o.
636 		 */
637 
638 		lbn = offset >> fs_bshift;
639 		error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run);
640 		if (error) {
641 			UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%jx -> %jd\n",
642 			    lbn,error,0,0);
643 			skipbytes += bytes;
644 			bytes = 0;
645 			goto loopdone;
646 		}
647 
648 		/*
649 		 * see how many pages can be read with this i/o.
650 		 * reduce the i/o size if necessary to avoid
651 		 * overwriting pages with valid data.
652 		 */
653 
654 		iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset,
655 		    bytes);
656 		if (offset + iobytes > round_page(offset)) {
657 			int pcount;
658 
659 			pcount = 1;
660 			while (pidx + pcount < npages &&
661 			    pgs[pidx + pcount]->flags & PG_FAKE) {
662 				pcount++;
663 			}
664 			iobytes = MIN(iobytes, (pcount << PAGE_SHIFT) -
665 			    (offset - trunc_page(offset)));
666 		}
667 
668 		/*
669 		 * if this block isn't allocated, zero it instead of
670 		 * reading it.  unless we are going to allocate blocks,
671 		 * mark the pages we zeroed PG_RDONLY.
672 		 */
673 
674 		if (blkno == (daddr_t)-1) {
675 			int holepages = (round_page(offset + iobytes) -
676 			    trunc_page(offset)) >> PAGE_SHIFT;
677 			UVMHIST_LOG(ubchist, "lbn 0x%jx -> HOLE", lbn,0,0,0);
678 
679 			sawhole = true;
680 			memset((char *)kva + (offset - startoffset), 0,
681 			    iobytes);
682 			skipbytes += iobytes;
683 
684 			mutex_enter(uobj->vmobjlock);
685 			for (i = 0; i < holepages; i++) {
686 				if (memwrite) {
687 					pgs[pidx + i]->flags &= ~PG_CLEAN;
688 				}
689 				if (!blockalloc) {
690 					pgs[pidx + i]->flags |= PG_RDONLY;
691 				}
692 			}
693 			mutex_exit(uobj->vmobjlock);
694 			continue;
695 		}
696 
697 		/*
698 		 * allocate a sub-buf for this piece of the i/o
699 		 * (or just use mbp if there's only 1 piece),
700 		 * and start it going.
701 		 */
702 
703 		if (offset == startoffset && iobytes == bytes) {
704 			bp = mbp;
705 		} else {
706 			UVMHIST_LOG(ubchist, "vp %#jx bp %#jx num now %jd",
707 			    (uintptr_t)vp, (uintptr_t)bp, vp->v_numoutput, 0);
708 			bp = getiobuf(vp, true);
709 			nestiobuf_setup(mbp, bp, offset - startoffset, iobytes);
710 		}
711 		bp->b_lblkno = 0;
712 
713 		/* adjust physical blkno for partial blocks */
714 		bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >>
715 		    dev_bshift);
716 
717 		UVMHIST_LOG(ubchist,
718 		    "bp %#jx offset 0x%x bcount 0x%x blkno 0x%x",
719 		    (uintptr_t)bp, offset, bp->b_bcount, bp->b_blkno);
720 
721 		VOP_STRATEGY(devvp, bp);
722 	}
723 
724 loopdone:
725 	nestiobuf_done(mbp, skipbytes, error);
726 	if (async) {
727 		UVMHIST_LOG(ubchist, "returning 0 (async)",0,0,0,0);
728 		return 0;
729 	}
730 	if (bp != NULL) {
731 		error = biowait(mbp);
732 	}
733 
734 	/* Remove the mapping (make KVA available as soon as possible) */
735 	uvm_pagermapout(kva, npages);
736 
737 	/*
738 	 * if this we encountered a hole then we have to do a little more work.
739 	 * for read faults, we marked the page PG_RDONLY so that future
740 	 * write accesses to the page will fault again.
741 	 * for write faults, we must make sure that the backing store for
742 	 * the page is completely allocated while the pages are locked.
743 	 */
744 
745 	if (!error && sawhole && blockalloc) {
746 		error = GOP_ALLOC(vp, startoffset,
747 		    npages << PAGE_SHIFT, 0, cred);
748 		UVMHIST_LOG(ubchist, "gop_alloc off 0x%jx/0x%jx -> %jd",
749 		    startoffset, npages << PAGE_SHIFT, error,0);
750 		if (!error) {
751 			mutex_enter(uobj->vmobjlock);
752 			for (i = 0; i < npages; i++) {
753 				struct vm_page *pg = pgs[i];
754 
755 				if (pg == NULL) {
756 					continue;
757 				}
758 				pg->flags &= ~(PG_CLEAN|PG_RDONLY);
759 				UVMHIST_LOG(ubchist, "mark dirty pg %#jx",
760 				    (uintptr_t)pg, 0, 0, 0);
761 			}
762 			mutex_exit(uobj->vmobjlock);
763 		}
764 	}
765 
766 	putiobuf(mbp);
767 	return error;
768 }
769 
770 /*
771  * generic VM putpages routine.
772  * Write the given range of pages to backing store.
773  *
774  * => "offhi == 0" means flush all pages at or after "offlo".
775  * => object should be locked by caller.  we return with the
776  *      object unlocked.
777  * => if PGO_CLEANIT or PGO_SYNCIO is set, we may block (due to I/O).
778  *	thus, a caller might want to unlock higher level resources
779  *	(e.g. vm_map) before calling flush.
780  * => if neither PGO_CLEANIT nor PGO_SYNCIO is set, we will not block
781  * => if PGO_ALLPAGES is set, then all pages in the object will be processed.
782  * => NOTE: we rely on the fact that the object's memq is a TAILQ and
783  *	that new pages are inserted on the tail end of the list.   thus,
784  *	we can make a complete pass through the object in one go by starting
785  *	at the head and working towards the tail (new pages are put in
786  *	front of us).
787  * => NOTE: we are allowed to lock the page queues, so the caller
788  *	must not be holding the page queue lock.
789  *
790  * note on "cleaning" object and PG_BUSY pages:
791  *	this routine is holding the lock on the object.   the only time
792  *	that it can run into a PG_BUSY page that it does not own is if
793  *	some other process has started I/O on the page (e.g. either
794  *	a pagein, or a pageout).    if the PG_BUSY page is being paged
795  *	in, then it can not be dirty (!PG_CLEAN) because no one has
796  *	had a chance to modify it yet.    if the PG_BUSY page is being
797  *	paged out then it means that someone else has already started
798  *	cleaning the page for us (how nice!).    in this case, if we
799  *	have syncio specified, then after we make our pass through the
800  *	object we need to wait for the other PG_BUSY pages to clear
801  *	off (i.e. we need to do an iosync).   also note that once a
802  *	page is PG_BUSY it must stay in its object until it is un-busyed.
803  *
804  * note on page traversal:
805  *	we can traverse the pages in an object either by going down the
806  *	linked list in "uobj->memq", or we can go over the address range
807  *	by page doing hash table lookups for each address.    depending
808  *	on how many pages are in the object it may be cheaper to do one
809  *	or the other.   we set "by_list" to true if we are using memq.
810  *	if the cost of a hash lookup was equal to the cost of the list
811  *	traversal we could compare the number of pages in the start->stop
812  *	range to the total number of pages in the object.   however, it
813  *	seems that a hash table lookup is more expensive than the linked
814  *	list traversal, so we multiply the number of pages in the
815  *	range by an estimate of the relatively higher cost of the hash lookup.
816  */
817 
818 int
819 genfs_putpages(void *v)
820 {
821 	struct vop_putpages_args /* {
822 		struct vnode *a_vp;
823 		voff_t a_offlo;
824 		voff_t a_offhi;
825 		int a_flags;
826 	} */ * const ap = v;
827 
828 	return genfs_do_putpages(ap->a_vp, ap->a_offlo, ap->a_offhi,
829 	    ap->a_flags, NULL);
830 }
831 
832 int
833 genfs_do_putpages(struct vnode *vp, off_t startoff, off_t endoff,
834     int origflags, struct vm_page **busypg)
835 {
836 	struct uvm_object * const uobj = &vp->v_uobj;
837 	kmutex_t * const slock = uobj->vmobjlock;
838 	off_t off;
839 	int i, error, npages, nback;
840 	int freeflag;
841 	/*
842 	 * This array is larger than it should so that it's size is constant.
843 	 * The right size is MAXPAGES.
844 	 */
845 	struct vm_page *pgs[MAXPHYS / MIN_PAGE_SIZE];
846 #define MAXPAGES (MAXPHYS / PAGE_SIZE)
847 	struct vm_page *pg, *nextpg, *tpg, curmp, endmp;
848 	bool wasclean, by_list, needs_clean, yld;
849 	bool async = (origflags & PGO_SYNCIO) == 0;
850 	bool pagedaemon = curlwp == uvm.pagedaemon_lwp;
851 	struct lwp * const l = curlwp ? curlwp : &lwp0;
852 	struct genfs_node * const gp = VTOG(vp);
853 	struct mount *trans_mp;
854 	int flags;
855 	int dirtygen;
856 	bool modified;
857 	bool holds_wapbl;
858 	bool cleanall;
859 	bool onworklst;
860 
861 	UVMHIST_FUNC("genfs_putpages"); UVMHIST_CALLED(ubchist);
862 
863 	KASSERT(origflags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE));
864 	KASSERT((startoff & PAGE_MASK) == 0 && (endoff & PAGE_MASK) == 0);
865 	KASSERT(startoff < endoff || endoff == 0);
866 
867 	UVMHIST_LOG(ubchist, "vp %#jx pages %jd off 0x%jx len 0x%jx",
868 	    (uintptr_t)vp, uobj->uo_npages, startoff, endoff - startoff);
869 
870 	trans_mp = NULL;
871 	holds_wapbl = false;
872 
873 retry:
874 	modified = false;
875 	flags = origflags;
876 	KASSERT((vp->v_iflag & VI_ONWORKLST) != 0 ||
877 	    (vp->v_iflag & VI_WRMAPDIRTY) == 0);
878 	if (uobj->uo_npages == 0) {
879 		if (vp->v_iflag & VI_ONWORKLST) {
880 			vp->v_iflag &= ~VI_WRMAPDIRTY;
881 			if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL)
882 				vn_syncer_remove_from_worklist(vp);
883 		}
884 		if (trans_mp) {
885 			if (holds_wapbl)
886 				WAPBL_END(trans_mp);
887 			fstrans_done(trans_mp);
888 		}
889 		mutex_exit(slock);
890 		return (0);
891 	}
892 
893 	/*
894 	 * the vnode has pages, set up to process the request.
895 	 */
896 
897 	if (trans_mp == NULL && (flags & PGO_CLEANIT) != 0) {
898 		if (pagedaemon) {
899 			/* Pagedaemon must not sleep here. */
900 			trans_mp = vp->v_mount;
901 			error = fstrans_start_nowait(trans_mp);
902 			if (error) {
903 				mutex_exit(slock);
904 				return error;
905 			}
906 		} else {
907 			/*
908 			 * Cannot use vdeadcheck() here as this operation
909 			 * usually gets used from VOP_RECLAIM().  Test for
910 			 * change of v_mount instead and retry on change.
911 			 */
912 			mutex_exit(slock);
913 			trans_mp = vp->v_mount;
914 			fstrans_start(trans_mp);
915 			if (vp->v_mount != trans_mp) {
916 				fstrans_done(trans_mp);
917 				trans_mp = NULL;
918 			} else {
919 				holds_wapbl = (trans_mp->mnt_wapbl &&
920 				    (origflags & PGO_JOURNALLOCKED) == 0);
921 				if (holds_wapbl) {
922 					error = WAPBL_BEGIN(trans_mp);
923 					if (error) {
924 						fstrans_done(trans_mp);
925 						return error;
926 					}
927 				}
928 			}
929 			mutex_enter(slock);
930 			goto retry;
931 		}
932 	}
933 
934 	error = 0;
935 	wasclean = (vp->v_numoutput == 0);
936 	off = startoff;
937 	if (endoff == 0 || flags & PGO_ALLPAGES) {
938 		endoff = trunc_page(LLONG_MAX);
939 	}
940 	by_list = (uobj->uo_npages <=
941 	    ((endoff - startoff) >> PAGE_SHIFT) * UVM_PAGE_TREE_PENALTY);
942 
943 	/*
944 	 * if this vnode is known not to have dirty pages,
945 	 * don't bother to clean it out.
946 	 */
947 
948 	if ((vp->v_iflag & VI_ONWORKLST) == 0) {
949 #if !defined(DEBUG)
950 		if ((flags & (PGO_FREE|PGO_DEACTIVATE)) == 0) {
951 			goto skip_scan;
952 		}
953 #endif /* !defined(DEBUG) */
954 		flags &= ~PGO_CLEANIT;
955 	}
956 
957 	/*
958 	 * start the loop.  when scanning by list, hold the last page
959 	 * in the list before we start.  pages allocated after we start
960 	 * will be added to the end of the list, so we can stop at the
961 	 * current last page.
962 	 */
963 
964 	cleanall = (flags & PGO_CLEANIT) != 0 && wasclean &&
965 	    startoff == 0 && endoff == trunc_page(LLONG_MAX) &&
966 	    (vp->v_iflag & VI_ONWORKLST) != 0;
967 	dirtygen = gp->g_dirtygen;
968 	freeflag = pagedaemon ? PG_PAGEOUT : PG_RELEASED;
969 	if (by_list) {
970 		curmp.flags = PG_MARKER;
971 		endmp.flags = PG_MARKER;
972 		pg = TAILQ_FIRST(&uobj->memq);
973 		TAILQ_INSERT_TAIL(&uobj->memq, &endmp, listq.queue);
974 	} else {
975 		pg = uvm_pagelookup(uobj, off);
976 	}
977 	nextpg = NULL;
978 	while (by_list || off < endoff) {
979 
980 		/*
981 		 * if the current page is not interesting, move on to the next.
982 		 */
983 
984 		KASSERT(pg == NULL || pg->uobject == uobj ||
985 		    (pg->flags & PG_MARKER) != 0);
986 		KASSERT(pg == NULL ||
987 		    (pg->flags & (PG_RELEASED|PG_PAGEOUT)) == 0 ||
988 		    (pg->flags & (PG_BUSY|PG_MARKER)) != 0);
989 		if (by_list) {
990 			if (pg == &endmp) {
991 				break;
992 			}
993 			if (pg->flags & PG_MARKER) {
994 				pg = TAILQ_NEXT(pg, listq.queue);
995 				continue;
996 			}
997 			if (pg->offset < startoff || pg->offset >= endoff ||
998 			    pg->flags & (PG_RELEASED|PG_PAGEOUT)) {
999 				if (pg->flags & (PG_RELEASED|PG_PAGEOUT)) {
1000 					wasclean = false;
1001 				}
1002 				pg = TAILQ_NEXT(pg, listq.queue);
1003 				continue;
1004 			}
1005 			off = pg->offset;
1006 		} else if (pg == NULL || pg->flags & (PG_RELEASED|PG_PAGEOUT)) {
1007 			if (pg != NULL) {
1008 				wasclean = false;
1009 			}
1010 			off += PAGE_SIZE;
1011 			if (off < endoff) {
1012 				pg = uvm_pagelookup(uobj, off);
1013 			}
1014 			continue;
1015 		}
1016 
1017 		/*
1018 		 * if the current page needs to be cleaned and it's busy,
1019 		 * wait for it to become unbusy.
1020 		 */
1021 
1022 		yld = (l->l_cpu->ci_schedstate.spc_flags &
1023 		    SPCF_SHOULDYIELD) && !pagedaemon;
1024 		if (pg->flags & PG_BUSY || yld) {
1025 			UVMHIST_LOG(ubchist, "busy %#jx", (uintptr_t)pg,
1026 			   0, 0, 0);
1027 			if (flags & PGO_BUSYFAIL && pg->flags & PG_BUSY) {
1028 				UVMHIST_LOG(ubchist, "busyfail %#jx",
1029 				    (uintptr_t)pg, 0, 0, 0);
1030 				error = EDEADLK;
1031 				if (busypg != NULL)
1032 					*busypg = pg;
1033 				break;
1034 			}
1035 			if (pagedaemon) {
1036 				/*
1037 				 * someone has taken the page while we
1038 				 * dropped the lock for fstrans_start.
1039 				 */
1040 				break;
1041 			}
1042 			if (by_list) {
1043 				TAILQ_INSERT_BEFORE(pg, &curmp, listq.queue);
1044 				UVMHIST_LOG(ubchist, "curmp next %#jx",
1045 				    (uintptr_t)TAILQ_NEXT(&curmp, listq.queue),
1046 				    0, 0, 0);
1047 			}
1048 			if (yld) {
1049 				mutex_exit(slock);
1050 				preempt();
1051 				mutex_enter(slock);
1052 			} else {
1053 				pg->flags |= PG_WANTED;
1054 				UVM_UNLOCK_AND_WAIT(pg, slock, 0, "genput", 0);
1055 				mutex_enter(slock);
1056 			}
1057 			if (by_list) {
1058 				UVMHIST_LOG(ubchist, "after next %#jx",
1059 				    (uintptr_t)TAILQ_NEXT(&curmp, listq.queue),
1060 				    0, 0, 0);
1061 				pg = TAILQ_NEXT(&curmp, listq.queue);
1062 				TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue);
1063 			} else {
1064 				pg = uvm_pagelookup(uobj, off);
1065 			}
1066 			continue;
1067 		}
1068 
1069 		/*
1070 		 * if we're freeing, remove all mappings of the page now.
1071 		 * if we're cleaning, check if the page is needs to be cleaned.
1072 		 */
1073 
1074 		if (flags & PGO_FREE) {
1075 			pmap_page_protect(pg, VM_PROT_NONE);
1076 		} else if (flags & PGO_CLEANIT) {
1077 
1078 			/*
1079 			 * if we still have some hope to pull this vnode off
1080 			 * from the syncer queue, write-protect the page.
1081 			 */
1082 
1083 			if (cleanall && wasclean &&
1084 			    gp->g_dirtygen == dirtygen) {
1085 
1086 				/*
1087 				 * uobj pages get wired only by uvm_fault
1088 				 * where uobj is locked.
1089 				 */
1090 
1091 				if (pg->wire_count == 0) {
1092 					pmap_page_protect(pg,
1093 					    VM_PROT_READ|VM_PROT_EXECUTE);
1094 				} else {
1095 					cleanall = false;
1096 				}
1097 			}
1098 		}
1099 
1100 		if (flags & PGO_CLEANIT) {
1101 			needs_clean = pmap_clear_modify(pg) ||
1102 			    (pg->flags & PG_CLEAN) == 0;
1103 			pg->flags |= PG_CLEAN;
1104 		} else {
1105 			needs_clean = false;
1106 		}
1107 
1108 		/*
1109 		 * if we're cleaning, build a cluster.
1110 		 * the cluster will consist of pages which are currently dirty,
1111 		 * but they will be returned to us marked clean.
1112 		 * if not cleaning, just operate on the one page.
1113 		 */
1114 
1115 		if (needs_clean) {
1116 			KDASSERT((vp->v_iflag & VI_ONWORKLST));
1117 			wasclean = false;
1118 			memset(pgs, 0, sizeof(pgs));
1119 			pg->flags |= PG_BUSY;
1120 			UVM_PAGE_OWN(pg, "genfs_putpages");
1121 
1122 			/*
1123 			 * let the fs constrain the offset range of the cluster.
1124 			 * we additionally constrain the range here such that
1125 			 * it fits in the "pgs" pages array.
1126 			 */
1127 
1128 			off_t fslo, fshi, genlo, lo;
1129 			GOP_PUTRANGE(vp, off, &fslo, &fshi);
1130 			KASSERT(fslo == trunc_page(fslo));
1131 			KASSERT(fslo <= off);
1132 			KASSERT(fshi == trunc_page(fshi));
1133 			KASSERT(fshi == 0 || off < fshi);
1134 
1135 			if (off > MAXPHYS / 2)
1136 				genlo = trunc_page(off - (MAXPHYS / 2));
1137 			else
1138 				genlo = 0;
1139 			lo = MAX(fslo, genlo);
1140 
1141 			/*
1142 			 * first look backward.
1143 			 */
1144 
1145 			npages = (off - lo) >> PAGE_SHIFT;
1146 			nback = npages;
1147 			uvn_findpages(uobj, off - PAGE_SIZE, &nback, &pgs[0],
1148 			    UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY|UFP_BACKWARD);
1149 			if (nback) {
1150 				memmove(&pgs[0], &pgs[npages - nback],
1151 				    nback * sizeof(pgs[0]));
1152 				if (npages - nback < nback)
1153 					memset(&pgs[nback], 0,
1154 					    (npages - nback) * sizeof(pgs[0]));
1155 				else
1156 					memset(&pgs[npages - nback], 0,
1157 					    nback * sizeof(pgs[0]));
1158 			}
1159 
1160 			/*
1161 			 * then plug in our page of interest.
1162 			 */
1163 
1164 			pgs[nback] = pg;
1165 
1166 			/*
1167 			 * then look forward to fill in the remaining space in
1168 			 * the array of pages.
1169 			 */
1170 
1171 			npages = MAXPAGES - nback - 1;
1172 			if (fshi)
1173 				npages = MIN(npages,
1174 					     (fshi - off - 1) >> PAGE_SHIFT);
1175 			uvn_findpages(uobj, off + PAGE_SIZE, &npages,
1176 			    &pgs[nback + 1],
1177 			    UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY);
1178 			npages += nback + 1;
1179 		} else {
1180 			pgs[0] = pg;
1181 			npages = 1;
1182 			nback = 0;
1183 		}
1184 
1185 		/*
1186 		 * apply FREE or DEACTIVATE options if requested.
1187 		 */
1188 
1189 		if (flags & (PGO_DEACTIVATE|PGO_FREE)) {
1190 			mutex_enter(&uvm_pageqlock);
1191 		}
1192 		for (i = 0; i < npages; i++) {
1193 			tpg = pgs[i];
1194 			KASSERT(tpg->uobject == uobj);
1195 			if (by_list && tpg == TAILQ_NEXT(pg, listq.queue))
1196 				pg = tpg;
1197 			if (tpg->offset < startoff || tpg->offset >= endoff)
1198 				continue;
1199 			if (flags & PGO_DEACTIVATE && tpg->wire_count == 0) {
1200 				uvm_pagedeactivate(tpg);
1201 			} else if (flags & PGO_FREE) {
1202 				pmap_page_protect(tpg, VM_PROT_NONE);
1203 				if (tpg->flags & PG_BUSY) {
1204 					tpg->flags |= freeflag;
1205 					if (pagedaemon) {
1206 						uvm_pageout_start(1);
1207 						uvm_pagedequeue(tpg);
1208 					}
1209 				} else {
1210 
1211 					/*
1212 					 * ``page is not busy''
1213 					 * implies that npages is 1
1214 					 * and needs_clean is false.
1215 					 */
1216 
1217 					nextpg = TAILQ_NEXT(tpg, listq.queue);
1218 					uvm_pagefree(tpg);
1219 					if (pagedaemon)
1220 						uvmexp.pdfreed++;
1221 				}
1222 			}
1223 		}
1224 		if (flags & (PGO_DEACTIVATE|PGO_FREE)) {
1225 			mutex_exit(&uvm_pageqlock);
1226 		}
1227 		if (needs_clean) {
1228 			modified = true;
1229 
1230 			/*
1231 			 * start the i/o.  if we're traversing by list,
1232 			 * keep our place in the list with a marker page.
1233 			 */
1234 
1235 			if (by_list) {
1236 				TAILQ_INSERT_AFTER(&uobj->memq, pg, &curmp,
1237 				    listq.queue);
1238 			}
1239 			mutex_exit(slock);
1240 			error = GOP_WRITE(vp, pgs, npages, flags);
1241 			mutex_enter(slock);
1242 			if (by_list) {
1243 				pg = TAILQ_NEXT(&curmp, listq.queue);
1244 				TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue);
1245 			}
1246 			if (error) {
1247 				break;
1248 			}
1249 			if (by_list) {
1250 				continue;
1251 			}
1252 		}
1253 
1254 		/*
1255 		 * find the next page and continue if there was no error.
1256 		 */
1257 
1258 		if (by_list) {
1259 			if (nextpg) {
1260 				pg = nextpg;
1261 				nextpg = NULL;
1262 			} else {
1263 				pg = TAILQ_NEXT(pg, listq.queue);
1264 			}
1265 		} else {
1266 			off += (npages - nback) << PAGE_SHIFT;
1267 			if (off < endoff) {
1268 				pg = uvm_pagelookup(uobj, off);
1269 			}
1270 		}
1271 	}
1272 	if (by_list) {
1273 		TAILQ_REMOVE(&uobj->memq, &endmp, listq.queue);
1274 	}
1275 
1276 	if (modified && (vp->v_iflag & VI_WRMAPDIRTY) != 0 &&
1277 	    (vp->v_type != VBLK ||
1278 	    (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) {
1279 		GOP_MARKUPDATE(vp, GOP_UPDATE_MODIFIED);
1280 	}
1281 
1282 	/*
1283 	 * if we're cleaning and there was nothing to clean,
1284 	 * take us off the syncer list.  if we started any i/o
1285 	 * and we're doing sync i/o, wait for all writes to finish.
1286 	 */
1287 
1288 	if (cleanall && wasclean && gp->g_dirtygen == dirtygen &&
1289 	    (vp->v_iflag & VI_ONWORKLST) != 0) {
1290 #if defined(DEBUG)
1291 		TAILQ_FOREACH(pg, &uobj->memq, listq.queue) {
1292 			if ((pg->flags & (PG_FAKE | PG_MARKER)) != 0) {
1293 				continue;
1294 			}
1295 			if ((pg->flags & PG_CLEAN) == 0) {
1296 				printf("%s: %p: !CLEAN\n", __func__, pg);
1297 			}
1298 			if (pmap_is_modified(pg)) {
1299 				printf("%s: %p: modified\n", __func__, pg);
1300 			}
1301 		}
1302 #endif /* defined(DEBUG) */
1303 		vp->v_iflag &= ~VI_WRMAPDIRTY;
1304 		if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL)
1305 			vn_syncer_remove_from_worklist(vp);
1306 	}
1307 
1308 #if !defined(DEBUG)
1309 skip_scan:
1310 #endif /* !defined(DEBUG) */
1311 
1312 	/* Wait for output to complete. */
1313 	if (!wasclean && !async && vp->v_numoutput != 0) {
1314 		while (vp->v_numoutput != 0)
1315 			cv_wait(&vp->v_cv, slock);
1316 	}
1317 	onworklst = (vp->v_iflag & VI_ONWORKLST) != 0;
1318 	mutex_exit(slock);
1319 
1320 	if ((flags & PGO_RECLAIM) != 0 && onworklst) {
1321 		/*
1322 		 * in the case of PGO_RECLAIM, ensure to make the vnode clean.
1323 		 * retrying is not a big deal because, in many cases,
1324 		 * uobj->uo_npages is already 0 here.
1325 		 */
1326 		mutex_enter(slock);
1327 		goto retry;
1328 	}
1329 
1330 	if (trans_mp) {
1331 		if (holds_wapbl)
1332 			WAPBL_END(trans_mp);
1333 		fstrans_done(trans_mp);
1334 	}
1335 
1336 	return (error);
1337 }
1338 
1339 /*
1340  * Default putrange method for file systems that do not care
1341  * how many pages are given to one GOP_WRITE() call.
1342  */
1343 void
1344 genfs_gop_putrange(struct vnode *vp, off_t off, off_t *lop, off_t *hip)
1345 {
1346 
1347 	*lop = 0;
1348 	*hip = 0;
1349 }
1350 
1351 int
1352 genfs_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, int flags)
1353 {
1354 	off_t off;
1355 	vaddr_t kva;
1356 	size_t len;
1357 	int error;
1358 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist);
1359 
1360 	UVMHIST_LOG(ubchist, "vp %#jx pgs %#jx npages %jd flags 0x%jx",
1361 	    (uintptr_t)vp, (uintptr_t)pgs, npages, flags);
1362 
1363 	off = pgs[0]->offset;
1364 	kva = uvm_pagermapin(pgs, npages,
1365 	    UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK);
1366 	len = npages << PAGE_SHIFT;
1367 
1368 	error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE,
1369 			    uvm_aio_biodone);
1370 
1371 	return error;
1372 }
1373 
1374 int
1375 genfs_gop_write_rwmap(struct vnode *vp, struct vm_page **pgs, int npages, int flags)
1376 {
1377 	off_t off;
1378 	vaddr_t kva;
1379 	size_t len;
1380 	int error;
1381 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist);
1382 
1383 	UVMHIST_LOG(ubchist, "vp %#jx pgs %#jx npages %jd flags 0x%jx",
1384 	    (uintptr_t)vp, (uintptr_t)pgs, npages, flags);
1385 
1386 	off = pgs[0]->offset;
1387 	kva = uvm_pagermapin(pgs, npages,
1388 	    UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK);
1389 	len = npages << PAGE_SHIFT;
1390 
1391 	error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE,
1392 			    uvm_aio_biodone);
1393 
1394 	return error;
1395 }
1396 
1397 /*
1398  * Backend routine for doing I/O to vnode pages.  Pages are already locked
1399  * and mapped into kernel memory.  Here we just look up the underlying
1400  * device block addresses and call the strategy routine.
1401  */
1402 
1403 static int
1404 genfs_do_io(struct vnode *vp, off_t off, vaddr_t kva, size_t len, int flags,
1405     enum uio_rw rw, void (*iodone)(struct buf *))
1406 {
1407 	int s, error;
1408 	int fs_bshift, dev_bshift;
1409 	off_t eof, offset, startoffset;
1410 	size_t bytes, iobytes, skipbytes;
1411 	struct buf *mbp, *bp;
1412 	const bool async = (flags & PGO_SYNCIO) == 0;
1413 	const bool lazy = (flags & PGO_LAZY) == 0;
1414 	const bool iowrite = rw == UIO_WRITE;
1415 	const int brw = iowrite ? B_WRITE : B_READ;
1416 	UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist);
1417 
1418 	UVMHIST_LOG(ubchist, "vp %#jx kva %#jx len 0x%jx flags 0x%jx",
1419 	    (uintptr_t)vp, (uintptr_t)kva, len, flags);
1420 
1421 	KASSERT(vp->v_size <= vp->v_writesize);
1422 	GOP_SIZE(vp, vp->v_writesize, &eof, 0);
1423 	if (vp->v_type != VBLK) {
1424 		fs_bshift = vp->v_mount->mnt_fs_bshift;
1425 		dev_bshift = vp->v_mount->mnt_dev_bshift;
1426 	} else {
1427 		fs_bshift = DEV_BSHIFT;
1428 		dev_bshift = DEV_BSHIFT;
1429 	}
1430 	error = 0;
1431 	startoffset = off;
1432 	bytes = MIN(len, eof - startoffset);
1433 	skipbytes = 0;
1434 	KASSERT(bytes != 0);
1435 
1436 	if (iowrite) {
1437 		mutex_enter(vp->v_interlock);
1438 		vp->v_numoutput += 2;
1439 		mutex_exit(vp->v_interlock);
1440 	}
1441 	mbp = getiobuf(vp, true);
1442 	UVMHIST_LOG(ubchist, "vp %#jx mbp %#jx num now %jd bytes 0x%jx",
1443 	    (uintptr_t)vp, (uintptr_t)mbp, vp->v_numoutput, bytes);
1444 	mbp->b_bufsize = len;
1445 	mbp->b_data = (void *)kva;
1446 	mbp->b_resid = mbp->b_bcount = bytes;
1447 	mbp->b_cflags = BC_BUSY | BC_AGE;
1448 	if (async) {
1449 		mbp->b_flags = brw | B_ASYNC;
1450 		mbp->b_iodone = iodone;
1451 	} else {
1452 		mbp->b_flags = brw;
1453 		mbp->b_iodone = NULL;
1454 	}
1455 	if (curlwp == uvm.pagedaemon_lwp)
1456 		BIO_SETPRIO(mbp, BPRIO_TIMELIMITED);
1457 	else if (async || lazy)
1458 		BIO_SETPRIO(mbp, BPRIO_TIMENONCRITICAL);
1459 	else
1460 		BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL);
1461 
1462 	bp = NULL;
1463 	for (offset = startoffset;
1464 	    bytes > 0;
1465 	    offset += iobytes, bytes -= iobytes) {
1466 		int run;
1467 		daddr_t lbn, blkno;
1468 		struct vnode *devvp;
1469 
1470 		/*
1471 		 * bmap the file to find out the blkno to read from and
1472 		 * how much we can read in one i/o.  if bmap returns an error,
1473 		 * skip the rest of the top-level i/o.
1474 		 */
1475 
1476 		lbn = offset >> fs_bshift;
1477 		error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run);
1478 		if (error) {
1479 			UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%jx -> %jd\n",
1480 			    lbn, error, 0, 0);
1481 			skipbytes += bytes;
1482 			bytes = 0;
1483 			goto loopdone;
1484 		}
1485 
1486 		/*
1487 		 * see how many pages can be read with this i/o.
1488 		 * reduce the i/o size if necessary to avoid
1489 		 * overwriting pages with valid data.
1490 		 */
1491 
1492 		iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset,
1493 		    bytes);
1494 
1495 		/*
1496 		 * if this block isn't allocated, zero it instead of
1497 		 * reading it.  unless we are going to allocate blocks,
1498 		 * mark the pages we zeroed PG_RDONLY.
1499 		 */
1500 
1501 		if (blkno == (daddr_t)-1) {
1502 			if (!iowrite) {
1503 				memset((char *)kva + (offset - startoffset), 0,
1504 				    iobytes);
1505 			}
1506 			skipbytes += iobytes;
1507 			continue;
1508 		}
1509 
1510 		/*
1511 		 * allocate a sub-buf for this piece of the i/o
1512 		 * (or just use mbp if there's only 1 piece),
1513 		 * and start it going.
1514 		 */
1515 
1516 		if (offset == startoffset && iobytes == bytes) {
1517 			bp = mbp;
1518 		} else {
1519 			UVMHIST_LOG(ubchist, "vp %#jx bp %#jx num now %jd",
1520 			    (uintptr_t)vp, (uintptr_t)bp, vp->v_numoutput, 0);
1521 			bp = getiobuf(vp, true);
1522 			nestiobuf_setup(mbp, bp, offset - startoffset, iobytes);
1523 		}
1524 		bp->b_lblkno = 0;
1525 
1526 		/* adjust physical blkno for partial blocks */
1527 		bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >>
1528 		    dev_bshift);
1529 
1530 		UVMHIST_LOG(ubchist,
1531 		    "bp %#jx offset 0x%jx bcount 0x%jx blkno 0x%jx",
1532 		    (uintptr_t)bp, offset, bp->b_bcount, bp->b_blkno);
1533 
1534 		VOP_STRATEGY(devvp, bp);
1535 	}
1536 
1537 loopdone:
1538 	if (skipbytes) {
1539 		UVMHIST_LOG(ubchist, "skipbytes %jd", skipbytes, 0,0,0);
1540 	}
1541 	nestiobuf_done(mbp, skipbytes, error);
1542 	if (async) {
1543 		UVMHIST_LOG(ubchist, "returning 0 (async)", 0,0,0,0);
1544 		return (0);
1545 	}
1546 	UVMHIST_LOG(ubchist, "waiting for mbp %#jx", (uintptr_t)mbp, 0, 0, 0);
1547 	error = biowait(mbp);
1548 	s = splbio();
1549 	(*iodone)(mbp);
1550 	splx(s);
1551 	UVMHIST_LOG(ubchist, "returning, error %jd", error, 0, 0, 0);
1552 	return (error);
1553 }
1554 
1555 int
1556 genfs_compat_getpages(void *v)
1557 {
1558 	struct vop_getpages_args /* {
1559 		struct vnode *a_vp;
1560 		voff_t a_offset;
1561 		struct vm_page **a_m;
1562 		int *a_count;
1563 		int a_centeridx;
1564 		vm_prot_t a_access_type;
1565 		int a_advice;
1566 		int a_flags;
1567 	} */ *ap = v;
1568 
1569 	off_t origoffset;
1570 	struct vnode *vp = ap->a_vp;
1571 	struct uvm_object *uobj = &vp->v_uobj;
1572 	struct vm_page *pg, **pgs;
1573 	vaddr_t kva;
1574 	int i, error, orignpages, npages;
1575 	struct iovec iov;
1576 	struct uio uio;
1577 	kauth_cred_t cred = curlwp->l_cred;
1578 	const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0;
1579 
1580 	error = 0;
1581 	origoffset = ap->a_offset;
1582 	orignpages = *ap->a_count;
1583 	pgs = ap->a_m;
1584 
1585 	if (ap->a_flags & PGO_LOCKED) {
1586 		uvn_findpages(uobj, origoffset, ap->a_count, ap->a_m,
1587 		    UFP_NOWAIT|UFP_NOALLOC| (memwrite ? UFP_NORDONLY : 0));
1588 
1589 		error = ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0;
1590 		if (error == 0 && memwrite) {
1591 			genfs_markdirty(vp);
1592 		}
1593 		return error;
1594 	}
1595 	if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= vp->v_size) {
1596 		mutex_exit(uobj->vmobjlock);
1597 		return EINVAL;
1598 	}
1599 	if ((ap->a_flags & PGO_SYNCIO) == 0) {
1600 		mutex_exit(uobj->vmobjlock);
1601 		return 0;
1602 	}
1603 	npages = orignpages;
1604 	uvn_findpages(uobj, origoffset, &npages, pgs, UFP_ALL);
1605 	mutex_exit(uobj->vmobjlock);
1606 	kva = uvm_pagermapin(pgs, npages,
1607 	    UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK);
1608 	for (i = 0; i < npages; i++) {
1609 		pg = pgs[i];
1610 		if ((pg->flags & PG_FAKE) == 0) {
1611 			continue;
1612 		}
1613 		iov.iov_base = (char *)kva + (i << PAGE_SHIFT);
1614 		iov.iov_len = PAGE_SIZE;
1615 		uio.uio_iov = &iov;
1616 		uio.uio_iovcnt = 1;
1617 		uio.uio_offset = origoffset + (i << PAGE_SHIFT);
1618 		uio.uio_rw = UIO_READ;
1619 		uio.uio_resid = PAGE_SIZE;
1620 		UIO_SETUP_SYSSPACE(&uio);
1621 		/* XXX vn_lock */
1622 		error = VOP_READ(vp, &uio, 0, cred);
1623 		if (error) {
1624 			break;
1625 		}
1626 		if (uio.uio_resid) {
1627 			memset(iov.iov_base, 0, uio.uio_resid);
1628 		}
1629 	}
1630 	uvm_pagermapout(kva, npages);
1631 	mutex_enter(uobj->vmobjlock);
1632 	mutex_enter(&uvm_pageqlock);
1633 	for (i = 0; i < npages; i++) {
1634 		pg = pgs[i];
1635 		if (error && (pg->flags & PG_FAKE) != 0) {
1636 			pg->flags |= PG_RELEASED;
1637 		} else {
1638 			pmap_clear_modify(pg);
1639 			uvm_pageactivate(pg);
1640 		}
1641 	}
1642 	if (error) {
1643 		uvm_page_unbusy(pgs, npages);
1644 	}
1645 	mutex_exit(&uvm_pageqlock);
1646 	if (error == 0 && memwrite) {
1647 		genfs_markdirty(vp);
1648 	}
1649 	mutex_exit(uobj->vmobjlock);
1650 	return error;
1651 }
1652 
1653 int
1654 genfs_compat_gop_write(struct vnode *vp, struct vm_page **pgs, int npages,
1655     int flags)
1656 {
1657 	off_t offset;
1658 	struct iovec iov;
1659 	struct uio uio;
1660 	kauth_cred_t cred = curlwp->l_cred;
1661 	struct buf *bp;
1662 	vaddr_t kva;
1663 	int error;
1664 
1665 	offset = pgs[0]->offset;
1666 	kva = uvm_pagermapin(pgs, npages,
1667 	    UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK);
1668 
1669 	iov.iov_base = (void *)kva;
1670 	iov.iov_len = npages << PAGE_SHIFT;
1671 	uio.uio_iov = &iov;
1672 	uio.uio_iovcnt = 1;
1673 	uio.uio_offset = offset;
1674 	uio.uio_rw = UIO_WRITE;
1675 	uio.uio_resid = npages << PAGE_SHIFT;
1676 	UIO_SETUP_SYSSPACE(&uio);
1677 	/* XXX vn_lock */
1678 	error = VOP_WRITE(vp, &uio, 0, cred);
1679 
1680 	mutex_enter(vp->v_interlock);
1681 	vp->v_numoutput++;
1682 	mutex_exit(vp->v_interlock);
1683 
1684 	bp = getiobuf(vp, true);
1685 	bp->b_cflags = BC_BUSY | BC_AGE;
1686 	bp->b_lblkno = offset >> vp->v_mount->mnt_fs_bshift;
1687 	bp->b_data = (char *)kva;
1688 	bp->b_bcount = npages << PAGE_SHIFT;
1689 	bp->b_bufsize = npages << PAGE_SHIFT;
1690 	bp->b_resid = 0;
1691 	bp->b_error = error;
1692 	uvm_aio_aiodone(bp);
1693 	return (error);
1694 }
1695 
1696 /*
1697  * Process a uio using direct I/O.  If we reach a part of the request
1698  * which cannot be processed in this fashion for some reason, just return.
1699  * The caller must handle some additional part of the request using
1700  * buffered I/O before trying direct I/O again.
1701  */
1702 
1703 void
1704 genfs_directio(struct vnode *vp, struct uio *uio, int ioflag)
1705 {
1706 	struct vmspace *vs;
1707 	struct iovec *iov;
1708 	vaddr_t va;
1709 	size_t len;
1710 	const int mask = DEV_BSIZE - 1;
1711 	int error;
1712 	bool need_wapbl = (vp->v_mount && vp->v_mount->mnt_wapbl &&
1713 	    (ioflag & IO_JOURNALLOCKED) == 0);
1714 
1715 	/*
1716 	 * We only support direct I/O to user space for now.
1717 	 */
1718 
1719 	if (VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) {
1720 		return;
1721 	}
1722 
1723 	/*
1724 	 * If the vnode is mapped, we would need to get the getpages lock
1725 	 * to stabilize the bmap, but then we would get into trouble while
1726 	 * locking the pages if the pages belong to this same vnode (or a
1727 	 * multi-vnode cascade to the same effect).  Just fall back to
1728 	 * buffered I/O if the vnode is mapped to avoid this mess.
1729 	 */
1730 
1731 	if (vp->v_vflag & VV_MAPPED) {
1732 		return;
1733 	}
1734 
1735 	if (need_wapbl) {
1736 		error = WAPBL_BEGIN(vp->v_mount);
1737 		if (error)
1738 			return;
1739 	}
1740 
1741 	/*
1742 	 * Do as much of the uio as possible with direct I/O.
1743 	 */
1744 
1745 	vs = uio->uio_vmspace;
1746 	while (uio->uio_resid) {
1747 		iov = uio->uio_iov;
1748 		if (iov->iov_len == 0) {
1749 			uio->uio_iov++;
1750 			uio->uio_iovcnt--;
1751 			continue;
1752 		}
1753 		va = (vaddr_t)iov->iov_base;
1754 		len = MIN(iov->iov_len, genfs_maxdio);
1755 		len &= ~mask;
1756 
1757 		/*
1758 		 * If the next chunk is smaller than DEV_BSIZE or extends past
1759 		 * the current EOF, then fall back to buffered I/O.
1760 		 */
1761 
1762 		if (len == 0 || uio->uio_offset + len > vp->v_size) {
1763 			break;
1764 		}
1765 
1766 		/*
1767 		 * Check alignment.  The file offset must be at least
1768 		 * sector-aligned.  The exact constraint on memory alignment
1769 		 * is very hardware-dependent, but requiring sector-aligned
1770 		 * addresses there too is safe.
1771 		 */
1772 
1773 		if (uio->uio_offset & mask || va & mask) {
1774 			break;
1775 		}
1776 		error = genfs_do_directio(vs, va, len, vp, uio->uio_offset,
1777 					  uio->uio_rw);
1778 		if (error) {
1779 			break;
1780 		}
1781 		iov->iov_base = (char *)iov->iov_base + len;
1782 		iov->iov_len -= len;
1783 		uio->uio_offset += len;
1784 		uio->uio_resid -= len;
1785 	}
1786 
1787 	if (need_wapbl)
1788 		WAPBL_END(vp->v_mount);
1789 }
1790 
1791 /*
1792  * Iodone routine for direct I/O.  We don't do much here since the request is
1793  * always synchronous, so the caller will do most of the work after biowait().
1794  */
1795 
1796 static void
1797 genfs_dio_iodone(struct buf *bp)
1798 {
1799 
1800 	KASSERT((bp->b_flags & B_ASYNC) == 0);
1801 	if ((bp->b_flags & B_READ) == 0 && (bp->b_cflags & BC_AGE) != 0) {
1802 		mutex_enter(bp->b_objlock);
1803 		vwakeup(bp);
1804 		mutex_exit(bp->b_objlock);
1805 	}
1806 	putiobuf(bp);
1807 }
1808 
1809 /*
1810  * Process one chunk of a direct I/O request.
1811  */
1812 
1813 static int
1814 genfs_do_directio(struct vmspace *vs, vaddr_t uva, size_t len, struct vnode *vp,
1815     off_t off, enum uio_rw rw)
1816 {
1817 	struct vm_map *map;
1818 	struct pmap *upm, *kpm __unused;
1819 	size_t klen = round_page(uva + len) - trunc_page(uva);
1820 	off_t spoff, epoff;
1821 	vaddr_t kva, puva;
1822 	paddr_t pa;
1823 	vm_prot_t prot;
1824 	int error, rv __diagused, poff, koff;
1825 	const int pgoflags = PGO_CLEANIT | PGO_SYNCIO | PGO_JOURNALLOCKED |
1826 		(rw == UIO_WRITE ? PGO_FREE : 0);
1827 
1828 	/*
1829 	 * For writes, verify that this range of the file already has fully
1830 	 * allocated backing store.  If there are any holes, just punt and
1831 	 * make the caller take the buffered write path.
1832 	 */
1833 
1834 	if (rw == UIO_WRITE) {
1835 		daddr_t lbn, elbn, blkno;
1836 		int bsize, bshift, run;
1837 
1838 		bshift = vp->v_mount->mnt_fs_bshift;
1839 		bsize = 1 << bshift;
1840 		lbn = off >> bshift;
1841 		elbn = (off + len + bsize - 1) >> bshift;
1842 		while (lbn < elbn) {
1843 			error = VOP_BMAP(vp, lbn, NULL, &blkno, &run);
1844 			if (error) {
1845 				return error;
1846 			}
1847 			if (blkno == (daddr_t)-1) {
1848 				return ENOSPC;
1849 			}
1850 			lbn += 1 + run;
1851 		}
1852 	}
1853 
1854 	/*
1855 	 * Flush any cached pages for parts of the file that we're about to
1856 	 * access.  If we're writing, invalidate pages as well.
1857 	 */
1858 
1859 	spoff = trunc_page(off);
1860 	epoff = round_page(off + len);
1861 	mutex_enter(vp->v_interlock);
1862 	error = VOP_PUTPAGES(vp, spoff, epoff, pgoflags);
1863 	if (error) {
1864 		return error;
1865 	}
1866 
1867 	/*
1868 	 * Wire the user pages and remap them into kernel memory.
1869 	 */
1870 
1871 	prot = rw == UIO_READ ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_READ;
1872 	error = uvm_vslock(vs, (void *)uva, len, prot);
1873 	if (error) {
1874 		return error;
1875 	}
1876 
1877 	map = &vs->vm_map;
1878 	upm = vm_map_pmap(map);
1879 	kpm = vm_map_pmap(kernel_map);
1880 	puva = trunc_page(uva);
1881 	kva = uvm_km_alloc(kernel_map, klen, atop(puva) & uvmexp.colormask,
1882 	    UVM_KMF_VAONLY | UVM_KMF_WAITVA | UVM_KMF_COLORMATCH);
1883 	for (poff = 0; poff < klen; poff += PAGE_SIZE) {
1884 		rv = pmap_extract(upm, puva + poff, &pa);
1885 		KASSERT(rv);
1886 		pmap_kenter_pa(kva + poff, pa, prot, PMAP_WIRED);
1887 	}
1888 	pmap_update(kpm);
1889 
1890 	/*
1891 	 * Do the I/O.
1892 	 */
1893 
1894 	koff = uva - trunc_page(uva);
1895 	error = genfs_do_io(vp, off, kva + koff, len, PGO_SYNCIO, rw,
1896 			    genfs_dio_iodone);
1897 
1898 	/*
1899 	 * Tear down the kernel mapping.
1900 	 */
1901 
1902 	pmap_kremove(kva, klen);
1903 	pmap_update(kpm);
1904 	uvm_km_free(kernel_map, kva, klen, UVM_KMF_VAONLY);
1905 
1906 	/*
1907 	 * Unwire the user pages.
1908 	 */
1909 
1910 	uvm_vsunlock(vs, (void *)uva, len);
1911 	return error;
1912 }
1913