1 /* $NetBSD: genfs_io.c,v 1.16 2008/12/01 11:22:12 joerg Exp $ */ 2 3 /* 4 * Copyright (c) 1982, 1986, 1989, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the University nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 */ 32 33 #include <sys/cdefs.h> 34 __KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.16 2008/12/01 11:22:12 joerg Exp $"); 35 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/proc.h> 39 #include <sys/kernel.h> 40 #include <sys/mount.h> 41 #include <sys/namei.h> 42 #include <sys/vnode.h> 43 #include <sys/fcntl.h> 44 #include <sys/kmem.h> 45 #include <sys/poll.h> 46 #include <sys/mman.h> 47 #include <sys/file.h> 48 #include <sys/kauth.h> 49 #include <sys/fstrans.h> 50 #include <sys/buf.h> 51 52 #include <miscfs/genfs/genfs.h> 53 #include <miscfs/genfs/genfs_node.h> 54 #include <miscfs/specfs/specdev.h> 55 56 #include <uvm/uvm.h> 57 #include <uvm/uvm_pager.h> 58 59 static int genfs_do_directio(struct vmspace *, vaddr_t, size_t, struct vnode *, 60 off_t, enum uio_rw); 61 static void genfs_dio_iodone(struct buf *); 62 63 static int genfs_do_io(struct vnode *, off_t, vaddr_t, size_t, int, enum uio_rw, 64 void (*)(struct buf *)); 65 static inline void genfs_rel_pages(struct vm_page **, int); 66 67 int genfs_maxdio = MAXPHYS; 68 69 static inline void 70 genfs_rel_pages(struct vm_page **pgs, int npages) 71 { 72 int i; 73 74 for (i = 0; i < npages; i++) { 75 struct vm_page *pg = pgs[i]; 76 77 if (pg == NULL || pg == PGO_DONTCARE) 78 continue; 79 if (pg->flags & PG_FAKE) { 80 pg->flags |= PG_RELEASED; 81 } 82 } 83 mutex_enter(&uvm_pageqlock); 84 uvm_page_unbusy(pgs, npages); 85 mutex_exit(&uvm_pageqlock); 86 } 87 88 /* 89 * generic VM getpages routine. 90 * Return PG_BUSY pages for the given range, 91 * reading from backing store if necessary. 92 */ 93 94 int 95 genfs_getpages(void *v) 96 { 97 struct vop_getpages_args /* { 98 struct vnode *a_vp; 99 voff_t a_offset; 100 struct vm_page **a_m; 101 int *a_count; 102 int a_centeridx; 103 vm_prot_t a_access_type; 104 int a_advice; 105 int a_flags; 106 } */ *ap = v; 107 108 off_t newsize, diskeof, memeof; 109 off_t offset, origoffset, startoffset, endoffset; 110 daddr_t lbn, blkno; 111 int i, error, npages, orignpages, npgs, run, ridx, pidx, pcount; 112 int fs_bshift, fs_bsize, dev_bshift; 113 const int flags = ap->a_flags; 114 size_t bytes, iobytes, tailstart, tailbytes, totalbytes, skipbytes; 115 vaddr_t kva; 116 struct buf *bp, *mbp; 117 struct vnode *vp = ap->a_vp; 118 struct vnode *devvp; 119 struct genfs_node *gp = VTOG(vp); 120 struct uvm_object *uobj = &vp->v_uobj; 121 struct vm_page *pg, **pgs, *pgs_onstack[UBC_MAX_PAGES]; 122 int pgs_size; 123 kauth_cred_t cred = curlwp->l_cred; /* XXXUBC curlwp */ 124 const bool async = (flags & PGO_SYNCIO) == 0; 125 const bool write = (ap->a_access_type & VM_PROT_WRITE) != 0; 126 bool sawhole = false; 127 bool has_trans = false; 128 const bool overwrite = (flags & PGO_OVERWRITE) != 0; 129 const bool blockalloc = write && (flags & PGO_NOBLOCKALLOC) == 0; 130 voff_t origvsize; 131 UVMHIST_FUNC("genfs_getpages"); UVMHIST_CALLED(ubchist); 132 133 UVMHIST_LOG(ubchist, "vp %p off 0x%x/%x count %d", 134 vp, ap->a_offset >> 32, ap->a_offset, *ap->a_count); 135 136 KASSERT(vp->v_type == VREG || vp->v_type == VDIR || 137 vp->v_type == VLNK || vp->v_type == VBLK); 138 139 pgs = NULL; 140 pgs_size = 0; 141 142 startover: 143 error = 0; 144 origvsize = vp->v_size; 145 origoffset = ap->a_offset; 146 orignpages = *ap->a_count; 147 GOP_SIZE(vp, origvsize, &diskeof, 0); 148 if (flags & PGO_PASTEOF) { 149 #if defined(DIAGNOSTIC) 150 off_t writeeof; 151 #endif /* defined(DIAGNOSTIC) */ 152 153 newsize = MAX(origvsize, 154 origoffset + (orignpages << PAGE_SHIFT)); 155 GOP_SIZE(vp, newsize, &memeof, GOP_SIZE_MEM); 156 #if defined(DIAGNOSTIC) 157 GOP_SIZE(vp, vp->v_writesize, &writeeof, GOP_SIZE_MEM); 158 if (newsize > round_page(writeeof)) { 159 panic("%s: past eof", __func__); 160 } 161 #endif /* defined(DIAGNOSTIC) */ 162 } else { 163 GOP_SIZE(vp, origvsize, &memeof, GOP_SIZE_MEM); 164 } 165 KASSERT(ap->a_centeridx >= 0 || ap->a_centeridx <= orignpages); 166 KASSERT((origoffset & (PAGE_SIZE - 1)) == 0 && origoffset >= 0); 167 KASSERT(orignpages > 0); 168 169 /* 170 * Bounds-check the request. 171 */ 172 173 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= memeof) { 174 if ((flags & PGO_LOCKED) == 0) { 175 mutex_exit(&uobj->vmobjlock); 176 } 177 UVMHIST_LOG(ubchist, "off 0x%x count %d goes past EOF 0x%x", 178 origoffset, *ap->a_count, memeof,0); 179 error = EINVAL; 180 goto out_err; 181 } 182 183 /* uobj is locked */ 184 185 if ((flags & PGO_NOTIMESTAMP) == 0 && 186 (vp->v_type != VBLK || 187 (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) { 188 int updflags = 0; 189 190 if ((vp->v_mount->mnt_flag & MNT_NOATIME) == 0) { 191 updflags = GOP_UPDATE_ACCESSED; 192 } 193 if (write) { 194 updflags |= GOP_UPDATE_MODIFIED; 195 } 196 if (updflags != 0) { 197 GOP_MARKUPDATE(vp, updflags); 198 } 199 } 200 201 if (write) { 202 gp->g_dirtygen++; 203 if ((vp->v_iflag & VI_ONWORKLST) == 0) { 204 vn_syncer_add_to_worklist(vp, filedelay); 205 } 206 if ((vp->v_iflag & (VI_WRMAP|VI_WRMAPDIRTY)) == VI_WRMAP) { 207 vp->v_iflag |= VI_WRMAPDIRTY; 208 } 209 } 210 211 /* 212 * For PGO_LOCKED requests, just return whatever's in memory. 213 */ 214 215 if (flags & PGO_LOCKED) { 216 int nfound; 217 218 npages = *ap->a_count; 219 #if defined(DEBUG) 220 for (i = 0; i < npages; i++) { 221 pg = ap->a_m[i]; 222 KASSERT(pg == NULL || pg == PGO_DONTCARE); 223 } 224 #endif /* defined(DEBUG) */ 225 nfound = uvn_findpages(uobj, origoffset, &npages, 226 ap->a_m, UFP_NOWAIT|UFP_NOALLOC|(write ? UFP_NORDONLY : 0)); 227 KASSERT(npages == *ap->a_count); 228 if (nfound == 0) { 229 error = EBUSY; 230 goto out_err; 231 } 232 if (!rw_tryenter(&gp->g_glock, RW_READER)) { 233 genfs_rel_pages(ap->a_m, npages); 234 235 /* 236 * restore the array. 237 */ 238 239 for (i = 0; i < npages; i++) { 240 pg = ap->a_m[i]; 241 242 if (pg != NULL || pg != PGO_DONTCARE) { 243 ap->a_m[i] = NULL; 244 } 245 } 246 } else { 247 rw_exit(&gp->g_glock); 248 } 249 error = (ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0); 250 goto out_err; 251 } 252 mutex_exit(&uobj->vmobjlock); 253 254 /* 255 * find the requested pages and make some simple checks. 256 * leave space in the page array for a whole block. 257 */ 258 259 if (vp->v_type != VBLK) { 260 fs_bshift = vp->v_mount->mnt_fs_bshift; 261 dev_bshift = vp->v_mount->mnt_dev_bshift; 262 } else { 263 fs_bshift = DEV_BSHIFT; 264 dev_bshift = DEV_BSHIFT; 265 } 266 fs_bsize = 1 << fs_bshift; 267 268 orignpages = MIN(orignpages, 269 round_page(memeof - origoffset) >> PAGE_SHIFT); 270 npages = orignpages; 271 startoffset = origoffset & ~(fs_bsize - 1); 272 endoffset = round_page((origoffset + (npages << PAGE_SHIFT) + 273 fs_bsize - 1) & ~(fs_bsize - 1)); 274 endoffset = MIN(endoffset, round_page(memeof)); 275 ridx = (origoffset - startoffset) >> PAGE_SHIFT; 276 277 pgs_size = sizeof(struct vm_page *) * 278 ((endoffset - startoffset) >> PAGE_SHIFT); 279 if (pgs_size > sizeof(pgs_onstack)) { 280 pgs = kmem_zalloc(pgs_size, async ? KM_NOSLEEP : KM_SLEEP); 281 if (pgs == NULL) { 282 pgs = pgs_onstack; 283 error = ENOMEM; 284 goto out_err; 285 } 286 } else { 287 pgs = pgs_onstack; 288 (void)memset(pgs, 0, pgs_size); 289 } 290 291 292 UVMHIST_LOG(ubchist, "ridx %d npages %d startoff %ld endoff %ld", 293 ridx, npages, startoffset, endoffset); 294 295 if (!has_trans) { 296 fstrans_start(vp->v_mount, FSTRANS_SHARED); 297 has_trans = true; 298 } 299 300 /* 301 * hold g_glock to prevent a race with truncate. 302 * 303 * check if our idea of v_size is still valid. 304 */ 305 306 if (blockalloc) { 307 rw_enter(&gp->g_glock, RW_WRITER); 308 } else { 309 rw_enter(&gp->g_glock, RW_READER); 310 } 311 mutex_enter(&uobj->vmobjlock); 312 if (vp->v_size < origvsize) { 313 rw_exit(&gp->g_glock); 314 if (pgs != pgs_onstack) 315 kmem_free(pgs, pgs_size); 316 goto startover; 317 } 318 319 if (uvn_findpages(uobj, origoffset, &npages, &pgs[ridx], 320 async ? UFP_NOWAIT : UFP_ALL) != orignpages) { 321 rw_exit(&gp->g_glock); 322 KASSERT(async != 0); 323 genfs_rel_pages(&pgs[ridx], orignpages); 324 mutex_exit(&uobj->vmobjlock); 325 error = EBUSY; 326 goto out_err; 327 } 328 329 /* 330 * if the pages are already resident, just return them. 331 */ 332 333 for (i = 0; i < npages; i++) { 334 struct vm_page *pg1 = pgs[ridx + i]; 335 336 if ((pg1->flags & PG_FAKE) || 337 (blockalloc && (pg1->flags & PG_RDONLY))) { 338 break; 339 } 340 } 341 if (i == npages) { 342 rw_exit(&gp->g_glock); 343 UVMHIST_LOG(ubchist, "returning cached pages", 0,0,0,0); 344 npages += ridx; 345 goto out; 346 } 347 348 /* 349 * if PGO_OVERWRITE is set, don't bother reading the pages. 350 */ 351 352 if (overwrite) { 353 rw_exit(&gp->g_glock); 354 UVMHIST_LOG(ubchist, "PGO_OVERWRITE",0,0,0,0); 355 356 for (i = 0; i < npages; i++) { 357 struct vm_page *pg1 = pgs[ridx + i]; 358 359 pg1->flags &= ~(PG_RDONLY|PG_CLEAN); 360 } 361 npages += ridx; 362 goto out; 363 } 364 365 /* 366 * the page wasn't resident and we're not overwriting, 367 * so we're going to have to do some i/o. 368 * find any additional pages needed to cover the expanded range. 369 */ 370 371 npages = (endoffset - startoffset) >> PAGE_SHIFT; 372 if (startoffset != origoffset || npages != orignpages) { 373 374 /* 375 * we need to avoid deadlocks caused by locking 376 * additional pages at lower offsets than pages we 377 * already have locked. unlock them all and start over. 378 */ 379 380 genfs_rel_pages(&pgs[ridx], orignpages); 381 memset(pgs, 0, pgs_size); 382 383 UVMHIST_LOG(ubchist, "reset npages start 0x%x end 0x%x", 384 startoffset, endoffset, 0,0); 385 npgs = npages; 386 if (uvn_findpages(uobj, startoffset, &npgs, pgs, 387 async ? UFP_NOWAIT : UFP_ALL) != npages) { 388 rw_exit(&gp->g_glock); 389 KASSERT(async != 0); 390 genfs_rel_pages(pgs, npages); 391 mutex_exit(&uobj->vmobjlock); 392 error = EBUSY; 393 goto out_err; 394 } 395 } 396 mutex_exit(&uobj->vmobjlock); 397 398 /* 399 * read the desired page(s). 400 */ 401 402 totalbytes = npages << PAGE_SHIFT; 403 bytes = MIN(totalbytes, MAX(diskeof - startoffset, 0)); 404 tailbytes = totalbytes - bytes; 405 skipbytes = 0; 406 407 kva = uvm_pagermapin(pgs, npages, 408 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 409 410 mbp = getiobuf(vp, true); 411 mbp->b_bufsize = totalbytes; 412 mbp->b_data = (void *)kva; 413 mbp->b_resid = mbp->b_bcount = bytes; 414 mbp->b_cflags = BC_BUSY; 415 if (async) { 416 mbp->b_flags = B_READ | B_ASYNC; 417 mbp->b_iodone = uvm_aio_biodone; 418 } else { 419 mbp->b_flags = B_READ; 420 mbp->b_iodone = NULL; 421 } 422 if (async) 423 BIO_SETPRIO(mbp, BPRIO_TIMELIMITED); 424 else 425 BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL); 426 427 /* 428 * if EOF is in the middle of the range, zero the part past EOF. 429 * skip over pages which are not PG_FAKE since in that case they have 430 * valid data that we need to preserve. 431 */ 432 433 tailstart = bytes; 434 while (tailbytes > 0) { 435 const int len = PAGE_SIZE - (tailstart & PAGE_MASK); 436 437 KASSERT(len <= tailbytes); 438 if ((pgs[tailstart >> PAGE_SHIFT]->flags & PG_FAKE) != 0) { 439 memset((void *)(kva + tailstart), 0, len); 440 UVMHIST_LOG(ubchist, "tailbytes %p 0x%x 0x%x", 441 kva, tailstart, len, 0); 442 } 443 tailstart += len; 444 tailbytes -= len; 445 } 446 447 /* 448 * now loop over the pages, reading as needed. 449 */ 450 451 bp = NULL; 452 for (offset = startoffset; 453 bytes > 0; 454 offset += iobytes, bytes -= iobytes) { 455 456 /* 457 * skip pages which don't need to be read. 458 */ 459 460 pidx = (offset - startoffset) >> PAGE_SHIFT; 461 while ((pgs[pidx]->flags & PG_FAKE) == 0) { 462 size_t b; 463 464 KASSERT((offset & (PAGE_SIZE - 1)) == 0); 465 if ((pgs[pidx]->flags & PG_RDONLY)) { 466 sawhole = true; 467 } 468 b = MIN(PAGE_SIZE, bytes); 469 offset += b; 470 bytes -= b; 471 skipbytes += b; 472 pidx++; 473 UVMHIST_LOG(ubchist, "skipping, new offset 0x%x", 474 offset, 0,0,0); 475 if (bytes == 0) { 476 goto loopdone; 477 } 478 } 479 480 /* 481 * bmap the file to find out the blkno to read from and 482 * how much we can read in one i/o. if bmap returns an error, 483 * skip the rest of the top-level i/o. 484 */ 485 486 lbn = offset >> fs_bshift; 487 error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run); 488 if (error) { 489 UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%x -> %d\n", 490 lbn, error,0,0); 491 skipbytes += bytes; 492 goto loopdone; 493 } 494 495 /* 496 * see how many pages can be read with this i/o. 497 * reduce the i/o size if necessary to avoid 498 * overwriting pages with valid data. 499 */ 500 501 iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset, 502 bytes); 503 if (offset + iobytes > round_page(offset)) { 504 pcount = 1; 505 while (pidx + pcount < npages && 506 pgs[pidx + pcount]->flags & PG_FAKE) { 507 pcount++; 508 } 509 iobytes = MIN(iobytes, (pcount << PAGE_SHIFT) - 510 (offset - trunc_page(offset))); 511 } 512 513 /* 514 * if this block isn't allocated, zero it instead of 515 * reading it. unless we are going to allocate blocks, 516 * mark the pages we zeroed PG_RDONLY. 517 */ 518 519 if (blkno < 0) { 520 int holepages = (round_page(offset + iobytes) - 521 trunc_page(offset)) >> PAGE_SHIFT; 522 UVMHIST_LOG(ubchist, "lbn 0x%x -> HOLE", lbn,0,0,0); 523 524 sawhole = true; 525 memset((char *)kva + (offset - startoffset), 0, 526 iobytes); 527 skipbytes += iobytes; 528 529 for (i = 0; i < holepages; i++) { 530 if (write) { 531 pgs[pidx + i]->flags &= ~PG_CLEAN; 532 } 533 if (!blockalloc) { 534 pgs[pidx + i]->flags |= PG_RDONLY; 535 } 536 } 537 continue; 538 } 539 540 /* 541 * allocate a sub-buf for this piece of the i/o 542 * (or just use mbp if there's only 1 piece), 543 * and start it going. 544 */ 545 546 if (offset == startoffset && iobytes == bytes) { 547 bp = mbp; 548 } else { 549 bp = getiobuf(vp, true); 550 nestiobuf_setup(mbp, bp, offset - startoffset, iobytes); 551 } 552 bp->b_lblkno = 0; 553 554 /* adjust physical blkno for partial blocks */ 555 bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >> 556 dev_bshift); 557 558 UVMHIST_LOG(ubchist, 559 "bp %p offset 0x%x bcount 0x%x blkno 0x%x", 560 bp, offset, iobytes, bp->b_blkno); 561 562 VOP_STRATEGY(devvp, bp); 563 } 564 565 loopdone: 566 nestiobuf_done(mbp, skipbytes, error); 567 if (async) { 568 UVMHIST_LOG(ubchist, "returning 0 (async)",0,0,0,0); 569 rw_exit(&gp->g_glock); 570 error = 0; 571 goto out_err; 572 } 573 if (bp != NULL) { 574 error = biowait(mbp); 575 } 576 putiobuf(mbp); 577 uvm_pagermapout(kva, npages); 578 579 /* 580 * if this we encountered a hole then we have to do a little more work. 581 * for read faults, we marked the page PG_RDONLY so that future 582 * write accesses to the page will fault again. 583 * for write faults, we must make sure that the backing store for 584 * the page is completely allocated while the pages are locked. 585 */ 586 587 if (!error && sawhole && blockalloc) { 588 /* 589 * XXX: This assumes that we come here only via 590 * the mmio path 591 */ 592 if (vp->v_mount->mnt_wapbl) { 593 error = WAPBL_BEGIN(vp->v_mount); 594 } 595 596 if (!error) { 597 error = GOP_ALLOC(vp, startoffset, 598 npages << PAGE_SHIFT, 0, cred); 599 if (vp->v_mount->mnt_wapbl) { 600 WAPBL_END(vp->v_mount); 601 } 602 } 603 604 UVMHIST_LOG(ubchist, "gop_alloc off 0x%x/0x%x -> %d", 605 startoffset, npages << PAGE_SHIFT, error,0); 606 if (!error) { 607 for (i = 0; i < npages; i++) { 608 if (pgs[i] == NULL) { 609 continue; 610 } 611 pgs[i]->flags &= ~(PG_CLEAN|PG_RDONLY); 612 UVMHIST_LOG(ubchist, "mark dirty pg %p", 613 pgs[i],0,0,0); 614 } 615 } 616 } 617 rw_exit(&gp->g_glock); 618 mutex_enter(&uobj->vmobjlock); 619 620 /* 621 * we're almost done! release the pages... 622 * for errors, we free the pages. 623 * otherwise we activate them and mark them as valid and clean. 624 * also, unbusy pages that were not actually requested. 625 */ 626 627 if (error) { 628 for (i = 0; i < npages; i++) { 629 if (pgs[i] == NULL) { 630 continue; 631 } 632 UVMHIST_LOG(ubchist, "examining pg %p flags 0x%x", 633 pgs[i], pgs[i]->flags, 0,0); 634 if (pgs[i]->flags & PG_FAKE) { 635 pgs[i]->flags |= PG_RELEASED; 636 } 637 } 638 mutex_enter(&uvm_pageqlock); 639 uvm_page_unbusy(pgs, npages); 640 mutex_exit(&uvm_pageqlock); 641 mutex_exit(&uobj->vmobjlock); 642 UVMHIST_LOG(ubchist, "returning error %d", error,0,0,0); 643 goto out_err; 644 } 645 646 out: 647 UVMHIST_LOG(ubchist, "succeeding, npages %d", npages,0,0,0); 648 error = 0; 649 mutex_enter(&uvm_pageqlock); 650 for (i = 0; i < npages; i++) { 651 pg = pgs[i]; 652 if (pg == NULL) { 653 continue; 654 } 655 UVMHIST_LOG(ubchist, "examining pg %p flags 0x%x", 656 pg, pg->flags, 0,0); 657 if (pg->flags & PG_FAKE && !overwrite) { 658 pg->flags &= ~(PG_FAKE); 659 pmap_clear_modify(pgs[i]); 660 } 661 KASSERT(!write || !blockalloc || (pg->flags & PG_RDONLY) == 0); 662 if (i < ridx || i >= ridx + orignpages || async) { 663 UVMHIST_LOG(ubchist, "unbusy pg %p offset 0x%x", 664 pg, pg->offset,0,0); 665 if (pg->flags & PG_WANTED) { 666 wakeup(pg); 667 } 668 if (pg->flags & PG_FAKE) { 669 KASSERT(overwrite); 670 uvm_pagezero(pg); 671 } 672 if (pg->flags & PG_RELEASED) { 673 uvm_pagefree(pg); 674 continue; 675 } 676 uvm_pageenqueue(pg); 677 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE); 678 UVM_PAGE_OWN(pg, NULL); 679 } 680 } 681 mutex_exit(&uvm_pageqlock); 682 mutex_exit(&uobj->vmobjlock); 683 if (ap->a_m != NULL) { 684 memcpy(ap->a_m, &pgs[ridx], 685 orignpages * sizeof(struct vm_page *)); 686 } 687 688 out_err: 689 if (pgs != NULL && pgs != pgs_onstack) 690 kmem_free(pgs, pgs_size); 691 if (has_trans) 692 fstrans_done(vp->v_mount); 693 return (error); 694 } 695 696 /* 697 * generic VM putpages routine. 698 * Write the given range of pages to backing store. 699 * 700 * => "offhi == 0" means flush all pages at or after "offlo". 701 * => object should be locked by caller. we return with the 702 * object unlocked. 703 * => if PGO_CLEANIT or PGO_SYNCIO is set, we may block (due to I/O). 704 * thus, a caller might want to unlock higher level resources 705 * (e.g. vm_map) before calling flush. 706 * => if neither PGO_CLEANIT nor PGO_SYNCIO is set, we will not block 707 * => if PGO_ALLPAGES is set, then all pages in the object will be processed. 708 * => NOTE: we rely on the fact that the object's memq is a TAILQ and 709 * that new pages are inserted on the tail end of the list. thus, 710 * we can make a complete pass through the object in one go by starting 711 * at the head and working towards the tail (new pages are put in 712 * front of us). 713 * => NOTE: we are allowed to lock the page queues, so the caller 714 * must not be holding the page queue lock. 715 * 716 * note on "cleaning" object and PG_BUSY pages: 717 * this routine is holding the lock on the object. the only time 718 * that it can run into a PG_BUSY page that it does not own is if 719 * some other process has started I/O on the page (e.g. either 720 * a pagein, or a pageout). if the PG_BUSY page is being paged 721 * in, then it can not be dirty (!PG_CLEAN) because no one has 722 * had a chance to modify it yet. if the PG_BUSY page is being 723 * paged out then it means that someone else has already started 724 * cleaning the page for us (how nice!). in this case, if we 725 * have syncio specified, then after we make our pass through the 726 * object we need to wait for the other PG_BUSY pages to clear 727 * off (i.e. we need to do an iosync). also note that once a 728 * page is PG_BUSY it must stay in its object until it is un-busyed. 729 * 730 * note on page traversal: 731 * we can traverse the pages in an object either by going down the 732 * linked list in "uobj->memq", or we can go over the address range 733 * by page doing hash table lookups for each address. depending 734 * on how many pages are in the object it may be cheaper to do one 735 * or the other. we set "by_list" to true if we are using memq. 736 * if the cost of a hash lookup was equal to the cost of the list 737 * traversal we could compare the number of pages in the start->stop 738 * range to the total number of pages in the object. however, it 739 * seems that a hash table lookup is more expensive than the linked 740 * list traversal, so we multiply the number of pages in the 741 * range by an estimate of the relatively higher cost of the hash lookup. 742 */ 743 744 int 745 genfs_putpages(void *v) 746 { 747 struct vop_putpages_args /* { 748 struct vnode *a_vp; 749 voff_t a_offlo; 750 voff_t a_offhi; 751 int a_flags; 752 } */ *ap = v; 753 754 return genfs_do_putpages(ap->a_vp, ap->a_offlo, ap->a_offhi, 755 ap->a_flags, NULL); 756 } 757 758 int 759 genfs_do_putpages(struct vnode *vp, off_t startoff, off_t endoff, 760 int origflags, struct vm_page **busypg) 761 { 762 struct uvm_object *uobj = &vp->v_uobj; 763 kmutex_t *slock = &uobj->vmobjlock; 764 off_t off; 765 /* Even for strange MAXPHYS, the shift rounds down to a page */ 766 #define maxpages (MAXPHYS >> PAGE_SHIFT) 767 int i, error, npages, nback; 768 int freeflag; 769 struct vm_page *pgs[maxpages], *pg, *nextpg, *tpg, curmp, endmp; 770 bool wasclean, by_list, needs_clean, yld; 771 bool async = (origflags & PGO_SYNCIO) == 0; 772 bool pagedaemon = curlwp == uvm.pagedaemon_lwp; 773 struct lwp *l = curlwp ? curlwp : &lwp0; 774 struct genfs_node *gp = VTOG(vp); 775 int flags; 776 int dirtygen; 777 bool modified; 778 bool need_wapbl; 779 bool has_trans; 780 bool cleanall; 781 bool onworklst; 782 783 UVMHIST_FUNC("genfs_putpages"); UVMHIST_CALLED(ubchist); 784 785 KASSERT(origflags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)); 786 KASSERT((startoff & PAGE_MASK) == 0 && (endoff & PAGE_MASK) == 0); 787 KASSERT(startoff < endoff || endoff == 0); 788 789 UVMHIST_LOG(ubchist, "vp %p pages %d off 0x%x len 0x%x", 790 vp, uobj->uo_npages, startoff, endoff - startoff); 791 792 has_trans = false; 793 need_wapbl = (!pagedaemon && vp->v_mount && vp->v_mount->mnt_wapbl && 794 (origflags & PGO_JOURNALLOCKED) == 0); 795 796 retry: 797 modified = false; 798 flags = origflags; 799 KASSERT((vp->v_iflag & VI_ONWORKLST) != 0 || 800 (vp->v_iflag & VI_WRMAPDIRTY) == 0); 801 if (uobj->uo_npages == 0) { 802 if (vp->v_iflag & VI_ONWORKLST) { 803 vp->v_iflag &= ~VI_WRMAPDIRTY; 804 if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL) 805 vn_syncer_remove_from_worklist(vp); 806 } 807 if (has_trans) { 808 if (need_wapbl) 809 WAPBL_END(vp->v_mount); 810 fstrans_done(vp->v_mount); 811 } 812 mutex_exit(slock); 813 return (0); 814 } 815 816 /* 817 * the vnode has pages, set up to process the request. 818 */ 819 820 if (!has_trans && (flags & PGO_CLEANIT) != 0) { 821 mutex_exit(slock); 822 if (pagedaemon) { 823 error = fstrans_start_nowait(vp->v_mount, FSTRANS_LAZY); 824 if (error) 825 return error; 826 } else 827 fstrans_start(vp->v_mount, FSTRANS_LAZY); 828 if (need_wapbl) { 829 error = WAPBL_BEGIN(vp->v_mount); 830 if (error) { 831 fstrans_done(vp->v_mount); 832 return error; 833 } 834 } 835 has_trans = true; 836 mutex_enter(slock); 837 goto retry; 838 } 839 840 error = 0; 841 wasclean = (vp->v_numoutput == 0); 842 off = startoff; 843 if (endoff == 0 || flags & PGO_ALLPAGES) { 844 endoff = trunc_page(LLONG_MAX); 845 } 846 by_list = (uobj->uo_npages <= 847 ((endoff - startoff) >> PAGE_SHIFT) * UVM_PAGE_HASH_PENALTY); 848 849 #if !defined(DEBUG) 850 /* 851 * if this vnode is known not to have dirty pages, 852 * don't bother to clean it out. 853 */ 854 855 if ((vp->v_iflag & VI_ONWORKLST) == 0) { 856 if ((flags & (PGO_FREE|PGO_DEACTIVATE)) == 0) { 857 goto skip_scan; 858 } 859 flags &= ~PGO_CLEANIT; 860 } 861 #endif /* !defined(DEBUG) */ 862 863 /* 864 * start the loop. when scanning by list, hold the last page 865 * in the list before we start. pages allocated after we start 866 * will be added to the end of the list, so we can stop at the 867 * current last page. 868 */ 869 870 cleanall = (flags & PGO_CLEANIT) != 0 && wasclean && 871 startoff == 0 && endoff == trunc_page(LLONG_MAX) && 872 (vp->v_iflag & VI_ONWORKLST) != 0; 873 dirtygen = gp->g_dirtygen; 874 freeflag = pagedaemon ? PG_PAGEOUT : PG_RELEASED; 875 if (by_list) { 876 curmp.uobject = uobj; 877 curmp.offset = (voff_t)-1; 878 curmp.flags = PG_BUSY; 879 endmp.uobject = uobj; 880 endmp.offset = (voff_t)-1; 881 endmp.flags = PG_BUSY; 882 pg = TAILQ_FIRST(&uobj->memq); 883 TAILQ_INSERT_TAIL(&uobj->memq, &endmp, listq.queue); 884 uvm_lwp_hold(l); 885 } else { 886 pg = uvm_pagelookup(uobj, off); 887 } 888 nextpg = NULL; 889 while (by_list || off < endoff) { 890 891 /* 892 * if the current page is not interesting, move on to the next. 893 */ 894 895 KASSERT(pg == NULL || pg->uobject == uobj); 896 KASSERT(pg == NULL || 897 (pg->flags & (PG_RELEASED|PG_PAGEOUT)) == 0 || 898 (pg->flags & PG_BUSY) != 0); 899 if (by_list) { 900 if (pg == &endmp) { 901 break; 902 } 903 if (pg->offset < startoff || pg->offset >= endoff || 904 pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 905 if (pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 906 wasclean = false; 907 } 908 pg = TAILQ_NEXT(pg, listq.queue); 909 continue; 910 } 911 off = pg->offset; 912 } else if (pg == NULL || pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 913 if (pg != NULL) { 914 wasclean = false; 915 } 916 off += PAGE_SIZE; 917 if (off < endoff) { 918 pg = uvm_pagelookup(uobj, off); 919 } 920 continue; 921 } 922 923 /* 924 * if the current page needs to be cleaned and it's busy, 925 * wait for it to become unbusy. 926 */ 927 928 yld = (l->l_cpu->ci_schedstate.spc_flags & 929 SPCF_SHOULDYIELD) && !pagedaemon; 930 if (pg->flags & PG_BUSY || yld) { 931 UVMHIST_LOG(ubchist, "busy %p", pg,0,0,0); 932 if (flags & PGO_BUSYFAIL && pg->flags & PG_BUSY) { 933 UVMHIST_LOG(ubchist, "busyfail %p", pg, 0,0,0); 934 error = EDEADLK; 935 if (busypg != NULL) 936 *busypg = pg; 937 break; 938 } 939 if (pagedaemon) { 940 /* 941 * someone has taken the page while we 942 * dropped the lock for fstrans_start. 943 */ 944 break; 945 } 946 if (by_list) { 947 TAILQ_INSERT_BEFORE(pg, &curmp, listq.queue); 948 UVMHIST_LOG(ubchist, "curmp next %p", 949 TAILQ_NEXT(&curmp, listq.queue), 0,0,0); 950 } 951 if (yld) { 952 mutex_exit(slock); 953 preempt(); 954 mutex_enter(slock); 955 } else { 956 pg->flags |= PG_WANTED; 957 UVM_UNLOCK_AND_WAIT(pg, slock, 0, "genput", 0); 958 mutex_enter(slock); 959 } 960 if (by_list) { 961 UVMHIST_LOG(ubchist, "after next %p", 962 TAILQ_NEXT(&curmp, listq.queue), 0,0,0); 963 pg = TAILQ_NEXT(&curmp, listq.queue); 964 TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue); 965 } else { 966 pg = uvm_pagelookup(uobj, off); 967 } 968 continue; 969 } 970 971 /* 972 * if we're freeing, remove all mappings of the page now. 973 * if we're cleaning, check if the page is needs to be cleaned. 974 */ 975 976 if (flags & PGO_FREE) { 977 pmap_page_protect(pg, VM_PROT_NONE); 978 } else if (flags & PGO_CLEANIT) { 979 980 /* 981 * if we still have some hope to pull this vnode off 982 * from the syncer queue, write-protect the page. 983 */ 984 985 if (cleanall && wasclean && 986 gp->g_dirtygen == dirtygen) { 987 988 /* 989 * uobj pages get wired only by uvm_fault 990 * where uobj is locked. 991 */ 992 993 if (pg->wire_count == 0) { 994 pmap_page_protect(pg, 995 VM_PROT_READ|VM_PROT_EXECUTE); 996 } else { 997 cleanall = false; 998 } 999 } 1000 } 1001 1002 if (flags & PGO_CLEANIT) { 1003 needs_clean = pmap_clear_modify(pg) || 1004 (pg->flags & PG_CLEAN) == 0; 1005 pg->flags |= PG_CLEAN; 1006 } else { 1007 needs_clean = false; 1008 } 1009 1010 /* 1011 * if we're cleaning, build a cluster. 1012 * the cluster will consist of pages which are currently dirty, 1013 * but they will be returned to us marked clean. 1014 * if not cleaning, just operate on the one page. 1015 */ 1016 1017 if (needs_clean) { 1018 KDASSERT((vp->v_iflag & VI_ONWORKLST)); 1019 wasclean = false; 1020 memset(pgs, 0, sizeof(pgs)); 1021 pg->flags |= PG_BUSY; 1022 UVM_PAGE_OWN(pg, "genfs_putpages"); 1023 1024 /* 1025 * first look backward. 1026 */ 1027 1028 npages = MIN(maxpages >> 1, off >> PAGE_SHIFT); 1029 nback = npages; 1030 uvn_findpages(uobj, off - PAGE_SIZE, &nback, &pgs[0], 1031 UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY|UFP_BACKWARD); 1032 if (nback) { 1033 memmove(&pgs[0], &pgs[npages - nback], 1034 nback * sizeof(pgs[0])); 1035 if (npages - nback < nback) 1036 memset(&pgs[nback], 0, 1037 (npages - nback) * sizeof(pgs[0])); 1038 else 1039 memset(&pgs[npages - nback], 0, 1040 nback * sizeof(pgs[0])); 1041 } 1042 1043 /* 1044 * then plug in our page of interest. 1045 */ 1046 1047 pgs[nback] = pg; 1048 1049 /* 1050 * then look forward to fill in the remaining space in 1051 * the array of pages. 1052 */ 1053 1054 npages = maxpages - nback - 1; 1055 uvn_findpages(uobj, off + PAGE_SIZE, &npages, 1056 &pgs[nback + 1], 1057 UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY); 1058 npages += nback + 1; 1059 } else { 1060 pgs[0] = pg; 1061 npages = 1; 1062 nback = 0; 1063 } 1064 1065 /* 1066 * apply FREE or DEACTIVATE options if requested. 1067 */ 1068 1069 if (flags & (PGO_DEACTIVATE|PGO_FREE)) { 1070 mutex_enter(&uvm_pageqlock); 1071 } 1072 for (i = 0; i < npages; i++) { 1073 tpg = pgs[i]; 1074 KASSERT(tpg->uobject == uobj); 1075 if (by_list && tpg == TAILQ_NEXT(pg, listq.queue)) 1076 pg = tpg; 1077 if (tpg->offset < startoff || tpg->offset >= endoff) 1078 continue; 1079 if (flags & PGO_DEACTIVATE && tpg->wire_count == 0) { 1080 uvm_pagedeactivate(tpg); 1081 } else if (flags & PGO_FREE) { 1082 pmap_page_protect(tpg, VM_PROT_NONE); 1083 if (tpg->flags & PG_BUSY) { 1084 tpg->flags |= freeflag; 1085 if (pagedaemon) { 1086 uvm_pageout_start(1); 1087 uvm_pagedequeue(tpg); 1088 } 1089 } else { 1090 1091 /* 1092 * ``page is not busy'' 1093 * implies that npages is 1 1094 * and needs_clean is false. 1095 */ 1096 1097 nextpg = TAILQ_NEXT(tpg, listq.queue); 1098 uvm_pagefree(tpg); 1099 if (pagedaemon) 1100 uvmexp.pdfreed++; 1101 } 1102 } 1103 } 1104 if (flags & (PGO_DEACTIVATE|PGO_FREE)) { 1105 mutex_exit(&uvm_pageqlock); 1106 } 1107 if (needs_clean) { 1108 modified = true; 1109 1110 /* 1111 * start the i/o. if we're traversing by list, 1112 * keep our place in the list with a marker page. 1113 */ 1114 1115 if (by_list) { 1116 TAILQ_INSERT_AFTER(&uobj->memq, pg, &curmp, 1117 listq.queue); 1118 } 1119 mutex_exit(slock); 1120 error = GOP_WRITE(vp, pgs, npages, flags); 1121 mutex_enter(slock); 1122 if (by_list) { 1123 pg = TAILQ_NEXT(&curmp, listq.queue); 1124 TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue); 1125 } 1126 if (error) { 1127 break; 1128 } 1129 if (by_list) { 1130 continue; 1131 } 1132 } 1133 1134 /* 1135 * find the next page and continue if there was no error. 1136 */ 1137 1138 if (by_list) { 1139 if (nextpg) { 1140 pg = nextpg; 1141 nextpg = NULL; 1142 } else { 1143 pg = TAILQ_NEXT(pg, listq.queue); 1144 } 1145 } else { 1146 off += (npages - nback) << PAGE_SHIFT; 1147 if (off < endoff) { 1148 pg = uvm_pagelookup(uobj, off); 1149 } 1150 } 1151 } 1152 if (by_list) { 1153 TAILQ_REMOVE(&uobj->memq, &endmp, listq.queue); 1154 uvm_lwp_rele(l); 1155 } 1156 1157 if (modified && (vp->v_iflag & VI_WRMAPDIRTY) != 0 && 1158 (vp->v_type != VBLK || 1159 (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) { 1160 GOP_MARKUPDATE(vp, GOP_UPDATE_MODIFIED); 1161 } 1162 1163 /* 1164 * if we're cleaning and there was nothing to clean, 1165 * take us off the syncer list. if we started any i/o 1166 * and we're doing sync i/o, wait for all writes to finish. 1167 */ 1168 1169 if (cleanall && wasclean && gp->g_dirtygen == dirtygen && 1170 (vp->v_iflag & VI_ONWORKLST) != 0) { 1171 #if defined(DEBUG) 1172 TAILQ_FOREACH(pg, &uobj->memq, listq.queue) { 1173 if ((pg->flags & PG_CLEAN) == 0) { 1174 printf("%s: %p: !CLEAN\n", __func__, pg); 1175 } 1176 if (pmap_is_modified(pg)) { 1177 printf("%s: %p: modified\n", __func__, pg); 1178 } 1179 } 1180 #endif /* defined(DEBUG) */ 1181 vp->v_iflag &= ~VI_WRMAPDIRTY; 1182 if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL) 1183 vn_syncer_remove_from_worklist(vp); 1184 } 1185 1186 #if !defined(DEBUG) 1187 skip_scan: 1188 #endif /* !defined(DEBUG) */ 1189 1190 /* Wait for output to complete. */ 1191 if (!wasclean && !async && vp->v_numoutput != 0) { 1192 while (vp->v_numoutput != 0) 1193 cv_wait(&vp->v_cv, slock); 1194 } 1195 onworklst = (vp->v_iflag & VI_ONWORKLST) != 0; 1196 mutex_exit(slock); 1197 1198 if ((flags & PGO_RECLAIM) != 0 && onworklst) { 1199 /* 1200 * in the case of PGO_RECLAIM, ensure to make the vnode clean. 1201 * retrying is not a big deal because, in many cases, 1202 * uobj->uo_npages is already 0 here. 1203 */ 1204 mutex_enter(slock); 1205 goto retry; 1206 } 1207 1208 if (has_trans) { 1209 if (need_wapbl) 1210 WAPBL_END(vp->v_mount); 1211 fstrans_done(vp->v_mount); 1212 } 1213 1214 return (error); 1215 } 1216 1217 int 1218 genfs_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, int flags) 1219 { 1220 off_t off; 1221 vaddr_t kva; 1222 size_t len; 1223 int error; 1224 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1225 1226 UVMHIST_LOG(ubchist, "vp %p pgs %p npages %d flags 0x%x", 1227 vp, pgs, npages, flags); 1228 1229 off = pgs[0]->offset; 1230 kva = uvm_pagermapin(pgs, npages, 1231 UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK); 1232 len = npages << PAGE_SHIFT; 1233 1234 error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE, 1235 uvm_aio_biodone); 1236 1237 return error; 1238 } 1239 1240 int 1241 genfs_gop_write_rwmap(struct vnode *vp, struct vm_page **pgs, int npages, int flags) 1242 { 1243 off_t off; 1244 vaddr_t kva; 1245 size_t len; 1246 int error; 1247 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1248 1249 UVMHIST_LOG(ubchist, "vp %p pgs %p npages %d flags 0x%x", 1250 vp, pgs, npages, flags); 1251 1252 off = pgs[0]->offset; 1253 kva = uvm_pagermapin(pgs, npages, 1254 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 1255 len = npages << PAGE_SHIFT; 1256 1257 error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE, 1258 uvm_aio_biodone); 1259 1260 return error; 1261 } 1262 1263 /* 1264 * Backend routine for doing I/O to vnode pages. Pages are already locked 1265 * and mapped into kernel memory. Here we just look up the underlying 1266 * device block addresses and call the strategy routine. 1267 */ 1268 1269 static int 1270 genfs_do_io(struct vnode *vp, off_t off, vaddr_t kva, size_t len, int flags, 1271 enum uio_rw rw, void (*iodone)(struct buf *)) 1272 { 1273 int s, error, run; 1274 int fs_bshift, dev_bshift; 1275 off_t eof, offset, startoffset; 1276 size_t bytes, iobytes, skipbytes; 1277 daddr_t lbn, blkno; 1278 struct buf *mbp, *bp; 1279 struct vnode *devvp; 1280 bool async = (flags & PGO_SYNCIO) == 0; 1281 bool write = rw == UIO_WRITE; 1282 int brw = write ? B_WRITE : B_READ; 1283 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1284 1285 UVMHIST_LOG(ubchist, "vp %p kva %p len 0x%x flags 0x%x", 1286 vp, kva, len, flags); 1287 1288 KASSERT(vp->v_size <= vp->v_writesize); 1289 GOP_SIZE(vp, vp->v_writesize, &eof, 0); 1290 if (vp->v_type != VBLK) { 1291 fs_bshift = vp->v_mount->mnt_fs_bshift; 1292 dev_bshift = vp->v_mount->mnt_dev_bshift; 1293 } else { 1294 fs_bshift = DEV_BSHIFT; 1295 dev_bshift = DEV_BSHIFT; 1296 } 1297 error = 0; 1298 startoffset = off; 1299 bytes = MIN(len, eof - startoffset); 1300 skipbytes = 0; 1301 KASSERT(bytes != 0); 1302 1303 if (write) { 1304 mutex_enter(&vp->v_interlock); 1305 vp->v_numoutput += 2; 1306 mutex_exit(&vp->v_interlock); 1307 } 1308 mbp = getiobuf(vp, true); 1309 UVMHIST_LOG(ubchist, "vp %p mbp %p num now %d bytes 0x%x", 1310 vp, mbp, vp->v_numoutput, bytes); 1311 mbp->b_bufsize = len; 1312 mbp->b_data = (void *)kva; 1313 mbp->b_resid = mbp->b_bcount = bytes; 1314 mbp->b_cflags = BC_BUSY | BC_AGE; 1315 if (async) { 1316 mbp->b_flags = brw | B_ASYNC; 1317 mbp->b_iodone = iodone; 1318 } else { 1319 mbp->b_flags = brw; 1320 mbp->b_iodone = NULL; 1321 } 1322 if (curlwp == uvm.pagedaemon_lwp) 1323 BIO_SETPRIO(mbp, BPRIO_TIMELIMITED); 1324 else if (async) 1325 BIO_SETPRIO(mbp, BPRIO_TIMENONCRITICAL); 1326 else 1327 BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL); 1328 1329 bp = NULL; 1330 for (offset = startoffset; 1331 bytes > 0; 1332 offset += iobytes, bytes -= iobytes) { 1333 lbn = offset >> fs_bshift; 1334 error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run); 1335 if (error) { 1336 UVMHIST_LOG(ubchist, "VOP_BMAP() -> %d", error,0,0,0); 1337 skipbytes += bytes; 1338 bytes = 0; 1339 break; 1340 } 1341 1342 iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset, 1343 bytes); 1344 if (blkno == (daddr_t)-1) { 1345 if (!write) { 1346 memset((char *)kva + (offset - startoffset), 0, 1347 iobytes); 1348 } 1349 skipbytes += iobytes; 1350 continue; 1351 } 1352 1353 /* if it's really one i/o, don't make a second buf */ 1354 if (offset == startoffset && iobytes == bytes) { 1355 bp = mbp; 1356 } else { 1357 UVMHIST_LOG(ubchist, "vp %p bp %p num now %d", 1358 vp, bp, vp->v_numoutput, 0); 1359 bp = getiobuf(vp, true); 1360 nestiobuf_setup(mbp, bp, offset - startoffset, iobytes); 1361 } 1362 bp->b_lblkno = 0; 1363 1364 /* adjust physical blkno for partial blocks */ 1365 bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >> 1366 dev_bshift); 1367 UVMHIST_LOG(ubchist, 1368 "vp %p offset 0x%x bcount 0x%x blkno 0x%x", 1369 vp, offset, bp->b_bcount, bp->b_blkno); 1370 1371 VOP_STRATEGY(devvp, bp); 1372 } 1373 if (skipbytes) { 1374 UVMHIST_LOG(ubchist, "skipbytes %d", skipbytes, 0,0,0); 1375 } 1376 nestiobuf_done(mbp, skipbytes, error); 1377 if (async) { 1378 UVMHIST_LOG(ubchist, "returning 0 (async)", 0,0,0,0); 1379 return (0); 1380 } 1381 UVMHIST_LOG(ubchist, "waiting for mbp %p", mbp,0,0,0); 1382 error = biowait(mbp); 1383 s = splbio(); 1384 (*iodone)(mbp); 1385 splx(s); 1386 UVMHIST_LOG(ubchist, "returning, error %d", error,0,0,0); 1387 return (error); 1388 } 1389 1390 /* 1391 * VOP_PUTPAGES() for vnodes which never have pages. 1392 */ 1393 1394 int 1395 genfs_null_putpages(void *v) 1396 { 1397 struct vop_putpages_args /* { 1398 struct vnode *a_vp; 1399 voff_t a_offlo; 1400 voff_t a_offhi; 1401 int a_flags; 1402 } */ *ap = v; 1403 struct vnode *vp = ap->a_vp; 1404 1405 KASSERT(vp->v_uobj.uo_npages == 0); 1406 mutex_exit(&vp->v_interlock); 1407 return (0); 1408 } 1409 1410 int 1411 genfs_compat_getpages(void *v) 1412 { 1413 struct vop_getpages_args /* { 1414 struct vnode *a_vp; 1415 voff_t a_offset; 1416 struct vm_page **a_m; 1417 int *a_count; 1418 int a_centeridx; 1419 vm_prot_t a_access_type; 1420 int a_advice; 1421 int a_flags; 1422 } */ *ap = v; 1423 1424 off_t origoffset; 1425 struct vnode *vp = ap->a_vp; 1426 struct uvm_object *uobj = &vp->v_uobj; 1427 struct vm_page *pg, **pgs; 1428 vaddr_t kva; 1429 int i, error, orignpages, npages; 1430 struct iovec iov; 1431 struct uio uio; 1432 kauth_cred_t cred = curlwp->l_cred; 1433 bool write = (ap->a_access_type & VM_PROT_WRITE) != 0; 1434 1435 error = 0; 1436 origoffset = ap->a_offset; 1437 orignpages = *ap->a_count; 1438 pgs = ap->a_m; 1439 1440 if (write && (vp->v_iflag & VI_ONWORKLST) == 0) { 1441 vn_syncer_add_to_worklist(vp, filedelay); 1442 } 1443 if (ap->a_flags & PGO_LOCKED) { 1444 uvn_findpages(uobj, origoffset, ap->a_count, ap->a_m, 1445 UFP_NOWAIT|UFP_NOALLOC| (write ? UFP_NORDONLY : 0)); 1446 1447 return (ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0); 1448 } 1449 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= vp->v_size) { 1450 mutex_exit(&uobj->vmobjlock); 1451 return (EINVAL); 1452 } 1453 if ((ap->a_flags & PGO_SYNCIO) == 0) { 1454 mutex_exit(&uobj->vmobjlock); 1455 return 0; 1456 } 1457 npages = orignpages; 1458 uvn_findpages(uobj, origoffset, &npages, pgs, UFP_ALL); 1459 mutex_exit(&uobj->vmobjlock); 1460 kva = uvm_pagermapin(pgs, npages, 1461 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 1462 for (i = 0; i < npages; i++) { 1463 pg = pgs[i]; 1464 if ((pg->flags & PG_FAKE) == 0) { 1465 continue; 1466 } 1467 iov.iov_base = (char *)kva + (i << PAGE_SHIFT); 1468 iov.iov_len = PAGE_SIZE; 1469 uio.uio_iov = &iov; 1470 uio.uio_iovcnt = 1; 1471 uio.uio_offset = origoffset + (i << PAGE_SHIFT); 1472 uio.uio_rw = UIO_READ; 1473 uio.uio_resid = PAGE_SIZE; 1474 UIO_SETUP_SYSSPACE(&uio); 1475 /* XXX vn_lock */ 1476 error = VOP_READ(vp, &uio, 0, cred); 1477 if (error) { 1478 break; 1479 } 1480 if (uio.uio_resid) { 1481 memset(iov.iov_base, 0, uio.uio_resid); 1482 } 1483 } 1484 uvm_pagermapout(kva, npages); 1485 mutex_enter(&uobj->vmobjlock); 1486 mutex_enter(&uvm_pageqlock); 1487 for (i = 0; i < npages; i++) { 1488 pg = pgs[i]; 1489 if (error && (pg->flags & PG_FAKE) != 0) { 1490 pg->flags |= PG_RELEASED; 1491 } else { 1492 pmap_clear_modify(pg); 1493 uvm_pageactivate(pg); 1494 } 1495 } 1496 if (error) { 1497 uvm_page_unbusy(pgs, npages); 1498 } 1499 mutex_exit(&uvm_pageqlock); 1500 mutex_exit(&uobj->vmobjlock); 1501 return (error); 1502 } 1503 1504 int 1505 genfs_compat_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, 1506 int flags) 1507 { 1508 off_t offset; 1509 struct iovec iov; 1510 struct uio uio; 1511 kauth_cred_t cred = curlwp->l_cred; 1512 struct buf *bp; 1513 vaddr_t kva; 1514 int error; 1515 1516 offset = pgs[0]->offset; 1517 kva = uvm_pagermapin(pgs, npages, 1518 UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK); 1519 1520 iov.iov_base = (void *)kva; 1521 iov.iov_len = npages << PAGE_SHIFT; 1522 uio.uio_iov = &iov; 1523 uio.uio_iovcnt = 1; 1524 uio.uio_offset = offset; 1525 uio.uio_rw = UIO_WRITE; 1526 uio.uio_resid = npages << PAGE_SHIFT; 1527 UIO_SETUP_SYSSPACE(&uio); 1528 /* XXX vn_lock */ 1529 error = VOP_WRITE(vp, &uio, 0, cred); 1530 1531 mutex_enter(&vp->v_interlock); 1532 vp->v_numoutput++; 1533 mutex_exit(&vp->v_interlock); 1534 1535 bp = getiobuf(vp, true); 1536 bp->b_cflags = BC_BUSY | BC_AGE; 1537 bp->b_lblkno = offset >> vp->v_mount->mnt_fs_bshift; 1538 bp->b_data = (char *)kva; 1539 bp->b_bcount = npages << PAGE_SHIFT; 1540 bp->b_bufsize = npages << PAGE_SHIFT; 1541 bp->b_resid = 0; 1542 bp->b_error = error; 1543 uvm_aio_aiodone(bp); 1544 return (error); 1545 } 1546 1547 /* 1548 * Process a uio using direct I/O. If we reach a part of the request 1549 * which cannot be processed in this fashion for some reason, just return. 1550 * The caller must handle some additional part of the request using 1551 * buffered I/O before trying direct I/O again. 1552 */ 1553 1554 void 1555 genfs_directio(struct vnode *vp, struct uio *uio, int ioflag) 1556 { 1557 struct vmspace *vs; 1558 struct iovec *iov; 1559 vaddr_t va; 1560 size_t len; 1561 const int mask = DEV_BSIZE - 1; 1562 int error; 1563 bool need_wapbl = (vp->v_mount && vp->v_mount->mnt_wapbl && 1564 (ioflag & IO_JOURNALLOCKED) == 0); 1565 1566 /* 1567 * We only support direct I/O to user space for now. 1568 */ 1569 1570 if (VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) { 1571 return; 1572 } 1573 1574 /* 1575 * If the vnode is mapped, we would need to get the getpages lock 1576 * to stabilize the bmap, but then we would get into trouble whil e 1577 * locking the pages if the pages belong to this same vnode (or a 1578 * multi-vnode cascade to the same effect). Just fall back to 1579 * buffered I/O if the vnode is mapped to avoid this mess. 1580 */ 1581 1582 if (vp->v_vflag & VV_MAPPED) { 1583 return; 1584 } 1585 1586 if (need_wapbl) { 1587 error = WAPBL_BEGIN(vp->v_mount); 1588 if (error) 1589 return; 1590 } 1591 1592 /* 1593 * Do as much of the uio as possible with direct I/O. 1594 */ 1595 1596 vs = uio->uio_vmspace; 1597 while (uio->uio_resid) { 1598 iov = uio->uio_iov; 1599 if (iov->iov_len == 0) { 1600 uio->uio_iov++; 1601 uio->uio_iovcnt--; 1602 continue; 1603 } 1604 va = (vaddr_t)iov->iov_base; 1605 len = MIN(iov->iov_len, genfs_maxdio); 1606 len &= ~mask; 1607 1608 /* 1609 * If the next chunk is smaller than DEV_BSIZE or extends past 1610 * the current EOF, then fall back to buffered I/O. 1611 */ 1612 1613 if (len == 0 || uio->uio_offset + len > vp->v_size) { 1614 break; 1615 } 1616 1617 /* 1618 * Check alignment. The file offset must be at least 1619 * sector-aligned. The exact constraint on memory alignment 1620 * is very hardware-dependent, but requiring sector-aligned 1621 * addresses there too is safe. 1622 */ 1623 1624 if (uio->uio_offset & mask || va & mask) { 1625 break; 1626 } 1627 error = genfs_do_directio(vs, va, len, vp, uio->uio_offset, 1628 uio->uio_rw); 1629 if (error) { 1630 break; 1631 } 1632 iov->iov_base = (char *)iov->iov_base + len; 1633 iov->iov_len -= len; 1634 uio->uio_offset += len; 1635 uio->uio_resid -= len; 1636 } 1637 1638 if (need_wapbl) 1639 WAPBL_END(vp->v_mount); 1640 } 1641 1642 /* 1643 * Iodone routine for direct I/O. We don't do much here since the request is 1644 * always synchronous, so the caller will do most of the work after biowait(). 1645 */ 1646 1647 static void 1648 genfs_dio_iodone(struct buf *bp) 1649 { 1650 1651 KASSERT((bp->b_flags & B_ASYNC) == 0); 1652 if ((bp->b_flags & B_READ) == 0 && (bp->b_cflags & BC_AGE) != 0) { 1653 mutex_enter(bp->b_objlock); 1654 vwakeup(bp); 1655 mutex_exit(bp->b_objlock); 1656 } 1657 putiobuf(bp); 1658 } 1659 1660 /* 1661 * Process one chunk of a direct I/O request. 1662 */ 1663 1664 static int 1665 genfs_do_directio(struct vmspace *vs, vaddr_t uva, size_t len, struct vnode *vp, 1666 off_t off, enum uio_rw rw) 1667 { 1668 struct vm_map *map; 1669 struct pmap *upm, *kpm; 1670 size_t klen = round_page(uva + len) - trunc_page(uva); 1671 off_t spoff, epoff; 1672 vaddr_t kva, puva; 1673 paddr_t pa; 1674 vm_prot_t prot; 1675 int error, rv, poff, koff; 1676 const int pgoflags = PGO_CLEANIT | PGO_SYNCIO | PGO_JOURNALLOCKED | 1677 (rw == UIO_WRITE ? PGO_FREE : 0); 1678 1679 /* 1680 * For writes, verify that this range of the file already has fully 1681 * allocated backing store. If there are any holes, just punt and 1682 * make the caller take the buffered write path. 1683 */ 1684 1685 if (rw == UIO_WRITE) { 1686 daddr_t lbn, elbn, blkno; 1687 int bsize, bshift, run; 1688 1689 bshift = vp->v_mount->mnt_fs_bshift; 1690 bsize = 1 << bshift; 1691 lbn = off >> bshift; 1692 elbn = (off + len + bsize - 1) >> bshift; 1693 while (lbn < elbn) { 1694 error = VOP_BMAP(vp, lbn, NULL, &blkno, &run); 1695 if (error) { 1696 return error; 1697 } 1698 if (blkno == (daddr_t)-1) { 1699 return ENOSPC; 1700 } 1701 lbn += 1 + run; 1702 } 1703 } 1704 1705 /* 1706 * Flush any cached pages for parts of the file that we're about to 1707 * access. If we're writing, invalidate pages as well. 1708 */ 1709 1710 spoff = trunc_page(off); 1711 epoff = round_page(off + len); 1712 mutex_enter(&vp->v_interlock); 1713 error = VOP_PUTPAGES(vp, spoff, epoff, pgoflags); 1714 if (error) { 1715 return error; 1716 } 1717 1718 /* 1719 * Wire the user pages and remap them into kernel memory. 1720 */ 1721 1722 prot = rw == UIO_READ ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_READ; 1723 error = uvm_vslock(vs, (void *)uva, len, prot); 1724 if (error) { 1725 return error; 1726 } 1727 1728 map = &vs->vm_map; 1729 upm = vm_map_pmap(map); 1730 kpm = vm_map_pmap(kernel_map); 1731 kva = uvm_km_alloc(kernel_map, klen, 0, 1732 UVM_KMF_VAONLY | UVM_KMF_WAITVA); 1733 puva = trunc_page(uva); 1734 for (poff = 0; poff < klen; poff += PAGE_SIZE) { 1735 rv = pmap_extract(upm, puva + poff, &pa); 1736 KASSERT(rv); 1737 pmap_enter(kpm, kva + poff, pa, prot, prot | PMAP_WIRED); 1738 } 1739 pmap_update(kpm); 1740 1741 /* 1742 * Do the I/O. 1743 */ 1744 1745 koff = uva - trunc_page(uva); 1746 error = genfs_do_io(vp, off, kva + koff, len, PGO_SYNCIO, rw, 1747 genfs_dio_iodone); 1748 1749 /* 1750 * Tear down the kernel mapping. 1751 */ 1752 1753 pmap_remove(kpm, kva, kva + klen); 1754 pmap_update(kpm); 1755 uvm_km_free(kernel_map, kva, klen, UVM_KMF_VAONLY); 1756 1757 /* 1758 * Unwire the user pages. 1759 */ 1760 1761 uvm_vsunlock(vs, (void *)uva, len); 1762 return error; 1763 } 1764 1765