1 /* $NetBSD: genfs_io.c,v 1.36 2010/01/30 12:06:20 uebayasi Exp $ */ 2 3 /* 4 * Copyright (c) 1982, 1986, 1989, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the University nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 */ 32 33 #include <sys/cdefs.h> 34 __KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.36 2010/01/30 12:06:20 uebayasi Exp $"); 35 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/proc.h> 39 #include <sys/kernel.h> 40 #include <sys/mount.h> 41 #include <sys/namei.h> 42 #include <sys/vnode.h> 43 #include <sys/fcntl.h> 44 #include <sys/kmem.h> 45 #include <sys/poll.h> 46 #include <sys/mman.h> 47 #include <sys/file.h> 48 #include <sys/kauth.h> 49 #include <sys/fstrans.h> 50 #include <sys/buf.h> 51 52 #include <miscfs/genfs/genfs.h> 53 #include <miscfs/genfs/genfs_node.h> 54 #include <miscfs/specfs/specdev.h> 55 56 #include <uvm/uvm.h> 57 #include <uvm/uvm_pager.h> 58 59 static int genfs_do_directio(struct vmspace *, vaddr_t, size_t, struct vnode *, 60 off_t, enum uio_rw); 61 static void genfs_dio_iodone(struct buf *); 62 63 static int genfs_do_io(struct vnode *, off_t, vaddr_t, size_t, int, enum uio_rw, 64 void (*)(struct buf *)); 65 static inline void genfs_rel_pages(struct vm_page **, int); 66 67 int genfs_maxdio = MAXPHYS; 68 69 static inline void 70 genfs_rel_pages(struct vm_page **pgs, int npages) 71 { 72 int i; 73 74 for (i = 0; i < npages; i++) { 75 struct vm_page *pg = pgs[i]; 76 77 if (pg == NULL || pg == PGO_DONTCARE) 78 continue; 79 if (pg->flags & PG_FAKE) { 80 pg->flags |= PG_RELEASED; 81 } 82 } 83 mutex_enter(&uvm_pageqlock); 84 uvm_page_unbusy(pgs, npages); 85 mutex_exit(&uvm_pageqlock); 86 } 87 88 /* 89 * generic VM getpages routine. 90 * Return PG_BUSY pages for the given range, 91 * reading from backing store if necessary. 92 */ 93 94 int 95 genfs_getpages(void *v) 96 { 97 struct vop_getpages_args /* { 98 struct vnode *a_vp; 99 voff_t a_offset; 100 struct vm_page **a_m; 101 int *a_count; 102 int a_centeridx; 103 vm_prot_t a_access_type; 104 int a_advice; 105 int a_flags; 106 } */ * const ap = v; 107 108 off_t diskeof, memeof; 109 int i, error, npages; 110 const int flags = ap->a_flags; 111 struct vnode * const vp = ap->a_vp; 112 struct genfs_node * const gp = VTOG(vp); 113 struct uvm_object * const uobj = &vp->v_uobj; 114 kauth_cred_t const cred = curlwp->l_cred; /* XXXUBC curlwp */ 115 const bool async = (flags & PGO_SYNCIO) == 0; 116 const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0; 117 bool has_trans = false; 118 const bool overwrite = (flags & PGO_OVERWRITE) != 0; 119 const bool blockalloc = memwrite && (flags & PGO_NOBLOCKALLOC) == 0; 120 UVMHIST_FUNC("genfs_getpages"); UVMHIST_CALLED(ubchist); 121 122 UVMHIST_LOG(ubchist, "vp %p off 0x%x/%x count %d", 123 vp, ap->a_offset >> 32, ap->a_offset, *ap->a_count); 124 125 KASSERT(vp->v_type == VREG || vp->v_type == VDIR || 126 vp->v_type == VLNK || vp->v_type == VBLK); 127 128 startover: 129 error = 0; 130 const voff_t origvsize = vp->v_size; 131 const off_t origoffset = ap->a_offset; 132 const int orignpages = *ap->a_count; 133 134 GOP_SIZE(vp, origvsize, &diskeof, 0); 135 if (flags & PGO_PASTEOF) { 136 off_t newsize; 137 #if defined(DIAGNOSTIC) 138 off_t writeeof; 139 #endif /* defined(DIAGNOSTIC) */ 140 141 newsize = MAX(origvsize, 142 origoffset + (orignpages << PAGE_SHIFT)); 143 GOP_SIZE(vp, newsize, &memeof, GOP_SIZE_MEM); 144 #if defined(DIAGNOSTIC) 145 GOP_SIZE(vp, vp->v_writesize, &writeeof, GOP_SIZE_MEM); 146 if (newsize > round_page(writeeof)) { 147 panic("%s: past eof", __func__); 148 } 149 #endif /* defined(DIAGNOSTIC) */ 150 } else { 151 GOP_SIZE(vp, origvsize, &memeof, GOP_SIZE_MEM); 152 } 153 KASSERT(ap->a_centeridx >= 0 || ap->a_centeridx <= orignpages); 154 KASSERT((origoffset & (PAGE_SIZE - 1)) == 0 && origoffset >= 0); 155 KASSERT(orignpages > 0); 156 157 /* 158 * Bounds-check the request. 159 */ 160 161 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= memeof) { 162 if ((flags & PGO_LOCKED) == 0) { 163 mutex_exit(&uobj->vmobjlock); 164 } 165 UVMHIST_LOG(ubchist, "off 0x%x count %d goes past EOF 0x%x", 166 origoffset, *ap->a_count, memeof,0); 167 error = EINVAL; 168 goto out_err; 169 } 170 171 /* uobj is locked */ 172 173 if ((flags & PGO_NOTIMESTAMP) == 0 && 174 (vp->v_type != VBLK || 175 (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) { 176 int updflags = 0; 177 178 if ((vp->v_mount->mnt_flag & MNT_NOATIME) == 0) { 179 updflags = GOP_UPDATE_ACCESSED; 180 } 181 if (memwrite) { 182 updflags |= GOP_UPDATE_MODIFIED; 183 } 184 if (updflags != 0) { 185 GOP_MARKUPDATE(vp, updflags); 186 } 187 } 188 189 if (memwrite) { 190 gp->g_dirtygen++; 191 if ((vp->v_iflag & VI_ONWORKLST) == 0) { 192 vn_syncer_add_to_worklist(vp, filedelay); 193 } 194 if ((vp->v_iflag & (VI_WRMAP|VI_WRMAPDIRTY)) == VI_WRMAP) { 195 vp->v_iflag |= VI_WRMAPDIRTY; 196 } 197 } 198 199 /* 200 * For PGO_LOCKED requests, just return whatever's in memory. 201 */ 202 203 if (flags & PGO_LOCKED) { 204 int nfound; 205 struct vm_page *pg; 206 207 npages = *ap->a_count; 208 #if defined(DEBUG) 209 for (i = 0; i < npages; i++) { 210 pg = ap->a_m[i]; 211 KASSERT(pg == NULL || pg == PGO_DONTCARE); 212 } 213 #endif /* defined(DEBUG) */ 214 nfound = uvn_findpages(uobj, origoffset, &npages, 215 ap->a_m, UFP_NOWAIT|UFP_NOALLOC|(memwrite ? UFP_NORDONLY : 0)); 216 KASSERT(npages == *ap->a_count); 217 if (nfound == 0) { 218 error = EBUSY; 219 goto out_err; 220 } 221 if (!genfs_node_rdtrylock(vp)) { 222 genfs_rel_pages(ap->a_m, npages); 223 224 /* 225 * restore the array. 226 */ 227 228 for (i = 0; i < npages; i++) { 229 pg = ap->a_m[i]; 230 231 if (pg != NULL || pg != PGO_DONTCARE) { 232 ap->a_m[i] = NULL; 233 } 234 } 235 } else { 236 genfs_node_unlock(vp); 237 } 238 error = (ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0); 239 goto out_err; 240 } 241 mutex_exit(&uobj->vmobjlock); 242 243 /* 244 * find the requested pages and make some simple checks. 245 * leave space in the page array for a whole block. 246 */ 247 248 const int fs_bshift = (vp->v_type != VBLK) ? 249 vp->v_mount->mnt_fs_bshift : DEV_BSHIFT; 250 const int dev_bshift = (vp->v_type != VBLK) ? 251 vp->v_mount->mnt_dev_bshift : DEV_BSHIFT; 252 const int fs_bsize = 1 << fs_bshift; 253 #define blk_mask (fs_bsize - 1) 254 #define trunc_blk(x) ((x) & ~blk_mask) 255 #define round_blk(x) (((x) + blk_mask) & ~blk_mask) 256 257 const int orignmempages = MIN(orignpages, 258 round_page(memeof - origoffset) >> PAGE_SHIFT); 259 npages = orignmempages; 260 const off_t startoffset = trunc_blk(origoffset); 261 const off_t endoffset = MIN( 262 round_page(round_blk(origoffset + (npages << PAGE_SHIFT))), 263 round_page(memeof)); 264 const int ridx = (origoffset - startoffset) >> PAGE_SHIFT; 265 266 const int pgs_size = sizeof(struct vm_page *) * 267 ((endoffset - startoffset) >> PAGE_SHIFT); 268 struct vm_page **pgs, *pgs_onstack[UBC_MAX_PAGES]; 269 270 if (pgs_size > sizeof(pgs_onstack)) { 271 pgs = kmem_zalloc(pgs_size, async ? KM_NOSLEEP : KM_SLEEP); 272 if (pgs == NULL) { 273 pgs = pgs_onstack; 274 error = ENOMEM; 275 goto out_err; 276 } 277 } else { 278 pgs = pgs_onstack; 279 (void)memset(pgs, 0, pgs_size); 280 } 281 282 UVMHIST_LOG(ubchist, "ridx %d npages %d startoff %ld endoff %ld", 283 ridx, npages, startoffset, endoffset); 284 285 if (!has_trans) { 286 fstrans_start(vp->v_mount, FSTRANS_SHARED); 287 has_trans = true; 288 } 289 290 /* 291 * hold g_glock to prevent a race with truncate. 292 * 293 * check if our idea of v_size is still valid. 294 */ 295 296 if (blockalloc) { 297 rw_enter(&gp->g_glock, RW_WRITER); 298 } else { 299 rw_enter(&gp->g_glock, RW_READER); 300 } 301 mutex_enter(&uobj->vmobjlock); 302 if (vp->v_size < origvsize) { 303 genfs_node_unlock(vp); 304 if (pgs != pgs_onstack) 305 kmem_free(pgs, pgs_size); 306 goto startover; 307 } 308 309 if (uvn_findpages(uobj, origoffset, &npages, &pgs[ridx], 310 async ? UFP_NOWAIT : UFP_ALL) != orignmempages) { 311 genfs_node_unlock(vp); 312 KASSERT(async != 0); 313 genfs_rel_pages(&pgs[ridx], orignmempages); 314 mutex_exit(&uobj->vmobjlock); 315 error = EBUSY; 316 goto out_err_free; 317 } 318 319 /* 320 * if the pages are already resident, just return them. 321 */ 322 323 for (i = 0; i < npages; i++) { 324 struct vm_page *pg = pgs[ridx + i]; 325 326 if ((pg->flags & PG_FAKE) || 327 (blockalloc && (pg->flags & PG_RDONLY))) { 328 break; 329 } 330 } 331 if (i == npages) { 332 genfs_node_unlock(vp); 333 UVMHIST_LOG(ubchist, "returning cached pages", 0,0,0,0); 334 npages += ridx; 335 goto out; 336 } 337 338 /* 339 * if PGO_OVERWRITE is set, don't bother reading the pages. 340 */ 341 342 if (overwrite) { 343 genfs_node_unlock(vp); 344 UVMHIST_LOG(ubchist, "PGO_OVERWRITE",0,0,0,0); 345 346 for (i = 0; i < npages; i++) { 347 struct vm_page *pg = pgs[ridx + i]; 348 349 pg->flags &= ~(PG_RDONLY|PG_CLEAN); 350 } 351 npages += ridx; 352 goto out; 353 } 354 355 /* 356 * the page wasn't resident and we're not overwriting, 357 * so we're going to have to do some i/o. 358 * find any additional pages needed to cover the expanded range. 359 */ 360 361 npages = (endoffset - startoffset) >> PAGE_SHIFT; 362 if (startoffset != origoffset || npages != orignmempages) { 363 int npgs; 364 365 /* 366 * we need to avoid deadlocks caused by locking 367 * additional pages at lower offsets than pages we 368 * already have locked. unlock them all and start over. 369 */ 370 371 genfs_rel_pages(&pgs[ridx], orignmempages); 372 memset(pgs, 0, pgs_size); 373 374 UVMHIST_LOG(ubchist, "reset npages start 0x%x end 0x%x", 375 startoffset, endoffset, 0,0); 376 npgs = npages; 377 if (uvn_findpages(uobj, startoffset, &npgs, pgs, 378 async ? UFP_NOWAIT : UFP_ALL) != npages) { 379 genfs_node_unlock(vp); 380 KASSERT(async != 0); 381 genfs_rel_pages(pgs, npages); 382 mutex_exit(&uobj->vmobjlock); 383 error = EBUSY; 384 goto out_err_free; 385 } 386 } 387 388 mutex_exit(&uobj->vmobjlock); 389 390 { 391 size_t bytes, iobytes, tailstart, tailbytes, totalbytes, skipbytes; 392 vaddr_t kva; 393 struct buf *bp, *mbp; 394 bool sawhole = false; 395 396 /* 397 * read the desired page(s). 398 */ 399 400 totalbytes = npages << PAGE_SHIFT; 401 bytes = MIN(totalbytes, MAX(diskeof - startoffset, 0)); 402 tailbytes = totalbytes - bytes; 403 skipbytes = 0; 404 405 kva = uvm_pagermapin(pgs, npages, 406 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 407 408 mbp = getiobuf(vp, true); 409 mbp->b_bufsize = totalbytes; 410 mbp->b_data = (void *)kva; 411 mbp->b_resid = mbp->b_bcount = bytes; 412 mbp->b_cflags = BC_BUSY; 413 if (async) { 414 mbp->b_flags = B_READ | B_ASYNC; 415 mbp->b_iodone = uvm_aio_biodone; 416 } else { 417 mbp->b_flags = B_READ; 418 mbp->b_iodone = NULL; 419 } 420 if (async) 421 BIO_SETPRIO(mbp, BPRIO_TIMELIMITED); 422 else 423 BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL); 424 425 /* 426 * if EOF is in the middle of the range, zero the part past EOF. 427 * skip over pages which are not PG_FAKE since in that case they have 428 * valid data that we need to preserve. 429 */ 430 431 tailstart = bytes; 432 while (tailbytes > 0) { 433 const int len = PAGE_SIZE - (tailstart & PAGE_MASK); 434 435 KASSERT(len <= tailbytes); 436 if ((pgs[tailstart >> PAGE_SHIFT]->flags & PG_FAKE) != 0) { 437 memset((void *)(kva + tailstart), 0, len); 438 UVMHIST_LOG(ubchist, "tailbytes %p 0x%x 0x%x", 439 kva, tailstart, len, 0); 440 } 441 tailstart += len; 442 tailbytes -= len; 443 } 444 445 /* 446 * now loop over the pages, reading as needed. 447 */ 448 449 bp = NULL; 450 off_t offset; 451 for (offset = startoffset; 452 bytes > 0; 453 offset += iobytes, bytes -= iobytes) { 454 int run; 455 daddr_t lbn, blkno; 456 int pidx; 457 struct vnode *devvp; 458 459 /* 460 * skip pages which don't need to be read. 461 */ 462 463 pidx = (offset - startoffset) >> PAGE_SHIFT; 464 while ((pgs[pidx]->flags & PG_FAKE) == 0) { 465 size_t b; 466 467 KASSERT((offset & (PAGE_SIZE - 1)) == 0); 468 if ((pgs[pidx]->flags & PG_RDONLY)) { 469 sawhole = true; 470 } 471 b = MIN(PAGE_SIZE, bytes); 472 offset += b; 473 bytes -= b; 474 skipbytes += b; 475 pidx++; 476 UVMHIST_LOG(ubchist, "skipping, new offset 0x%x", 477 offset, 0,0,0); 478 if (bytes == 0) { 479 goto loopdone; 480 } 481 } 482 483 /* 484 * bmap the file to find out the blkno to read from and 485 * how much we can read in one i/o. if bmap returns an error, 486 * skip the rest of the top-level i/o. 487 */ 488 489 lbn = offset >> fs_bshift; 490 error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run); 491 if (error) { 492 UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%x -> %d\n", 493 lbn,error,0,0); 494 skipbytes += bytes; 495 bytes = 0; 496 goto loopdone; 497 } 498 499 /* 500 * see how many pages can be read with this i/o. 501 * reduce the i/o size if necessary to avoid 502 * overwriting pages with valid data. 503 */ 504 505 iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset, 506 bytes); 507 if (offset + iobytes > round_page(offset)) { 508 int pcount; 509 510 pcount = 1; 511 while (pidx + pcount < npages && 512 pgs[pidx + pcount]->flags & PG_FAKE) { 513 pcount++; 514 } 515 iobytes = MIN(iobytes, (pcount << PAGE_SHIFT) - 516 (offset - trunc_page(offset))); 517 } 518 519 /* 520 * if this block isn't allocated, zero it instead of 521 * reading it. unless we are going to allocate blocks, 522 * mark the pages we zeroed PG_RDONLY. 523 */ 524 525 if (blkno == (daddr_t)-1) { 526 int holepages = (round_page(offset + iobytes) - 527 trunc_page(offset)) >> PAGE_SHIFT; 528 UVMHIST_LOG(ubchist, "lbn 0x%x -> HOLE", lbn,0,0,0); 529 530 sawhole = true; 531 memset((char *)kva + (offset - startoffset), 0, 532 iobytes); 533 skipbytes += iobytes; 534 535 for (i = 0; i < holepages; i++) { 536 if (memwrite) { 537 pgs[pidx + i]->flags &= ~PG_CLEAN; 538 } 539 if (!blockalloc) { 540 pgs[pidx + i]->flags |= PG_RDONLY; 541 } 542 } 543 continue; 544 } 545 546 /* 547 * allocate a sub-buf for this piece of the i/o 548 * (or just use mbp if there's only 1 piece), 549 * and start it going. 550 */ 551 552 if (offset == startoffset && iobytes == bytes) { 553 bp = mbp; 554 } else { 555 UVMHIST_LOG(ubchist, "vp %p bp %p num now %d", 556 vp, bp, vp->v_numoutput, 0); 557 bp = getiobuf(vp, true); 558 nestiobuf_setup(mbp, bp, offset - startoffset, iobytes); 559 } 560 bp->b_lblkno = 0; 561 562 /* adjust physical blkno for partial blocks */ 563 bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >> 564 dev_bshift); 565 566 UVMHIST_LOG(ubchist, 567 "bp %p offset 0x%x bcount 0x%x blkno 0x%x", 568 bp, offset, bp->b_bcount, bp->b_blkno); 569 570 VOP_STRATEGY(devvp, bp); 571 } 572 573 loopdone: 574 nestiobuf_done(mbp, skipbytes, error); 575 if (async) { 576 UVMHIST_LOG(ubchist, "returning 0 (async)",0,0,0,0); 577 genfs_node_unlock(vp); 578 error = 0; 579 goto out_err_free; 580 } 581 if (bp != NULL) { 582 error = biowait(mbp); 583 } 584 585 /* Remove the mapping (make KVA available as soon as possible) */ 586 uvm_pagermapout(kva, npages); 587 588 /* 589 * if this we encountered a hole then we have to do a little more work. 590 * for read faults, we marked the page PG_RDONLY so that future 591 * write accesses to the page will fault again. 592 * for write faults, we must make sure that the backing store for 593 * the page is completely allocated while the pages are locked. 594 */ 595 596 if (!error && sawhole && blockalloc) { 597 /* 598 * XXX: This assumes that we come here only via 599 * the mmio path 600 */ 601 if (vp->v_mount->mnt_wapbl) { 602 error = WAPBL_BEGIN(vp->v_mount); 603 } 604 605 if (!error) { 606 error = GOP_ALLOC(vp, startoffset, 607 npages << PAGE_SHIFT, 0, cred); 608 if (vp->v_mount->mnt_wapbl) { 609 WAPBL_END(vp->v_mount); 610 } 611 } 612 613 UVMHIST_LOG(ubchist, "gop_alloc off 0x%x/0x%x -> %d", 614 startoffset, npages << PAGE_SHIFT, error,0); 615 if (!error) { 616 for (i = 0; i < npages; i++) { 617 struct vm_page *pg = pgs[i]; 618 619 if (pg == NULL) { 620 continue; 621 } 622 pg->flags &= ~(PG_CLEAN|PG_RDONLY); 623 UVMHIST_LOG(ubchist, "mark dirty pg %p", 624 pg,0,0,0); 625 } 626 } 627 } 628 genfs_node_unlock(vp); 629 630 putiobuf(mbp); 631 } 632 633 mutex_enter(&uobj->vmobjlock); 634 635 /* 636 * we're almost done! release the pages... 637 * for errors, we free the pages. 638 * otherwise we activate them and mark them as valid and clean. 639 * also, unbusy pages that were not actually requested. 640 */ 641 642 if (error) { 643 for (i = 0; i < npages; i++) { 644 struct vm_page *pg = pgs[i]; 645 646 if (pg == NULL) { 647 continue; 648 } 649 UVMHIST_LOG(ubchist, "examining pg %p flags 0x%x", 650 pg, pg->flags, 0,0); 651 if (pg->flags & PG_FAKE) { 652 pg->flags |= PG_RELEASED; 653 } 654 } 655 mutex_enter(&uvm_pageqlock); 656 uvm_page_unbusy(pgs, npages); 657 mutex_exit(&uvm_pageqlock); 658 mutex_exit(&uobj->vmobjlock); 659 UVMHIST_LOG(ubchist, "returning error %d", error,0,0,0); 660 goto out_err_free; 661 } 662 663 out: 664 UVMHIST_LOG(ubchist, "succeeding, npages %d", npages,0,0,0); 665 error = 0; 666 mutex_enter(&uvm_pageqlock); 667 for (i = 0; i < npages; i++) { 668 struct vm_page *pg = pgs[i]; 669 if (pg == NULL) { 670 continue; 671 } 672 UVMHIST_LOG(ubchist, "examining pg %p flags 0x%x", 673 pg, pg->flags, 0,0); 674 if (pg->flags & PG_FAKE && !overwrite) { 675 pg->flags &= ~(PG_FAKE); 676 pmap_clear_modify(pgs[i]); 677 } 678 KASSERT(!memwrite || !blockalloc || (pg->flags & PG_RDONLY) == 0); 679 if (i < ridx || i >= ridx + orignmempages || async) { 680 UVMHIST_LOG(ubchist, "unbusy pg %p offset 0x%x", 681 pg, pg->offset,0,0); 682 if (pg->flags & PG_WANTED) { 683 wakeup(pg); 684 } 685 if (pg->flags & PG_FAKE) { 686 KASSERT(overwrite); 687 uvm_pagezero(pg); 688 } 689 if (pg->flags & PG_RELEASED) { 690 uvm_pagefree(pg); 691 continue; 692 } 693 uvm_pageenqueue(pg); 694 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE); 695 UVM_PAGE_OWN(pg, NULL); 696 } 697 } 698 mutex_exit(&uvm_pageqlock); 699 mutex_exit(&uobj->vmobjlock); 700 if (ap->a_m != NULL) { 701 memcpy(ap->a_m, &pgs[ridx], 702 orignmempages * sizeof(struct vm_page *)); 703 } 704 705 out_err_free: 706 if (pgs != NULL && pgs != pgs_onstack) 707 kmem_free(pgs, pgs_size); 708 out_err: 709 if (has_trans) 710 fstrans_done(vp->v_mount); 711 return (error); 712 } 713 714 /* 715 * generic VM putpages routine. 716 * Write the given range of pages to backing store. 717 * 718 * => "offhi == 0" means flush all pages at or after "offlo". 719 * => object should be locked by caller. we return with the 720 * object unlocked. 721 * => if PGO_CLEANIT or PGO_SYNCIO is set, we may block (due to I/O). 722 * thus, a caller might want to unlock higher level resources 723 * (e.g. vm_map) before calling flush. 724 * => if neither PGO_CLEANIT nor PGO_SYNCIO is set, we will not block 725 * => if PGO_ALLPAGES is set, then all pages in the object will be processed. 726 * => NOTE: we rely on the fact that the object's memq is a TAILQ and 727 * that new pages are inserted on the tail end of the list. thus, 728 * we can make a complete pass through the object in one go by starting 729 * at the head and working towards the tail (new pages are put in 730 * front of us). 731 * => NOTE: we are allowed to lock the page queues, so the caller 732 * must not be holding the page queue lock. 733 * 734 * note on "cleaning" object and PG_BUSY pages: 735 * this routine is holding the lock on the object. the only time 736 * that it can run into a PG_BUSY page that it does not own is if 737 * some other process has started I/O on the page (e.g. either 738 * a pagein, or a pageout). if the PG_BUSY page is being paged 739 * in, then it can not be dirty (!PG_CLEAN) because no one has 740 * had a chance to modify it yet. if the PG_BUSY page is being 741 * paged out then it means that someone else has already started 742 * cleaning the page for us (how nice!). in this case, if we 743 * have syncio specified, then after we make our pass through the 744 * object we need to wait for the other PG_BUSY pages to clear 745 * off (i.e. we need to do an iosync). also note that once a 746 * page is PG_BUSY it must stay in its object until it is un-busyed. 747 * 748 * note on page traversal: 749 * we can traverse the pages in an object either by going down the 750 * linked list in "uobj->memq", or we can go over the address range 751 * by page doing hash table lookups for each address. depending 752 * on how many pages are in the object it may be cheaper to do one 753 * or the other. we set "by_list" to true if we are using memq. 754 * if the cost of a hash lookup was equal to the cost of the list 755 * traversal we could compare the number of pages in the start->stop 756 * range to the total number of pages in the object. however, it 757 * seems that a hash table lookup is more expensive than the linked 758 * list traversal, so we multiply the number of pages in the 759 * range by an estimate of the relatively higher cost of the hash lookup. 760 */ 761 762 int 763 genfs_putpages(void *v) 764 { 765 struct vop_putpages_args /* { 766 struct vnode *a_vp; 767 voff_t a_offlo; 768 voff_t a_offhi; 769 int a_flags; 770 } */ * const ap = v; 771 772 return genfs_do_putpages(ap->a_vp, ap->a_offlo, ap->a_offhi, 773 ap->a_flags, NULL); 774 } 775 776 int 777 genfs_do_putpages(struct vnode *vp, off_t startoff, off_t endoff, 778 int origflags, struct vm_page **busypg) 779 { 780 struct uvm_object * const uobj = &vp->v_uobj; 781 kmutex_t * const slock = &uobj->vmobjlock; 782 off_t off; 783 /* Even for strange MAXPHYS, the shift rounds down to a page */ 784 #define maxpages (MAXPHYS >> PAGE_SHIFT) 785 int i, error, npages, nback; 786 int freeflag; 787 struct vm_page *pgs[maxpages], *pg, *nextpg, *tpg, curmp, endmp; 788 bool wasclean, by_list, needs_clean, yld; 789 bool async = (origflags & PGO_SYNCIO) == 0; 790 bool pagedaemon = curlwp == uvm.pagedaemon_lwp; 791 struct lwp * const l = curlwp ? curlwp : &lwp0; 792 struct genfs_node * const gp = VTOG(vp); 793 int flags; 794 int dirtygen; 795 bool modified; 796 bool need_wapbl; 797 bool has_trans; 798 bool cleanall; 799 bool onworklst; 800 801 UVMHIST_FUNC("genfs_putpages"); UVMHIST_CALLED(ubchist); 802 803 KASSERT(origflags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)); 804 KASSERT((startoff & PAGE_MASK) == 0 && (endoff & PAGE_MASK) == 0); 805 KASSERT(startoff < endoff || endoff == 0); 806 807 UVMHIST_LOG(ubchist, "vp %p pages %d off 0x%x len 0x%x", 808 vp, uobj->uo_npages, startoff, endoff - startoff); 809 810 has_trans = false; 811 need_wapbl = (!pagedaemon && vp->v_mount && vp->v_mount->mnt_wapbl && 812 (origflags & PGO_JOURNALLOCKED) == 0); 813 814 retry: 815 modified = false; 816 flags = origflags; 817 KASSERT((vp->v_iflag & VI_ONWORKLST) != 0 || 818 (vp->v_iflag & VI_WRMAPDIRTY) == 0); 819 if (uobj->uo_npages == 0) { 820 if (vp->v_iflag & VI_ONWORKLST) { 821 vp->v_iflag &= ~VI_WRMAPDIRTY; 822 if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL) 823 vn_syncer_remove_from_worklist(vp); 824 } 825 if (has_trans) { 826 if (need_wapbl) 827 WAPBL_END(vp->v_mount); 828 fstrans_done(vp->v_mount); 829 } 830 mutex_exit(slock); 831 return (0); 832 } 833 834 /* 835 * the vnode has pages, set up to process the request. 836 */ 837 838 if (!has_trans && (flags & PGO_CLEANIT) != 0) { 839 mutex_exit(slock); 840 if (pagedaemon) { 841 error = fstrans_start_nowait(vp->v_mount, FSTRANS_LAZY); 842 if (error) 843 return error; 844 } else 845 fstrans_start(vp->v_mount, FSTRANS_LAZY); 846 if (need_wapbl) { 847 error = WAPBL_BEGIN(vp->v_mount); 848 if (error) { 849 fstrans_done(vp->v_mount); 850 return error; 851 } 852 } 853 has_trans = true; 854 mutex_enter(slock); 855 goto retry; 856 } 857 858 error = 0; 859 wasclean = (vp->v_numoutput == 0); 860 off = startoff; 861 if (endoff == 0 || flags & PGO_ALLPAGES) { 862 endoff = trunc_page(LLONG_MAX); 863 } 864 by_list = (uobj->uo_npages <= 865 ((endoff - startoff) >> PAGE_SHIFT) * UVM_PAGE_TREE_PENALTY); 866 867 #if !defined(DEBUG) 868 /* 869 * if this vnode is known not to have dirty pages, 870 * don't bother to clean it out. 871 */ 872 873 if ((vp->v_iflag & VI_ONWORKLST) == 0) { 874 if ((flags & (PGO_FREE|PGO_DEACTIVATE)) == 0) { 875 goto skip_scan; 876 } 877 flags &= ~PGO_CLEANIT; 878 } 879 #endif /* !defined(DEBUG) */ 880 881 /* 882 * start the loop. when scanning by list, hold the last page 883 * in the list before we start. pages allocated after we start 884 * will be added to the end of the list, so we can stop at the 885 * current last page. 886 */ 887 888 cleanall = (flags & PGO_CLEANIT) != 0 && wasclean && 889 startoff == 0 && endoff == trunc_page(LLONG_MAX) && 890 (vp->v_iflag & VI_ONWORKLST) != 0; 891 dirtygen = gp->g_dirtygen; 892 freeflag = pagedaemon ? PG_PAGEOUT : PG_RELEASED; 893 if (by_list) { 894 curmp.uobject = uobj; 895 curmp.offset = (voff_t)-1; 896 curmp.flags = PG_BUSY; 897 endmp.uobject = uobj; 898 endmp.offset = (voff_t)-1; 899 endmp.flags = PG_BUSY; 900 pg = TAILQ_FIRST(&uobj->memq); 901 TAILQ_INSERT_TAIL(&uobj->memq, &endmp, listq.queue); 902 } else { 903 pg = uvm_pagelookup(uobj, off); 904 } 905 nextpg = NULL; 906 while (by_list || off < endoff) { 907 908 /* 909 * if the current page is not interesting, move on to the next. 910 */ 911 912 KASSERT(pg == NULL || pg->uobject == uobj); 913 KASSERT(pg == NULL || 914 (pg->flags & (PG_RELEASED|PG_PAGEOUT)) == 0 || 915 (pg->flags & PG_BUSY) != 0); 916 if (by_list) { 917 if (pg == &endmp) { 918 break; 919 } 920 if (pg->offset < startoff || pg->offset >= endoff || 921 pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 922 if (pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 923 wasclean = false; 924 } 925 pg = TAILQ_NEXT(pg, listq.queue); 926 continue; 927 } 928 off = pg->offset; 929 } else if (pg == NULL || pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 930 if (pg != NULL) { 931 wasclean = false; 932 } 933 off += PAGE_SIZE; 934 if (off < endoff) { 935 pg = uvm_pagelookup(uobj, off); 936 } 937 continue; 938 } 939 940 /* 941 * if the current page needs to be cleaned and it's busy, 942 * wait for it to become unbusy. 943 */ 944 945 yld = (l->l_cpu->ci_schedstate.spc_flags & 946 SPCF_SHOULDYIELD) && !pagedaemon; 947 if (pg->flags & PG_BUSY || yld) { 948 UVMHIST_LOG(ubchist, "busy %p", pg,0,0,0); 949 if (flags & PGO_BUSYFAIL && pg->flags & PG_BUSY) { 950 UVMHIST_LOG(ubchist, "busyfail %p", pg, 0,0,0); 951 error = EDEADLK; 952 if (busypg != NULL) 953 *busypg = pg; 954 break; 955 } 956 if (pagedaemon) { 957 /* 958 * someone has taken the page while we 959 * dropped the lock for fstrans_start. 960 */ 961 break; 962 } 963 if (by_list) { 964 TAILQ_INSERT_BEFORE(pg, &curmp, listq.queue); 965 UVMHIST_LOG(ubchist, "curmp next %p", 966 TAILQ_NEXT(&curmp, listq.queue), 0,0,0); 967 } 968 if (yld) { 969 mutex_exit(slock); 970 preempt(); 971 mutex_enter(slock); 972 } else { 973 pg->flags |= PG_WANTED; 974 UVM_UNLOCK_AND_WAIT(pg, slock, 0, "genput", 0); 975 mutex_enter(slock); 976 } 977 if (by_list) { 978 UVMHIST_LOG(ubchist, "after next %p", 979 TAILQ_NEXT(&curmp, listq.queue), 0,0,0); 980 pg = TAILQ_NEXT(&curmp, listq.queue); 981 TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue); 982 } else { 983 pg = uvm_pagelookup(uobj, off); 984 } 985 continue; 986 } 987 988 /* 989 * if we're freeing, remove all mappings of the page now. 990 * if we're cleaning, check if the page is needs to be cleaned. 991 */ 992 993 if (flags & PGO_FREE) { 994 pmap_page_protect(pg, VM_PROT_NONE); 995 } else if (flags & PGO_CLEANIT) { 996 997 /* 998 * if we still have some hope to pull this vnode off 999 * from the syncer queue, write-protect the page. 1000 */ 1001 1002 if (cleanall && wasclean && 1003 gp->g_dirtygen == dirtygen) { 1004 1005 /* 1006 * uobj pages get wired only by uvm_fault 1007 * where uobj is locked. 1008 */ 1009 1010 if (pg->wire_count == 0) { 1011 pmap_page_protect(pg, 1012 VM_PROT_READ|VM_PROT_EXECUTE); 1013 } else { 1014 cleanall = false; 1015 } 1016 } 1017 } 1018 1019 if (flags & PGO_CLEANIT) { 1020 needs_clean = pmap_clear_modify(pg) || 1021 (pg->flags & PG_CLEAN) == 0; 1022 pg->flags |= PG_CLEAN; 1023 } else { 1024 needs_clean = false; 1025 } 1026 1027 /* 1028 * if we're cleaning, build a cluster. 1029 * the cluster will consist of pages which are currently dirty, 1030 * but they will be returned to us marked clean. 1031 * if not cleaning, just operate on the one page. 1032 */ 1033 1034 if (needs_clean) { 1035 KDASSERT((vp->v_iflag & VI_ONWORKLST)); 1036 wasclean = false; 1037 memset(pgs, 0, sizeof(pgs)); 1038 pg->flags |= PG_BUSY; 1039 UVM_PAGE_OWN(pg, "genfs_putpages"); 1040 1041 /* 1042 * first look backward. 1043 */ 1044 1045 npages = MIN(maxpages >> 1, off >> PAGE_SHIFT); 1046 nback = npages; 1047 uvn_findpages(uobj, off - PAGE_SIZE, &nback, &pgs[0], 1048 UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY|UFP_BACKWARD); 1049 if (nback) { 1050 memmove(&pgs[0], &pgs[npages - nback], 1051 nback * sizeof(pgs[0])); 1052 if (npages - nback < nback) 1053 memset(&pgs[nback], 0, 1054 (npages - nback) * sizeof(pgs[0])); 1055 else 1056 memset(&pgs[npages - nback], 0, 1057 nback * sizeof(pgs[0])); 1058 } 1059 1060 /* 1061 * then plug in our page of interest. 1062 */ 1063 1064 pgs[nback] = pg; 1065 1066 /* 1067 * then look forward to fill in the remaining space in 1068 * the array of pages. 1069 */ 1070 1071 npages = maxpages - nback - 1; 1072 uvn_findpages(uobj, off + PAGE_SIZE, &npages, 1073 &pgs[nback + 1], 1074 UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY); 1075 npages += nback + 1; 1076 } else { 1077 pgs[0] = pg; 1078 npages = 1; 1079 nback = 0; 1080 } 1081 1082 /* 1083 * apply FREE or DEACTIVATE options if requested. 1084 */ 1085 1086 if (flags & (PGO_DEACTIVATE|PGO_FREE)) { 1087 mutex_enter(&uvm_pageqlock); 1088 } 1089 for (i = 0; i < npages; i++) { 1090 tpg = pgs[i]; 1091 KASSERT(tpg->uobject == uobj); 1092 if (by_list && tpg == TAILQ_NEXT(pg, listq.queue)) 1093 pg = tpg; 1094 if (tpg->offset < startoff || tpg->offset >= endoff) 1095 continue; 1096 if (flags & PGO_DEACTIVATE && tpg->wire_count == 0) { 1097 uvm_pagedeactivate(tpg); 1098 } else if (flags & PGO_FREE) { 1099 pmap_page_protect(tpg, VM_PROT_NONE); 1100 if (tpg->flags & PG_BUSY) { 1101 tpg->flags |= freeflag; 1102 if (pagedaemon) { 1103 uvm_pageout_start(1); 1104 uvm_pagedequeue(tpg); 1105 } 1106 } else { 1107 1108 /* 1109 * ``page is not busy'' 1110 * implies that npages is 1 1111 * and needs_clean is false. 1112 */ 1113 1114 nextpg = TAILQ_NEXT(tpg, listq.queue); 1115 uvm_pagefree(tpg); 1116 if (pagedaemon) 1117 uvmexp.pdfreed++; 1118 } 1119 } 1120 } 1121 if (flags & (PGO_DEACTIVATE|PGO_FREE)) { 1122 mutex_exit(&uvm_pageqlock); 1123 } 1124 if (needs_clean) { 1125 modified = true; 1126 1127 /* 1128 * start the i/o. if we're traversing by list, 1129 * keep our place in the list with a marker page. 1130 */ 1131 1132 if (by_list) { 1133 TAILQ_INSERT_AFTER(&uobj->memq, pg, &curmp, 1134 listq.queue); 1135 } 1136 mutex_exit(slock); 1137 error = GOP_WRITE(vp, pgs, npages, flags); 1138 mutex_enter(slock); 1139 if (by_list) { 1140 pg = TAILQ_NEXT(&curmp, listq.queue); 1141 TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue); 1142 } 1143 if (error) { 1144 break; 1145 } 1146 if (by_list) { 1147 continue; 1148 } 1149 } 1150 1151 /* 1152 * find the next page and continue if there was no error. 1153 */ 1154 1155 if (by_list) { 1156 if (nextpg) { 1157 pg = nextpg; 1158 nextpg = NULL; 1159 } else { 1160 pg = TAILQ_NEXT(pg, listq.queue); 1161 } 1162 } else { 1163 off += (npages - nback) << PAGE_SHIFT; 1164 if (off < endoff) { 1165 pg = uvm_pagelookup(uobj, off); 1166 } 1167 } 1168 } 1169 if (by_list) { 1170 TAILQ_REMOVE(&uobj->memq, &endmp, listq.queue); 1171 } 1172 1173 if (modified && (vp->v_iflag & VI_WRMAPDIRTY) != 0 && 1174 (vp->v_type != VBLK || 1175 (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) { 1176 GOP_MARKUPDATE(vp, GOP_UPDATE_MODIFIED); 1177 } 1178 1179 /* 1180 * if we're cleaning and there was nothing to clean, 1181 * take us off the syncer list. if we started any i/o 1182 * and we're doing sync i/o, wait for all writes to finish. 1183 */ 1184 1185 if (cleanall && wasclean && gp->g_dirtygen == dirtygen && 1186 (vp->v_iflag & VI_ONWORKLST) != 0) { 1187 #if defined(DEBUG) 1188 TAILQ_FOREACH(pg, &uobj->memq, listq.queue) { 1189 if ((pg->flags & PG_CLEAN) == 0) { 1190 printf("%s: %p: !CLEAN\n", __func__, pg); 1191 } 1192 if (pmap_is_modified(pg)) { 1193 printf("%s: %p: modified\n", __func__, pg); 1194 } 1195 } 1196 #endif /* defined(DEBUG) */ 1197 vp->v_iflag &= ~VI_WRMAPDIRTY; 1198 if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL) 1199 vn_syncer_remove_from_worklist(vp); 1200 } 1201 1202 #if !defined(DEBUG) 1203 skip_scan: 1204 #endif /* !defined(DEBUG) */ 1205 1206 /* Wait for output to complete. */ 1207 if (!wasclean && !async && vp->v_numoutput != 0) { 1208 while (vp->v_numoutput != 0) 1209 cv_wait(&vp->v_cv, slock); 1210 } 1211 onworklst = (vp->v_iflag & VI_ONWORKLST) != 0; 1212 mutex_exit(slock); 1213 1214 if ((flags & PGO_RECLAIM) != 0 && onworklst) { 1215 /* 1216 * in the case of PGO_RECLAIM, ensure to make the vnode clean. 1217 * retrying is not a big deal because, in many cases, 1218 * uobj->uo_npages is already 0 here. 1219 */ 1220 mutex_enter(slock); 1221 goto retry; 1222 } 1223 1224 if (has_trans) { 1225 if (need_wapbl) 1226 WAPBL_END(vp->v_mount); 1227 fstrans_done(vp->v_mount); 1228 } 1229 1230 return (error); 1231 } 1232 1233 int 1234 genfs_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, int flags) 1235 { 1236 off_t off; 1237 vaddr_t kva; 1238 size_t len; 1239 int error; 1240 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1241 1242 UVMHIST_LOG(ubchist, "vp %p pgs %p npages %d flags 0x%x", 1243 vp, pgs, npages, flags); 1244 1245 off = pgs[0]->offset; 1246 kva = uvm_pagermapin(pgs, npages, 1247 UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK); 1248 len = npages << PAGE_SHIFT; 1249 1250 error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE, 1251 uvm_aio_biodone); 1252 1253 return error; 1254 } 1255 1256 int 1257 genfs_gop_write_rwmap(struct vnode *vp, struct vm_page **pgs, int npages, int flags) 1258 { 1259 off_t off; 1260 vaddr_t kva; 1261 size_t len; 1262 int error; 1263 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1264 1265 UVMHIST_LOG(ubchist, "vp %p pgs %p npages %d flags 0x%x", 1266 vp, pgs, npages, flags); 1267 1268 off = pgs[0]->offset; 1269 kva = uvm_pagermapin(pgs, npages, 1270 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 1271 len = npages << PAGE_SHIFT; 1272 1273 error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE, 1274 uvm_aio_biodone); 1275 1276 return error; 1277 } 1278 1279 /* 1280 * Backend routine for doing I/O to vnode pages. Pages are already locked 1281 * and mapped into kernel memory. Here we just look up the underlying 1282 * device block addresses and call the strategy routine. 1283 */ 1284 1285 static int 1286 genfs_do_io(struct vnode *vp, off_t off, vaddr_t kva, size_t len, int flags, 1287 enum uio_rw rw, void (*iodone)(struct buf *)) 1288 { 1289 int s, error; 1290 int fs_bshift, dev_bshift; 1291 off_t eof, offset, startoffset; 1292 size_t bytes, iobytes, skipbytes; 1293 struct buf *mbp, *bp; 1294 const bool async = (flags & PGO_SYNCIO) == 0; 1295 const bool iowrite = rw == UIO_WRITE; 1296 const int brw = iowrite ? B_WRITE : B_READ; 1297 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1298 1299 UVMHIST_LOG(ubchist, "vp %p kva %p len 0x%x flags 0x%x", 1300 vp, kva, len, flags); 1301 1302 KASSERT(vp->v_size <= vp->v_writesize); 1303 GOP_SIZE(vp, vp->v_writesize, &eof, 0); 1304 if (vp->v_type != VBLK) { 1305 fs_bshift = vp->v_mount->mnt_fs_bshift; 1306 dev_bshift = vp->v_mount->mnt_dev_bshift; 1307 } else { 1308 fs_bshift = DEV_BSHIFT; 1309 dev_bshift = DEV_BSHIFT; 1310 } 1311 error = 0; 1312 startoffset = off; 1313 bytes = MIN(len, eof - startoffset); 1314 skipbytes = 0; 1315 KASSERT(bytes != 0); 1316 1317 if (iowrite) { 1318 mutex_enter(&vp->v_interlock); 1319 vp->v_numoutput += 2; 1320 mutex_exit(&vp->v_interlock); 1321 } 1322 mbp = getiobuf(vp, true); 1323 UVMHIST_LOG(ubchist, "vp %p mbp %p num now %d bytes 0x%x", 1324 vp, mbp, vp->v_numoutput, bytes); 1325 mbp->b_bufsize = len; 1326 mbp->b_data = (void *)kva; 1327 mbp->b_resid = mbp->b_bcount = bytes; 1328 mbp->b_cflags = BC_BUSY | BC_AGE; 1329 if (async) { 1330 mbp->b_flags = brw | B_ASYNC; 1331 mbp->b_iodone = iodone; 1332 } else { 1333 mbp->b_flags = brw; 1334 mbp->b_iodone = NULL; 1335 } 1336 if (curlwp == uvm.pagedaemon_lwp) 1337 BIO_SETPRIO(mbp, BPRIO_TIMELIMITED); 1338 else if (async) 1339 BIO_SETPRIO(mbp, BPRIO_TIMENONCRITICAL); 1340 else 1341 BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL); 1342 1343 bp = NULL; 1344 for (offset = startoffset; 1345 bytes > 0; 1346 offset += iobytes, bytes -= iobytes) { 1347 int run; 1348 daddr_t lbn, blkno; 1349 struct vnode *devvp; 1350 1351 /* 1352 * bmap the file to find out the blkno to read from and 1353 * how much we can read in one i/o. if bmap returns an error, 1354 * skip the rest of the top-level i/o. 1355 */ 1356 1357 lbn = offset >> fs_bshift; 1358 error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run); 1359 if (error) { 1360 UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%x -> %d\n", 1361 lbn,error,0,0); 1362 skipbytes += bytes; 1363 bytes = 0; 1364 goto loopdone; 1365 } 1366 1367 /* 1368 * see how many pages can be read with this i/o. 1369 * reduce the i/o size if necessary to avoid 1370 * overwriting pages with valid data. 1371 */ 1372 1373 iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset, 1374 bytes); 1375 1376 /* 1377 * if this block isn't allocated, zero it instead of 1378 * reading it. unless we are going to allocate blocks, 1379 * mark the pages we zeroed PG_RDONLY. 1380 */ 1381 1382 if (blkno == (daddr_t)-1) { 1383 if (!iowrite) { 1384 memset((char *)kva + (offset - startoffset), 0, 1385 iobytes); 1386 } 1387 skipbytes += iobytes; 1388 continue; 1389 } 1390 1391 /* 1392 * allocate a sub-buf for this piece of the i/o 1393 * (or just use mbp if there's only 1 piece), 1394 * and start it going. 1395 */ 1396 1397 if (offset == startoffset && iobytes == bytes) { 1398 bp = mbp; 1399 } else { 1400 UVMHIST_LOG(ubchist, "vp %p bp %p num now %d", 1401 vp, bp, vp->v_numoutput, 0); 1402 bp = getiobuf(vp, true); 1403 nestiobuf_setup(mbp, bp, offset - startoffset, iobytes); 1404 } 1405 bp->b_lblkno = 0; 1406 1407 /* adjust physical blkno for partial blocks */ 1408 bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >> 1409 dev_bshift); 1410 1411 UVMHIST_LOG(ubchist, 1412 "bp %p offset 0x%x bcount 0x%x blkno 0x%x", 1413 bp, offset, bp->b_bcount, bp->b_blkno); 1414 1415 VOP_STRATEGY(devvp, bp); 1416 } 1417 1418 loopdone: 1419 if (skipbytes) { 1420 UVMHIST_LOG(ubchist, "skipbytes %d", skipbytes, 0,0,0); 1421 } 1422 nestiobuf_done(mbp, skipbytes, error); 1423 if (async) { 1424 UVMHIST_LOG(ubchist, "returning 0 (async)", 0,0,0,0); 1425 return (0); 1426 } 1427 UVMHIST_LOG(ubchist, "waiting for mbp %p", mbp,0,0,0); 1428 error = biowait(mbp); 1429 s = splbio(); 1430 (*iodone)(mbp); 1431 splx(s); 1432 UVMHIST_LOG(ubchist, "returning, error %d", error,0,0,0); 1433 return (error); 1434 } 1435 1436 int 1437 genfs_compat_getpages(void *v) 1438 { 1439 struct vop_getpages_args /* { 1440 struct vnode *a_vp; 1441 voff_t a_offset; 1442 struct vm_page **a_m; 1443 int *a_count; 1444 int a_centeridx; 1445 vm_prot_t a_access_type; 1446 int a_advice; 1447 int a_flags; 1448 } */ *ap = v; 1449 1450 off_t origoffset; 1451 struct vnode *vp = ap->a_vp; 1452 struct uvm_object *uobj = &vp->v_uobj; 1453 struct vm_page *pg, **pgs; 1454 vaddr_t kva; 1455 int i, error, orignpages, npages; 1456 struct iovec iov; 1457 struct uio uio; 1458 kauth_cred_t cred = curlwp->l_cred; 1459 const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0; 1460 1461 error = 0; 1462 origoffset = ap->a_offset; 1463 orignpages = *ap->a_count; 1464 pgs = ap->a_m; 1465 1466 if (memwrite && (vp->v_iflag & VI_ONWORKLST) == 0) { 1467 vn_syncer_add_to_worklist(vp, filedelay); 1468 } 1469 if (ap->a_flags & PGO_LOCKED) { 1470 uvn_findpages(uobj, origoffset, ap->a_count, ap->a_m, 1471 UFP_NOWAIT|UFP_NOALLOC| (memwrite ? UFP_NORDONLY : 0)); 1472 1473 return (ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0); 1474 } 1475 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= vp->v_size) { 1476 mutex_exit(&uobj->vmobjlock); 1477 return (EINVAL); 1478 } 1479 if ((ap->a_flags & PGO_SYNCIO) == 0) { 1480 mutex_exit(&uobj->vmobjlock); 1481 return 0; 1482 } 1483 npages = orignpages; 1484 uvn_findpages(uobj, origoffset, &npages, pgs, UFP_ALL); 1485 mutex_exit(&uobj->vmobjlock); 1486 kva = uvm_pagermapin(pgs, npages, 1487 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 1488 for (i = 0; i < npages; i++) { 1489 pg = pgs[i]; 1490 if ((pg->flags & PG_FAKE) == 0) { 1491 continue; 1492 } 1493 iov.iov_base = (char *)kva + (i << PAGE_SHIFT); 1494 iov.iov_len = PAGE_SIZE; 1495 uio.uio_iov = &iov; 1496 uio.uio_iovcnt = 1; 1497 uio.uio_offset = origoffset + (i << PAGE_SHIFT); 1498 uio.uio_rw = UIO_READ; 1499 uio.uio_resid = PAGE_SIZE; 1500 UIO_SETUP_SYSSPACE(&uio); 1501 /* XXX vn_lock */ 1502 error = VOP_READ(vp, &uio, 0, cred); 1503 if (error) { 1504 break; 1505 } 1506 if (uio.uio_resid) { 1507 memset(iov.iov_base, 0, uio.uio_resid); 1508 } 1509 } 1510 uvm_pagermapout(kva, npages); 1511 mutex_enter(&uobj->vmobjlock); 1512 mutex_enter(&uvm_pageqlock); 1513 for (i = 0; i < npages; i++) { 1514 pg = pgs[i]; 1515 if (error && (pg->flags & PG_FAKE) != 0) { 1516 pg->flags |= PG_RELEASED; 1517 } else { 1518 pmap_clear_modify(pg); 1519 uvm_pageactivate(pg); 1520 } 1521 } 1522 if (error) { 1523 uvm_page_unbusy(pgs, npages); 1524 } 1525 mutex_exit(&uvm_pageqlock); 1526 mutex_exit(&uobj->vmobjlock); 1527 return (error); 1528 } 1529 1530 int 1531 genfs_compat_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, 1532 int flags) 1533 { 1534 off_t offset; 1535 struct iovec iov; 1536 struct uio uio; 1537 kauth_cred_t cred = curlwp->l_cred; 1538 struct buf *bp; 1539 vaddr_t kva; 1540 int error; 1541 1542 offset = pgs[0]->offset; 1543 kva = uvm_pagermapin(pgs, npages, 1544 UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK); 1545 1546 iov.iov_base = (void *)kva; 1547 iov.iov_len = npages << PAGE_SHIFT; 1548 uio.uio_iov = &iov; 1549 uio.uio_iovcnt = 1; 1550 uio.uio_offset = offset; 1551 uio.uio_rw = UIO_WRITE; 1552 uio.uio_resid = npages << PAGE_SHIFT; 1553 UIO_SETUP_SYSSPACE(&uio); 1554 /* XXX vn_lock */ 1555 error = VOP_WRITE(vp, &uio, 0, cred); 1556 1557 mutex_enter(&vp->v_interlock); 1558 vp->v_numoutput++; 1559 mutex_exit(&vp->v_interlock); 1560 1561 bp = getiobuf(vp, true); 1562 bp->b_cflags = BC_BUSY | BC_AGE; 1563 bp->b_lblkno = offset >> vp->v_mount->mnt_fs_bshift; 1564 bp->b_data = (char *)kva; 1565 bp->b_bcount = npages << PAGE_SHIFT; 1566 bp->b_bufsize = npages << PAGE_SHIFT; 1567 bp->b_resid = 0; 1568 bp->b_error = error; 1569 uvm_aio_aiodone(bp); 1570 return (error); 1571 } 1572 1573 /* 1574 * Process a uio using direct I/O. If we reach a part of the request 1575 * which cannot be processed in this fashion for some reason, just return. 1576 * The caller must handle some additional part of the request using 1577 * buffered I/O before trying direct I/O again. 1578 */ 1579 1580 void 1581 genfs_directio(struct vnode *vp, struct uio *uio, int ioflag) 1582 { 1583 struct vmspace *vs; 1584 struct iovec *iov; 1585 vaddr_t va; 1586 size_t len; 1587 const int mask = DEV_BSIZE - 1; 1588 int error; 1589 bool need_wapbl = (vp->v_mount && vp->v_mount->mnt_wapbl && 1590 (ioflag & IO_JOURNALLOCKED) == 0); 1591 1592 /* 1593 * We only support direct I/O to user space for now. 1594 */ 1595 1596 if (VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) { 1597 return; 1598 } 1599 1600 /* 1601 * If the vnode is mapped, we would need to get the getpages lock 1602 * to stabilize the bmap, but then we would get into trouble whil e 1603 * locking the pages if the pages belong to this same vnode (or a 1604 * multi-vnode cascade to the same effect). Just fall back to 1605 * buffered I/O if the vnode is mapped to avoid this mess. 1606 */ 1607 1608 if (vp->v_vflag & VV_MAPPED) { 1609 return; 1610 } 1611 1612 if (need_wapbl) { 1613 error = WAPBL_BEGIN(vp->v_mount); 1614 if (error) 1615 return; 1616 } 1617 1618 /* 1619 * Do as much of the uio as possible with direct I/O. 1620 */ 1621 1622 vs = uio->uio_vmspace; 1623 while (uio->uio_resid) { 1624 iov = uio->uio_iov; 1625 if (iov->iov_len == 0) { 1626 uio->uio_iov++; 1627 uio->uio_iovcnt--; 1628 continue; 1629 } 1630 va = (vaddr_t)iov->iov_base; 1631 len = MIN(iov->iov_len, genfs_maxdio); 1632 len &= ~mask; 1633 1634 /* 1635 * If the next chunk is smaller than DEV_BSIZE or extends past 1636 * the current EOF, then fall back to buffered I/O. 1637 */ 1638 1639 if (len == 0 || uio->uio_offset + len > vp->v_size) { 1640 break; 1641 } 1642 1643 /* 1644 * Check alignment. The file offset must be at least 1645 * sector-aligned. The exact constraint on memory alignment 1646 * is very hardware-dependent, but requiring sector-aligned 1647 * addresses there too is safe. 1648 */ 1649 1650 if (uio->uio_offset & mask || va & mask) { 1651 break; 1652 } 1653 error = genfs_do_directio(vs, va, len, vp, uio->uio_offset, 1654 uio->uio_rw); 1655 if (error) { 1656 break; 1657 } 1658 iov->iov_base = (char *)iov->iov_base + len; 1659 iov->iov_len -= len; 1660 uio->uio_offset += len; 1661 uio->uio_resid -= len; 1662 } 1663 1664 if (need_wapbl) 1665 WAPBL_END(vp->v_mount); 1666 } 1667 1668 /* 1669 * Iodone routine for direct I/O. We don't do much here since the request is 1670 * always synchronous, so the caller will do most of the work after biowait(). 1671 */ 1672 1673 static void 1674 genfs_dio_iodone(struct buf *bp) 1675 { 1676 1677 KASSERT((bp->b_flags & B_ASYNC) == 0); 1678 if ((bp->b_flags & B_READ) == 0 && (bp->b_cflags & BC_AGE) != 0) { 1679 mutex_enter(bp->b_objlock); 1680 vwakeup(bp); 1681 mutex_exit(bp->b_objlock); 1682 } 1683 putiobuf(bp); 1684 } 1685 1686 /* 1687 * Process one chunk of a direct I/O request. 1688 */ 1689 1690 static int 1691 genfs_do_directio(struct vmspace *vs, vaddr_t uva, size_t len, struct vnode *vp, 1692 off_t off, enum uio_rw rw) 1693 { 1694 struct vm_map *map; 1695 struct pmap *upm, *kpm; 1696 size_t klen = round_page(uva + len) - trunc_page(uva); 1697 off_t spoff, epoff; 1698 vaddr_t kva, puva; 1699 paddr_t pa; 1700 vm_prot_t prot; 1701 int error, rv, poff, koff; 1702 const int pgoflags = PGO_CLEANIT | PGO_SYNCIO | PGO_JOURNALLOCKED | 1703 (rw == UIO_WRITE ? PGO_FREE : 0); 1704 1705 /* 1706 * For writes, verify that this range of the file already has fully 1707 * allocated backing store. If there are any holes, just punt and 1708 * make the caller take the buffered write path. 1709 */ 1710 1711 if (rw == UIO_WRITE) { 1712 daddr_t lbn, elbn, blkno; 1713 int bsize, bshift, run; 1714 1715 bshift = vp->v_mount->mnt_fs_bshift; 1716 bsize = 1 << bshift; 1717 lbn = off >> bshift; 1718 elbn = (off + len + bsize - 1) >> bshift; 1719 while (lbn < elbn) { 1720 error = VOP_BMAP(vp, lbn, NULL, &blkno, &run); 1721 if (error) { 1722 return error; 1723 } 1724 if (blkno == (daddr_t)-1) { 1725 return ENOSPC; 1726 } 1727 lbn += 1 + run; 1728 } 1729 } 1730 1731 /* 1732 * Flush any cached pages for parts of the file that we're about to 1733 * access. If we're writing, invalidate pages as well. 1734 */ 1735 1736 spoff = trunc_page(off); 1737 epoff = round_page(off + len); 1738 mutex_enter(&vp->v_interlock); 1739 error = VOP_PUTPAGES(vp, spoff, epoff, pgoflags); 1740 if (error) { 1741 return error; 1742 } 1743 1744 /* 1745 * Wire the user pages and remap them into kernel memory. 1746 */ 1747 1748 prot = rw == UIO_READ ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_READ; 1749 error = uvm_vslock(vs, (void *)uva, len, prot); 1750 if (error) { 1751 return error; 1752 } 1753 1754 map = &vs->vm_map; 1755 upm = vm_map_pmap(map); 1756 kpm = vm_map_pmap(kernel_map); 1757 kva = uvm_km_alloc(kernel_map, klen, 0, 1758 UVM_KMF_VAONLY | UVM_KMF_WAITVA); 1759 puva = trunc_page(uva); 1760 for (poff = 0; poff < klen; poff += PAGE_SIZE) { 1761 rv = pmap_extract(upm, puva + poff, &pa); 1762 KASSERT(rv); 1763 pmap_enter(kpm, kva + poff, pa, prot, prot | PMAP_WIRED); 1764 } 1765 pmap_update(kpm); 1766 1767 /* 1768 * Do the I/O. 1769 */ 1770 1771 koff = uva - trunc_page(uva); 1772 error = genfs_do_io(vp, off, kva + koff, len, PGO_SYNCIO, rw, 1773 genfs_dio_iodone); 1774 1775 /* 1776 * Tear down the kernel mapping. 1777 */ 1778 1779 pmap_remove(kpm, kva, kva + klen); 1780 pmap_update(kpm); 1781 uvm_km_free(kernel_map, kva, klen, UVM_KMF_VAONLY); 1782 1783 /* 1784 * Unwire the user pages. 1785 */ 1786 1787 uvm_vsunlock(vs, (void *)uva, len); 1788 return error; 1789 } 1790 1791