1 /* $NetBSD: genfs_io.c,v 1.39 2010/08/19 02:10:02 pooka Exp $ */ 2 3 /* 4 * Copyright (c) 1982, 1986, 1989, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the University nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 */ 32 33 #include <sys/cdefs.h> 34 __KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.39 2010/08/19 02:10:02 pooka Exp $"); 35 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/proc.h> 39 #include <sys/kernel.h> 40 #include <sys/mount.h> 41 #include <sys/namei.h> 42 #include <sys/vnode.h> 43 #include <sys/fcntl.h> 44 #include <sys/kmem.h> 45 #include <sys/poll.h> 46 #include <sys/mman.h> 47 #include <sys/file.h> 48 #include <sys/kauth.h> 49 #include <sys/fstrans.h> 50 #include <sys/buf.h> 51 52 #include <miscfs/genfs/genfs.h> 53 #include <miscfs/genfs/genfs_node.h> 54 #include <miscfs/specfs/specdev.h> 55 56 #include <uvm/uvm.h> 57 #include <uvm/uvm_pager.h> 58 59 static int genfs_do_directio(struct vmspace *, vaddr_t, size_t, struct vnode *, 60 off_t, enum uio_rw); 61 static void genfs_dio_iodone(struct buf *); 62 63 static int genfs_do_io(struct vnode *, off_t, vaddr_t, size_t, int, enum uio_rw, 64 void (*)(struct buf *)); 65 static void genfs_rel_pages(struct vm_page **, int); 66 static void genfs_markdirty(struct vnode *); 67 68 int genfs_maxdio = MAXPHYS; 69 70 static void 71 genfs_rel_pages(struct vm_page **pgs, int npages) 72 { 73 int i; 74 75 for (i = 0; i < npages; i++) { 76 struct vm_page *pg = pgs[i]; 77 78 if (pg == NULL || pg == PGO_DONTCARE) 79 continue; 80 if (pg->flags & PG_FAKE) { 81 pg->flags |= PG_RELEASED; 82 } 83 } 84 mutex_enter(&uvm_pageqlock); 85 uvm_page_unbusy(pgs, npages); 86 mutex_exit(&uvm_pageqlock); 87 } 88 89 static void 90 genfs_markdirty(struct vnode *vp) 91 { 92 struct genfs_node * const gp = VTOG(vp); 93 94 KASSERT(mutex_owned(&vp->v_interlock)); 95 gp->g_dirtygen++; 96 if ((vp->v_iflag & VI_ONWORKLST) == 0) { 97 vn_syncer_add_to_worklist(vp, filedelay); 98 } 99 if ((vp->v_iflag & (VI_WRMAP|VI_WRMAPDIRTY)) == VI_WRMAP) { 100 vp->v_iflag |= VI_WRMAPDIRTY; 101 } 102 } 103 104 /* 105 * generic VM getpages routine. 106 * Return PG_BUSY pages for the given range, 107 * reading from backing store if necessary. 108 */ 109 110 int 111 genfs_getpages(void *v) 112 { 113 struct vop_getpages_args /* { 114 struct vnode *a_vp; 115 voff_t a_offset; 116 struct vm_page **a_m; 117 int *a_count; 118 int a_centeridx; 119 vm_prot_t a_access_type; 120 int a_advice; 121 int a_flags; 122 } */ * const ap = v; 123 124 off_t diskeof, memeof; 125 int i, error, npages; 126 const int flags = ap->a_flags; 127 struct vnode * const vp = ap->a_vp; 128 struct genfs_node * const gp = VTOG(vp); 129 struct uvm_object * const uobj = &vp->v_uobj; 130 kauth_cred_t const cred = curlwp->l_cred; /* XXXUBC curlwp */ 131 const bool async = (flags & PGO_SYNCIO) == 0; 132 const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0; 133 bool has_trans = false; 134 const bool overwrite = (flags & PGO_OVERWRITE) != 0; 135 const bool blockalloc = memwrite && (flags & PGO_NOBLOCKALLOC) == 0; 136 UVMHIST_FUNC("genfs_getpages"); UVMHIST_CALLED(ubchist); 137 138 UVMHIST_LOG(ubchist, "vp %p off 0x%x/%x count %d", 139 vp, ap->a_offset >> 32, ap->a_offset, *ap->a_count); 140 141 KASSERT(vp->v_type == VREG || vp->v_type == VDIR || 142 vp->v_type == VLNK || vp->v_type == VBLK); 143 144 startover: 145 error = 0; 146 const voff_t origvsize = vp->v_size; 147 const off_t origoffset = ap->a_offset; 148 const int orignpages = *ap->a_count; 149 150 GOP_SIZE(vp, origvsize, &diskeof, 0); 151 if (flags & PGO_PASTEOF) { 152 off_t newsize; 153 #if defined(DIAGNOSTIC) 154 off_t writeeof; 155 #endif /* defined(DIAGNOSTIC) */ 156 157 newsize = MAX(origvsize, 158 origoffset + (orignpages << PAGE_SHIFT)); 159 GOP_SIZE(vp, newsize, &memeof, GOP_SIZE_MEM); 160 #if defined(DIAGNOSTIC) 161 GOP_SIZE(vp, vp->v_writesize, &writeeof, GOP_SIZE_MEM); 162 if (newsize > round_page(writeeof)) { 163 panic("%s: past eof: %" PRId64 " vs. %" PRId64, 164 __func__, newsize, round_page(writeeof)); 165 } 166 #endif /* defined(DIAGNOSTIC) */ 167 } else { 168 GOP_SIZE(vp, origvsize, &memeof, GOP_SIZE_MEM); 169 } 170 KASSERT(ap->a_centeridx >= 0 || ap->a_centeridx <= orignpages); 171 KASSERT((origoffset & (PAGE_SIZE - 1)) == 0 && origoffset >= 0); 172 KASSERT(orignpages > 0); 173 174 /* 175 * Bounds-check the request. 176 */ 177 178 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= memeof) { 179 if ((flags & PGO_LOCKED) == 0) { 180 mutex_exit(&uobj->vmobjlock); 181 } 182 UVMHIST_LOG(ubchist, "off 0x%x count %d goes past EOF 0x%x", 183 origoffset, *ap->a_count, memeof,0); 184 error = EINVAL; 185 goto out_err; 186 } 187 188 /* uobj is locked */ 189 190 if ((flags & PGO_NOTIMESTAMP) == 0 && 191 (vp->v_type != VBLK || 192 (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) { 193 int updflags = 0; 194 195 if ((vp->v_mount->mnt_flag & MNT_NOATIME) == 0) { 196 updflags = GOP_UPDATE_ACCESSED; 197 } 198 if (memwrite) { 199 updflags |= GOP_UPDATE_MODIFIED; 200 } 201 if (updflags != 0) { 202 GOP_MARKUPDATE(vp, updflags); 203 } 204 } 205 206 /* 207 * For PGO_LOCKED requests, just return whatever's in memory. 208 */ 209 210 if (flags & PGO_LOCKED) { 211 int nfound; 212 struct vm_page *pg; 213 214 npages = *ap->a_count; 215 #if defined(DEBUG) 216 for (i = 0; i < npages; i++) { 217 pg = ap->a_m[i]; 218 KASSERT(pg == NULL || pg == PGO_DONTCARE); 219 } 220 #endif /* defined(DEBUG) */ 221 nfound = uvn_findpages(uobj, origoffset, &npages, 222 ap->a_m, UFP_NOWAIT|UFP_NOALLOC|(memwrite ? UFP_NORDONLY : 0)); 223 KASSERT(npages == *ap->a_count); 224 if (nfound == 0) { 225 error = EBUSY; 226 goto out_err; 227 } 228 if (!genfs_node_rdtrylock(vp)) { 229 genfs_rel_pages(ap->a_m, npages); 230 231 /* 232 * restore the array. 233 */ 234 235 for (i = 0; i < npages; i++) { 236 pg = ap->a_m[i]; 237 238 if (pg != NULL || pg != PGO_DONTCARE) { 239 ap->a_m[i] = NULL; 240 } 241 } 242 } else { 243 genfs_node_unlock(vp); 244 } 245 error = (ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0); 246 if (error == 0 && memwrite) { 247 genfs_markdirty(vp); 248 } 249 goto out_err; 250 } 251 mutex_exit(&uobj->vmobjlock); 252 253 /* 254 * find the requested pages and make some simple checks. 255 * leave space in the page array for a whole block. 256 */ 257 258 const int fs_bshift = (vp->v_type != VBLK) ? 259 vp->v_mount->mnt_fs_bshift : DEV_BSHIFT; 260 const int dev_bshift = (vp->v_type != VBLK) ? 261 vp->v_mount->mnt_dev_bshift : DEV_BSHIFT; 262 const int fs_bsize = 1 << fs_bshift; 263 #define blk_mask (fs_bsize - 1) 264 #define trunc_blk(x) ((x) & ~blk_mask) 265 #define round_blk(x) (((x) + blk_mask) & ~blk_mask) 266 267 const int orignmempages = MIN(orignpages, 268 round_page(memeof - origoffset) >> PAGE_SHIFT); 269 npages = orignmempages; 270 const off_t startoffset = trunc_blk(origoffset); 271 const off_t endoffset = MIN( 272 round_page(round_blk(origoffset + (npages << PAGE_SHIFT))), 273 round_page(memeof)); 274 const int ridx = (origoffset - startoffset) >> PAGE_SHIFT; 275 276 const int pgs_size = sizeof(struct vm_page *) * 277 ((endoffset - startoffset) >> PAGE_SHIFT); 278 struct vm_page **pgs, *pgs_onstack[UBC_MAX_PAGES]; 279 280 if (pgs_size > sizeof(pgs_onstack)) { 281 pgs = kmem_zalloc(pgs_size, async ? KM_NOSLEEP : KM_SLEEP); 282 if (pgs == NULL) { 283 pgs = pgs_onstack; 284 error = ENOMEM; 285 goto out_err; 286 } 287 } else { 288 pgs = pgs_onstack; 289 (void)memset(pgs, 0, pgs_size); 290 } 291 292 UVMHIST_LOG(ubchist, "ridx %d npages %d startoff %ld endoff %ld", 293 ridx, npages, startoffset, endoffset); 294 295 if (!has_trans) { 296 fstrans_start(vp->v_mount, FSTRANS_SHARED); 297 has_trans = true; 298 } 299 300 /* 301 * hold g_glock to prevent a race with truncate. 302 * 303 * check if our idea of v_size is still valid. 304 */ 305 306 if (blockalloc) { 307 rw_enter(&gp->g_glock, RW_WRITER); 308 } else { 309 rw_enter(&gp->g_glock, RW_READER); 310 } 311 mutex_enter(&uobj->vmobjlock); 312 if (vp->v_size < origvsize) { 313 genfs_node_unlock(vp); 314 if (pgs != pgs_onstack) 315 kmem_free(pgs, pgs_size); 316 goto startover; 317 } 318 319 if (uvn_findpages(uobj, origoffset, &npages, &pgs[ridx], 320 async ? UFP_NOWAIT : UFP_ALL) != orignmempages) { 321 genfs_node_unlock(vp); 322 KASSERT(async != 0); 323 genfs_rel_pages(&pgs[ridx], orignmempages); 324 mutex_exit(&uobj->vmobjlock); 325 error = EBUSY; 326 goto out_err_free; 327 } 328 329 /* 330 * if the pages are already resident, just return them. 331 */ 332 333 for (i = 0; i < npages; i++) { 334 struct vm_page *pg = pgs[ridx + i]; 335 336 if ((pg->flags & PG_FAKE) || 337 (blockalloc && (pg->flags & PG_RDONLY))) { 338 break; 339 } 340 } 341 if (i == npages) { 342 genfs_node_unlock(vp); 343 UVMHIST_LOG(ubchist, "returning cached pages", 0,0,0,0); 344 npages += ridx; 345 goto out; 346 } 347 348 /* 349 * if PGO_OVERWRITE is set, don't bother reading the pages. 350 */ 351 352 if (overwrite) { 353 genfs_node_unlock(vp); 354 UVMHIST_LOG(ubchist, "PGO_OVERWRITE",0,0,0,0); 355 356 for (i = 0; i < npages; i++) { 357 struct vm_page *pg = pgs[ridx + i]; 358 359 pg->flags &= ~(PG_RDONLY|PG_CLEAN); 360 } 361 npages += ridx; 362 goto out; 363 } 364 365 /* 366 * the page wasn't resident and we're not overwriting, 367 * so we're going to have to do some i/o. 368 * find any additional pages needed to cover the expanded range. 369 */ 370 371 npages = (endoffset - startoffset) >> PAGE_SHIFT; 372 if (startoffset != origoffset || npages != orignmempages) { 373 int npgs; 374 375 /* 376 * we need to avoid deadlocks caused by locking 377 * additional pages at lower offsets than pages we 378 * already have locked. unlock them all and start over. 379 */ 380 381 genfs_rel_pages(&pgs[ridx], orignmempages); 382 memset(pgs, 0, pgs_size); 383 384 UVMHIST_LOG(ubchist, "reset npages start 0x%x end 0x%x", 385 startoffset, endoffset, 0,0); 386 npgs = npages; 387 if (uvn_findpages(uobj, startoffset, &npgs, pgs, 388 async ? UFP_NOWAIT : UFP_ALL) != npages) { 389 genfs_node_unlock(vp); 390 KASSERT(async != 0); 391 genfs_rel_pages(pgs, npages); 392 mutex_exit(&uobj->vmobjlock); 393 error = EBUSY; 394 goto out_err_free; 395 } 396 } 397 398 mutex_exit(&uobj->vmobjlock); 399 400 { 401 size_t bytes, iobytes, tailstart, tailbytes, totalbytes, skipbytes; 402 vaddr_t kva; 403 struct buf *bp, *mbp; 404 bool sawhole = false; 405 406 /* 407 * read the desired page(s). 408 */ 409 410 totalbytes = npages << PAGE_SHIFT; 411 bytes = MIN(totalbytes, MAX(diskeof - startoffset, 0)); 412 tailbytes = totalbytes - bytes; 413 skipbytes = 0; 414 415 kva = uvm_pagermapin(pgs, npages, 416 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 417 418 mbp = getiobuf(vp, true); 419 mbp->b_bufsize = totalbytes; 420 mbp->b_data = (void *)kva; 421 mbp->b_resid = mbp->b_bcount = bytes; 422 mbp->b_cflags = BC_BUSY; 423 if (async) { 424 mbp->b_flags = B_READ | B_ASYNC; 425 mbp->b_iodone = uvm_aio_biodone; 426 } else { 427 mbp->b_flags = B_READ; 428 mbp->b_iodone = NULL; 429 } 430 if (async) 431 BIO_SETPRIO(mbp, BPRIO_TIMELIMITED); 432 else 433 BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL); 434 435 /* 436 * if EOF is in the middle of the range, zero the part past EOF. 437 * skip over pages which are not PG_FAKE since in that case they have 438 * valid data that we need to preserve. 439 */ 440 441 tailstart = bytes; 442 while (tailbytes > 0) { 443 const int len = PAGE_SIZE - (tailstart & PAGE_MASK); 444 445 KASSERT(len <= tailbytes); 446 if ((pgs[tailstart >> PAGE_SHIFT]->flags & PG_FAKE) != 0) { 447 memset((void *)(kva + tailstart), 0, len); 448 UVMHIST_LOG(ubchist, "tailbytes %p 0x%x 0x%x", 449 kva, tailstart, len, 0); 450 } 451 tailstart += len; 452 tailbytes -= len; 453 } 454 455 /* 456 * now loop over the pages, reading as needed. 457 */ 458 459 bp = NULL; 460 off_t offset; 461 for (offset = startoffset; 462 bytes > 0; 463 offset += iobytes, bytes -= iobytes) { 464 int run; 465 daddr_t lbn, blkno; 466 int pidx; 467 struct vnode *devvp; 468 469 /* 470 * skip pages which don't need to be read. 471 */ 472 473 pidx = (offset - startoffset) >> PAGE_SHIFT; 474 while ((pgs[pidx]->flags & PG_FAKE) == 0) { 475 size_t b; 476 477 KASSERT((offset & (PAGE_SIZE - 1)) == 0); 478 if ((pgs[pidx]->flags & PG_RDONLY)) { 479 sawhole = true; 480 } 481 b = MIN(PAGE_SIZE, bytes); 482 offset += b; 483 bytes -= b; 484 skipbytes += b; 485 pidx++; 486 UVMHIST_LOG(ubchist, "skipping, new offset 0x%x", 487 offset, 0,0,0); 488 if (bytes == 0) { 489 goto loopdone; 490 } 491 } 492 493 /* 494 * bmap the file to find out the blkno to read from and 495 * how much we can read in one i/o. if bmap returns an error, 496 * skip the rest of the top-level i/o. 497 */ 498 499 lbn = offset >> fs_bshift; 500 error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run); 501 if (error) { 502 UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%x -> %d\n", 503 lbn,error,0,0); 504 skipbytes += bytes; 505 bytes = 0; 506 goto loopdone; 507 } 508 509 /* 510 * see how many pages can be read with this i/o. 511 * reduce the i/o size if necessary to avoid 512 * overwriting pages with valid data. 513 */ 514 515 iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset, 516 bytes); 517 if (offset + iobytes > round_page(offset)) { 518 int pcount; 519 520 pcount = 1; 521 while (pidx + pcount < npages && 522 pgs[pidx + pcount]->flags & PG_FAKE) { 523 pcount++; 524 } 525 iobytes = MIN(iobytes, (pcount << PAGE_SHIFT) - 526 (offset - trunc_page(offset))); 527 } 528 529 /* 530 * if this block isn't allocated, zero it instead of 531 * reading it. unless we are going to allocate blocks, 532 * mark the pages we zeroed PG_RDONLY. 533 */ 534 535 if (blkno == (daddr_t)-1) { 536 int holepages = (round_page(offset + iobytes) - 537 trunc_page(offset)) >> PAGE_SHIFT; 538 UVMHIST_LOG(ubchist, "lbn 0x%x -> HOLE", lbn,0,0,0); 539 540 sawhole = true; 541 memset((char *)kva + (offset - startoffset), 0, 542 iobytes); 543 skipbytes += iobytes; 544 545 for (i = 0; i < holepages; i++) { 546 if (memwrite) { 547 pgs[pidx + i]->flags &= ~PG_CLEAN; 548 } 549 if (!blockalloc) { 550 pgs[pidx + i]->flags |= PG_RDONLY; 551 } 552 } 553 continue; 554 } 555 556 /* 557 * allocate a sub-buf for this piece of the i/o 558 * (or just use mbp if there's only 1 piece), 559 * and start it going. 560 */ 561 562 if (offset == startoffset && iobytes == bytes) { 563 bp = mbp; 564 } else { 565 UVMHIST_LOG(ubchist, "vp %p bp %p num now %d", 566 vp, bp, vp->v_numoutput, 0); 567 bp = getiobuf(vp, true); 568 nestiobuf_setup(mbp, bp, offset - startoffset, iobytes); 569 } 570 bp->b_lblkno = 0; 571 572 /* adjust physical blkno for partial blocks */ 573 bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >> 574 dev_bshift); 575 576 UVMHIST_LOG(ubchist, 577 "bp %p offset 0x%x bcount 0x%x blkno 0x%x", 578 bp, offset, bp->b_bcount, bp->b_blkno); 579 580 VOP_STRATEGY(devvp, bp); 581 } 582 583 loopdone: 584 nestiobuf_done(mbp, skipbytes, error); 585 if (async) { 586 UVMHIST_LOG(ubchist, "returning 0 (async)",0,0,0,0); 587 genfs_node_unlock(vp); 588 error = 0; 589 goto out_err_free; 590 } 591 if (bp != NULL) { 592 error = biowait(mbp); 593 } 594 595 /* Remove the mapping (make KVA available as soon as possible) */ 596 uvm_pagermapout(kva, npages); 597 598 /* 599 * if this we encountered a hole then we have to do a little more work. 600 * for read faults, we marked the page PG_RDONLY so that future 601 * write accesses to the page will fault again. 602 * for write faults, we must make sure that the backing store for 603 * the page is completely allocated while the pages are locked. 604 */ 605 606 if (!error && sawhole && blockalloc) { 607 /* 608 * XXX: This assumes that we come here only via 609 * the mmio path 610 */ 611 if (vp->v_mount->mnt_wapbl) { 612 error = WAPBL_BEGIN(vp->v_mount); 613 } 614 615 if (!error) { 616 error = GOP_ALLOC(vp, startoffset, 617 npages << PAGE_SHIFT, 0, cred); 618 if (vp->v_mount->mnt_wapbl) { 619 WAPBL_END(vp->v_mount); 620 } 621 } 622 623 UVMHIST_LOG(ubchist, "gop_alloc off 0x%x/0x%x -> %d", 624 startoffset, npages << PAGE_SHIFT, error,0); 625 if (!error) { 626 for (i = 0; i < npages; i++) { 627 struct vm_page *pg = pgs[i]; 628 629 if (pg == NULL) { 630 continue; 631 } 632 pg->flags &= ~(PG_CLEAN|PG_RDONLY); 633 UVMHIST_LOG(ubchist, "mark dirty pg %p", 634 pg,0,0,0); 635 } 636 } 637 } 638 genfs_node_unlock(vp); 639 640 putiobuf(mbp); 641 } 642 643 mutex_enter(&uobj->vmobjlock); 644 645 /* 646 * we're almost done! release the pages... 647 * for errors, we free the pages. 648 * otherwise we activate them and mark them as valid and clean. 649 * also, unbusy pages that were not actually requested. 650 */ 651 652 if (error) { 653 for (i = 0; i < npages; i++) { 654 struct vm_page *pg = pgs[i]; 655 656 if (pg == NULL) { 657 continue; 658 } 659 UVMHIST_LOG(ubchist, "examining pg %p flags 0x%x", 660 pg, pg->flags, 0,0); 661 if (pg->flags & PG_FAKE) { 662 pg->flags |= PG_RELEASED; 663 } 664 } 665 mutex_enter(&uvm_pageqlock); 666 uvm_page_unbusy(pgs, npages); 667 mutex_exit(&uvm_pageqlock); 668 mutex_exit(&uobj->vmobjlock); 669 UVMHIST_LOG(ubchist, "returning error %d", error,0,0,0); 670 goto out_err_free; 671 } 672 673 out: 674 UVMHIST_LOG(ubchist, "succeeding, npages %d", npages,0,0,0); 675 error = 0; 676 mutex_enter(&uvm_pageqlock); 677 for (i = 0; i < npages; i++) { 678 struct vm_page *pg = pgs[i]; 679 if (pg == NULL) { 680 continue; 681 } 682 UVMHIST_LOG(ubchist, "examining pg %p flags 0x%x", 683 pg, pg->flags, 0,0); 684 if (pg->flags & PG_FAKE && !overwrite) { 685 pg->flags &= ~(PG_FAKE); 686 pmap_clear_modify(pgs[i]); 687 } 688 KASSERT(!memwrite || !blockalloc || (pg->flags & PG_RDONLY) == 0); 689 if (i < ridx || i >= ridx + orignmempages || async) { 690 UVMHIST_LOG(ubchist, "unbusy pg %p offset 0x%x", 691 pg, pg->offset,0,0); 692 if (pg->flags & PG_WANTED) { 693 wakeup(pg); 694 } 695 if (pg->flags & PG_FAKE) { 696 KASSERT(overwrite); 697 uvm_pagezero(pg); 698 } 699 if (pg->flags & PG_RELEASED) { 700 uvm_pagefree(pg); 701 continue; 702 } 703 uvm_pageenqueue(pg); 704 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE); 705 UVM_PAGE_OWN(pg, NULL); 706 } 707 } 708 mutex_exit(&uvm_pageqlock); 709 if (memwrite) { 710 genfs_markdirty(vp); 711 } 712 mutex_exit(&uobj->vmobjlock); 713 if (ap->a_m != NULL) { 714 memcpy(ap->a_m, &pgs[ridx], 715 orignmempages * sizeof(struct vm_page *)); 716 } 717 718 out_err_free: 719 if (pgs != NULL && pgs != pgs_onstack) 720 kmem_free(pgs, pgs_size); 721 out_err: 722 if (has_trans) 723 fstrans_done(vp->v_mount); 724 return error; 725 } 726 727 /* 728 * generic VM putpages routine. 729 * Write the given range of pages to backing store. 730 * 731 * => "offhi == 0" means flush all pages at or after "offlo". 732 * => object should be locked by caller. we return with the 733 * object unlocked. 734 * => if PGO_CLEANIT or PGO_SYNCIO is set, we may block (due to I/O). 735 * thus, a caller might want to unlock higher level resources 736 * (e.g. vm_map) before calling flush. 737 * => if neither PGO_CLEANIT nor PGO_SYNCIO is set, we will not block 738 * => if PGO_ALLPAGES is set, then all pages in the object will be processed. 739 * => NOTE: we rely on the fact that the object's memq is a TAILQ and 740 * that new pages are inserted on the tail end of the list. thus, 741 * we can make a complete pass through the object in one go by starting 742 * at the head and working towards the tail (new pages are put in 743 * front of us). 744 * => NOTE: we are allowed to lock the page queues, so the caller 745 * must not be holding the page queue lock. 746 * 747 * note on "cleaning" object and PG_BUSY pages: 748 * this routine is holding the lock on the object. the only time 749 * that it can run into a PG_BUSY page that it does not own is if 750 * some other process has started I/O on the page (e.g. either 751 * a pagein, or a pageout). if the PG_BUSY page is being paged 752 * in, then it can not be dirty (!PG_CLEAN) because no one has 753 * had a chance to modify it yet. if the PG_BUSY page is being 754 * paged out then it means that someone else has already started 755 * cleaning the page for us (how nice!). in this case, if we 756 * have syncio specified, then after we make our pass through the 757 * object we need to wait for the other PG_BUSY pages to clear 758 * off (i.e. we need to do an iosync). also note that once a 759 * page is PG_BUSY it must stay in its object until it is un-busyed. 760 * 761 * note on page traversal: 762 * we can traverse the pages in an object either by going down the 763 * linked list in "uobj->memq", or we can go over the address range 764 * by page doing hash table lookups for each address. depending 765 * on how many pages are in the object it may be cheaper to do one 766 * or the other. we set "by_list" to true if we are using memq. 767 * if the cost of a hash lookup was equal to the cost of the list 768 * traversal we could compare the number of pages in the start->stop 769 * range to the total number of pages in the object. however, it 770 * seems that a hash table lookup is more expensive than the linked 771 * list traversal, so we multiply the number of pages in the 772 * range by an estimate of the relatively higher cost of the hash lookup. 773 */ 774 775 int 776 genfs_putpages(void *v) 777 { 778 struct vop_putpages_args /* { 779 struct vnode *a_vp; 780 voff_t a_offlo; 781 voff_t a_offhi; 782 int a_flags; 783 } */ * const ap = v; 784 785 return genfs_do_putpages(ap->a_vp, ap->a_offlo, ap->a_offhi, 786 ap->a_flags, NULL); 787 } 788 789 int 790 genfs_do_putpages(struct vnode *vp, off_t startoff, off_t endoff, 791 int origflags, struct vm_page **busypg) 792 { 793 struct uvm_object * const uobj = &vp->v_uobj; 794 kmutex_t * const slock = &uobj->vmobjlock; 795 off_t off; 796 /* Even for strange MAXPHYS, the shift rounds down to a page */ 797 #define maxpages (MAXPHYS >> PAGE_SHIFT) 798 int i, error, npages, nback; 799 int freeflag; 800 struct vm_page *pgs[maxpages], *pg, *nextpg, *tpg, curmp, endmp; 801 bool wasclean, by_list, needs_clean, yld; 802 bool async = (origflags & PGO_SYNCIO) == 0; 803 bool pagedaemon = curlwp == uvm.pagedaemon_lwp; 804 struct lwp * const l = curlwp ? curlwp : &lwp0; 805 struct genfs_node * const gp = VTOG(vp); 806 int flags; 807 int dirtygen; 808 bool modified; 809 bool need_wapbl; 810 bool has_trans; 811 bool cleanall; 812 bool onworklst; 813 814 UVMHIST_FUNC("genfs_putpages"); UVMHIST_CALLED(ubchist); 815 816 KASSERT(origflags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)); 817 KASSERT((startoff & PAGE_MASK) == 0 && (endoff & PAGE_MASK) == 0); 818 KASSERT(startoff < endoff || endoff == 0); 819 820 UVMHIST_LOG(ubchist, "vp %p pages %d off 0x%x len 0x%x", 821 vp, uobj->uo_npages, startoff, endoff - startoff); 822 823 has_trans = false; 824 need_wapbl = (!pagedaemon && vp->v_mount && vp->v_mount->mnt_wapbl && 825 (origflags & PGO_JOURNALLOCKED) == 0); 826 827 retry: 828 modified = false; 829 flags = origflags; 830 KASSERT((vp->v_iflag & VI_ONWORKLST) != 0 || 831 (vp->v_iflag & VI_WRMAPDIRTY) == 0); 832 if (uobj->uo_npages == 0) { 833 if (vp->v_iflag & VI_ONWORKLST) { 834 vp->v_iflag &= ~VI_WRMAPDIRTY; 835 if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL) 836 vn_syncer_remove_from_worklist(vp); 837 } 838 if (has_trans) { 839 if (need_wapbl) 840 WAPBL_END(vp->v_mount); 841 fstrans_done(vp->v_mount); 842 } 843 mutex_exit(slock); 844 return (0); 845 } 846 847 /* 848 * the vnode has pages, set up to process the request. 849 */ 850 851 if (!has_trans && (flags & PGO_CLEANIT) != 0) { 852 mutex_exit(slock); 853 if (pagedaemon) { 854 error = fstrans_start_nowait(vp->v_mount, FSTRANS_LAZY); 855 if (error) 856 return error; 857 } else 858 fstrans_start(vp->v_mount, FSTRANS_LAZY); 859 if (need_wapbl) { 860 error = WAPBL_BEGIN(vp->v_mount); 861 if (error) { 862 fstrans_done(vp->v_mount); 863 return error; 864 } 865 } 866 has_trans = true; 867 mutex_enter(slock); 868 goto retry; 869 } 870 871 error = 0; 872 wasclean = (vp->v_numoutput == 0); 873 off = startoff; 874 if (endoff == 0 || flags & PGO_ALLPAGES) { 875 endoff = trunc_page(LLONG_MAX); 876 } 877 by_list = (uobj->uo_npages <= 878 ((endoff - startoff) >> PAGE_SHIFT) * UVM_PAGE_TREE_PENALTY); 879 880 #if !defined(DEBUG) 881 /* 882 * if this vnode is known not to have dirty pages, 883 * don't bother to clean it out. 884 */ 885 886 if ((vp->v_iflag & VI_ONWORKLST) == 0) { 887 if ((flags & (PGO_FREE|PGO_DEACTIVATE)) == 0) { 888 goto skip_scan; 889 } 890 flags &= ~PGO_CLEANIT; 891 } 892 #endif /* !defined(DEBUG) */ 893 894 /* 895 * start the loop. when scanning by list, hold the last page 896 * in the list before we start. pages allocated after we start 897 * will be added to the end of the list, so we can stop at the 898 * current last page. 899 */ 900 901 cleanall = (flags & PGO_CLEANIT) != 0 && wasclean && 902 startoff == 0 && endoff == trunc_page(LLONG_MAX) && 903 (vp->v_iflag & VI_ONWORKLST) != 0; 904 dirtygen = gp->g_dirtygen; 905 freeflag = pagedaemon ? PG_PAGEOUT : PG_RELEASED; 906 if (by_list) { 907 curmp.flags = PG_MARKER; 908 endmp.flags = PG_MARKER; 909 pg = TAILQ_FIRST(&uobj->memq); 910 TAILQ_INSERT_TAIL(&uobj->memq, &endmp, listq.queue); 911 } else { 912 pg = uvm_pagelookup(uobj, off); 913 } 914 nextpg = NULL; 915 while (by_list || off < endoff) { 916 917 /* 918 * if the current page is not interesting, move on to the next. 919 */ 920 921 KASSERT(pg == NULL || pg->uobject == uobj || 922 (pg->flags & PG_MARKER) != 0); 923 KASSERT(pg == NULL || 924 (pg->flags & (PG_RELEASED|PG_PAGEOUT)) == 0 || 925 (pg->flags & (PG_BUSY|PG_MARKER)) != 0); 926 if (by_list) { 927 if (pg == &endmp) { 928 break; 929 } 930 if (pg->flags & PG_MARKER) { 931 pg = TAILQ_NEXT(pg, listq.queue); 932 continue; 933 } 934 if (pg->offset < startoff || pg->offset >= endoff || 935 pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 936 if (pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 937 wasclean = false; 938 } 939 pg = TAILQ_NEXT(pg, listq.queue); 940 continue; 941 } 942 off = pg->offset; 943 } else if (pg == NULL || pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 944 if (pg != NULL) { 945 wasclean = false; 946 } 947 off += PAGE_SIZE; 948 if (off < endoff) { 949 pg = uvm_pagelookup(uobj, off); 950 } 951 continue; 952 } 953 954 /* 955 * if the current page needs to be cleaned and it's busy, 956 * wait for it to become unbusy. 957 */ 958 959 yld = (l->l_cpu->ci_schedstate.spc_flags & 960 SPCF_SHOULDYIELD) && !pagedaemon; 961 if (pg->flags & PG_BUSY || yld) { 962 UVMHIST_LOG(ubchist, "busy %p", pg,0,0,0); 963 if (flags & PGO_BUSYFAIL && pg->flags & PG_BUSY) { 964 UVMHIST_LOG(ubchist, "busyfail %p", pg, 0,0,0); 965 error = EDEADLK; 966 if (busypg != NULL) 967 *busypg = pg; 968 break; 969 } 970 if (pagedaemon) { 971 /* 972 * someone has taken the page while we 973 * dropped the lock for fstrans_start. 974 */ 975 break; 976 } 977 if (by_list) { 978 TAILQ_INSERT_BEFORE(pg, &curmp, listq.queue); 979 UVMHIST_LOG(ubchist, "curmp next %p", 980 TAILQ_NEXT(&curmp, listq.queue), 0,0,0); 981 } 982 if (yld) { 983 mutex_exit(slock); 984 preempt(); 985 mutex_enter(slock); 986 } else { 987 pg->flags |= PG_WANTED; 988 UVM_UNLOCK_AND_WAIT(pg, slock, 0, "genput", 0); 989 mutex_enter(slock); 990 } 991 if (by_list) { 992 UVMHIST_LOG(ubchist, "after next %p", 993 TAILQ_NEXT(&curmp, listq.queue), 0,0,0); 994 pg = TAILQ_NEXT(&curmp, listq.queue); 995 TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue); 996 } else { 997 pg = uvm_pagelookup(uobj, off); 998 } 999 continue; 1000 } 1001 1002 /* 1003 * if we're freeing, remove all mappings of the page now. 1004 * if we're cleaning, check if the page is needs to be cleaned. 1005 */ 1006 1007 if (flags & PGO_FREE) { 1008 pmap_page_protect(pg, VM_PROT_NONE); 1009 } else if (flags & PGO_CLEANIT) { 1010 1011 /* 1012 * if we still have some hope to pull this vnode off 1013 * from the syncer queue, write-protect the page. 1014 */ 1015 1016 if (cleanall && wasclean && 1017 gp->g_dirtygen == dirtygen) { 1018 1019 /* 1020 * uobj pages get wired only by uvm_fault 1021 * where uobj is locked. 1022 */ 1023 1024 if (pg->wire_count == 0) { 1025 pmap_page_protect(pg, 1026 VM_PROT_READ|VM_PROT_EXECUTE); 1027 } else { 1028 cleanall = false; 1029 } 1030 } 1031 } 1032 1033 if (flags & PGO_CLEANIT) { 1034 needs_clean = pmap_clear_modify(pg) || 1035 (pg->flags & PG_CLEAN) == 0; 1036 pg->flags |= PG_CLEAN; 1037 } else { 1038 needs_clean = false; 1039 } 1040 1041 /* 1042 * if we're cleaning, build a cluster. 1043 * the cluster will consist of pages which are currently dirty, 1044 * but they will be returned to us marked clean. 1045 * if not cleaning, just operate on the one page. 1046 */ 1047 1048 if (needs_clean) { 1049 KDASSERT((vp->v_iflag & VI_ONWORKLST)); 1050 wasclean = false; 1051 memset(pgs, 0, sizeof(pgs)); 1052 pg->flags |= PG_BUSY; 1053 UVM_PAGE_OWN(pg, "genfs_putpages"); 1054 1055 /* 1056 * first look backward. 1057 */ 1058 1059 npages = MIN(maxpages >> 1, off >> PAGE_SHIFT); 1060 nback = npages; 1061 uvn_findpages(uobj, off - PAGE_SIZE, &nback, &pgs[0], 1062 UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY|UFP_BACKWARD); 1063 if (nback) { 1064 memmove(&pgs[0], &pgs[npages - nback], 1065 nback * sizeof(pgs[0])); 1066 if (npages - nback < nback) 1067 memset(&pgs[nback], 0, 1068 (npages - nback) * sizeof(pgs[0])); 1069 else 1070 memset(&pgs[npages - nback], 0, 1071 nback * sizeof(pgs[0])); 1072 } 1073 1074 /* 1075 * then plug in our page of interest. 1076 */ 1077 1078 pgs[nback] = pg; 1079 1080 /* 1081 * then look forward to fill in the remaining space in 1082 * the array of pages. 1083 */ 1084 1085 npages = maxpages - nback - 1; 1086 uvn_findpages(uobj, off + PAGE_SIZE, &npages, 1087 &pgs[nback + 1], 1088 UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY); 1089 npages += nback + 1; 1090 } else { 1091 pgs[0] = pg; 1092 npages = 1; 1093 nback = 0; 1094 } 1095 1096 /* 1097 * apply FREE or DEACTIVATE options if requested. 1098 */ 1099 1100 if (flags & (PGO_DEACTIVATE|PGO_FREE)) { 1101 mutex_enter(&uvm_pageqlock); 1102 } 1103 for (i = 0; i < npages; i++) { 1104 tpg = pgs[i]; 1105 KASSERT(tpg->uobject == uobj); 1106 if (by_list && tpg == TAILQ_NEXT(pg, listq.queue)) 1107 pg = tpg; 1108 if (tpg->offset < startoff || tpg->offset >= endoff) 1109 continue; 1110 if (flags & PGO_DEACTIVATE && tpg->wire_count == 0) { 1111 uvm_pagedeactivate(tpg); 1112 } else if (flags & PGO_FREE) { 1113 pmap_page_protect(tpg, VM_PROT_NONE); 1114 if (tpg->flags & PG_BUSY) { 1115 tpg->flags |= freeflag; 1116 if (pagedaemon) { 1117 uvm_pageout_start(1); 1118 uvm_pagedequeue(tpg); 1119 } 1120 } else { 1121 1122 /* 1123 * ``page is not busy'' 1124 * implies that npages is 1 1125 * and needs_clean is false. 1126 */ 1127 1128 nextpg = TAILQ_NEXT(tpg, listq.queue); 1129 uvm_pagefree(tpg); 1130 if (pagedaemon) 1131 uvmexp.pdfreed++; 1132 } 1133 } 1134 } 1135 if (flags & (PGO_DEACTIVATE|PGO_FREE)) { 1136 mutex_exit(&uvm_pageqlock); 1137 } 1138 if (needs_clean) { 1139 modified = true; 1140 1141 /* 1142 * start the i/o. if we're traversing by list, 1143 * keep our place in the list with a marker page. 1144 */ 1145 1146 if (by_list) { 1147 TAILQ_INSERT_AFTER(&uobj->memq, pg, &curmp, 1148 listq.queue); 1149 } 1150 mutex_exit(slock); 1151 error = GOP_WRITE(vp, pgs, npages, flags); 1152 mutex_enter(slock); 1153 if (by_list) { 1154 pg = TAILQ_NEXT(&curmp, listq.queue); 1155 TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue); 1156 } 1157 if (error) { 1158 break; 1159 } 1160 if (by_list) { 1161 continue; 1162 } 1163 } 1164 1165 /* 1166 * find the next page and continue if there was no error. 1167 */ 1168 1169 if (by_list) { 1170 if (nextpg) { 1171 pg = nextpg; 1172 nextpg = NULL; 1173 } else { 1174 pg = TAILQ_NEXT(pg, listq.queue); 1175 } 1176 } else { 1177 off += (npages - nback) << PAGE_SHIFT; 1178 if (off < endoff) { 1179 pg = uvm_pagelookup(uobj, off); 1180 } 1181 } 1182 } 1183 if (by_list) { 1184 TAILQ_REMOVE(&uobj->memq, &endmp, listq.queue); 1185 } 1186 1187 if (modified && (vp->v_iflag & VI_WRMAPDIRTY) != 0 && 1188 (vp->v_type != VBLK || 1189 (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) { 1190 GOP_MARKUPDATE(vp, GOP_UPDATE_MODIFIED); 1191 } 1192 1193 /* 1194 * if we're cleaning and there was nothing to clean, 1195 * take us off the syncer list. if we started any i/o 1196 * and we're doing sync i/o, wait for all writes to finish. 1197 */ 1198 1199 if (cleanall && wasclean && gp->g_dirtygen == dirtygen && 1200 (vp->v_iflag & VI_ONWORKLST) != 0) { 1201 #if defined(DEBUG) 1202 TAILQ_FOREACH(pg, &uobj->memq, listq.queue) { 1203 if ((pg->flags & PG_MARKER) != 0) { 1204 continue; 1205 } 1206 if ((pg->flags & PG_CLEAN) == 0) { 1207 printf("%s: %p: !CLEAN\n", __func__, pg); 1208 } 1209 if (pmap_is_modified(pg)) { 1210 printf("%s: %p: modified\n", __func__, pg); 1211 } 1212 } 1213 #endif /* defined(DEBUG) */ 1214 vp->v_iflag &= ~VI_WRMAPDIRTY; 1215 if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL) 1216 vn_syncer_remove_from_worklist(vp); 1217 } 1218 1219 #if !defined(DEBUG) 1220 skip_scan: 1221 #endif /* !defined(DEBUG) */ 1222 1223 /* Wait for output to complete. */ 1224 if (!wasclean && !async && vp->v_numoutput != 0) { 1225 while (vp->v_numoutput != 0) 1226 cv_wait(&vp->v_cv, slock); 1227 } 1228 onworklst = (vp->v_iflag & VI_ONWORKLST) != 0; 1229 mutex_exit(slock); 1230 1231 if ((flags & PGO_RECLAIM) != 0 && onworklst) { 1232 /* 1233 * in the case of PGO_RECLAIM, ensure to make the vnode clean. 1234 * retrying is not a big deal because, in many cases, 1235 * uobj->uo_npages is already 0 here. 1236 */ 1237 mutex_enter(slock); 1238 goto retry; 1239 } 1240 1241 if (has_trans) { 1242 if (need_wapbl) 1243 WAPBL_END(vp->v_mount); 1244 fstrans_done(vp->v_mount); 1245 } 1246 1247 return (error); 1248 } 1249 1250 int 1251 genfs_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, int flags) 1252 { 1253 off_t off; 1254 vaddr_t kva; 1255 size_t len; 1256 int error; 1257 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1258 1259 UVMHIST_LOG(ubchist, "vp %p pgs %p npages %d flags 0x%x", 1260 vp, pgs, npages, flags); 1261 1262 off = pgs[0]->offset; 1263 kva = uvm_pagermapin(pgs, npages, 1264 UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK); 1265 len = npages << PAGE_SHIFT; 1266 1267 error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE, 1268 uvm_aio_biodone); 1269 1270 return error; 1271 } 1272 1273 int 1274 genfs_gop_write_rwmap(struct vnode *vp, struct vm_page **pgs, int npages, int flags) 1275 { 1276 off_t off; 1277 vaddr_t kva; 1278 size_t len; 1279 int error; 1280 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1281 1282 UVMHIST_LOG(ubchist, "vp %p pgs %p npages %d flags 0x%x", 1283 vp, pgs, npages, flags); 1284 1285 off = pgs[0]->offset; 1286 kva = uvm_pagermapin(pgs, npages, 1287 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 1288 len = npages << PAGE_SHIFT; 1289 1290 error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE, 1291 uvm_aio_biodone); 1292 1293 return error; 1294 } 1295 1296 /* 1297 * Backend routine for doing I/O to vnode pages. Pages are already locked 1298 * and mapped into kernel memory. Here we just look up the underlying 1299 * device block addresses and call the strategy routine. 1300 */ 1301 1302 static int 1303 genfs_do_io(struct vnode *vp, off_t off, vaddr_t kva, size_t len, int flags, 1304 enum uio_rw rw, void (*iodone)(struct buf *)) 1305 { 1306 int s, error; 1307 int fs_bshift, dev_bshift; 1308 off_t eof, offset, startoffset; 1309 size_t bytes, iobytes, skipbytes; 1310 struct buf *mbp, *bp; 1311 const bool async = (flags & PGO_SYNCIO) == 0; 1312 const bool iowrite = rw == UIO_WRITE; 1313 const int brw = iowrite ? B_WRITE : B_READ; 1314 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1315 1316 UVMHIST_LOG(ubchist, "vp %p kva %p len 0x%x flags 0x%x", 1317 vp, kva, len, flags); 1318 1319 KASSERT(vp->v_size <= vp->v_writesize); 1320 GOP_SIZE(vp, vp->v_writesize, &eof, 0); 1321 if (vp->v_type != VBLK) { 1322 fs_bshift = vp->v_mount->mnt_fs_bshift; 1323 dev_bshift = vp->v_mount->mnt_dev_bshift; 1324 } else { 1325 fs_bshift = DEV_BSHIFT; 1326 dev_bshift = DEV_BSHIFT; 1327 } 1328 error = 0; 1329 startoffset = off; 1330 bytes = MIN(len, eof - startoffset); 1331 skipbytes = 0; 1332 KASSERT(bytes != 0); 1333 1334 if (iowrite) { 1335 mutex_enter(&vp->v_interlock); 1336 vp->v_numoutput += 2; 1337 mutex_exit(&vp->v_interlock); 1338 } 1339 mbp = getiobuf(vp, true); 1340 UVMHIST_LOG(ubchist, "vp %p mbp %p num now %d bytes 0x%x", 1341 vp, mbp, vp->v_numoutput, bytes); 1342 mbp->b_bufsize = len; 1343 mbp->b_data = (void *)kva; 1344 mbp->b_resid = mbp->b_bcount = bytes; 1345 mbp->b_cflags = BC_BUSY | BC_AGE; 1346 if (async) { 1347 mbp->b_flags = brw | B_ASYNC; 1348 mbp->b_iodone = iodone; 1349 } else { 1350 mbp->b_flags = brw; 1351 mbp->b_iodone = NULL; 1352 } 1353 if (curlwp == uvm.pagedaemon_lwp) 1354 BIO_SETPRIO(mbp, BPRIO_TIMELIMITED); 1355 else if (async) 1356 BIO_SETPRIO(mbp, BPRIO_TIMENONCRITICAL); 1357 else 1358 BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL); 1359 1360 bp = NULL; 1361 for (offset = startoffset; 1362 bytes > 0; 1363 offset += iobytes, bytes -= iobytes) { 1364 int run; 1365 daddr_t lbn, blkno; 1366 struct vnode *devvp; 1367 1368 /* 1369 * bmap the file to find out the blkno to read from and 1370 * how much we can read in one i/o. if bmap returns an error, 1371 * skip the rest of the top-level i/o. 1372 */ 1373 1374 lbn = offset >> fs_bshift; 1375 error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run); 1376 if (error) { 1377 UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%x -> %d\n", 1378 lbn,error,0,0); 1379 skipbytes += bytes; 1380 bytes = 0; 1381 goto loopdone; 1382 } 1383 1384 /* 1385 * see how many pages can be read with this i/o. 1386 * reduce the i/o size if necessary to avoid 1387 * overwriting pages with valid data. 1388 */ 1389 1390 iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset, 1391 bytes); 1392 1393 /* 1394 * if this block isn't allocated, zero it instead of 1395 * reading it. unless we are going to allocate blocks, 1396 * mark the pages we zeroed PG_RDONLY. 1397 */ 1398 1399 if (blkno == (daddr_t)-1) { 1400 if (!iowrite) { 1401 memset((char *)kva + (offset - startoffset), 0, 1402 iobytes); 1403 } 1404 skipbytes += iobytes; 1405 continue; 1406 } 1407 1408 /* 1409 * allocate a sub-buf for this piece of the i/o 1410 * (or just use mbp if there's only 1 piece), 1411 * and start it going. 1412 */ 1413 1414 if (offset == startoffset && iobytes == bytes) { 1415 bp = mbp; 1416 } else { 1417 UVMHIST_LOG(ubchist, "vp %p bp %p num now %d", 1418 vp, bp, vp->v_numoutput, 0); 1419 bp = getiobuf(vp, true); 1420 nestiobuf_setup(mbp, bp, offset - startoffset, iobytes); 1421 } 1422 bp->b_lblkno = 0; 1423 1424 /* adjust physical blkno for partial blocks */ 1425 bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >> 1426 dev_bshift); 1427 1428 UVMHIST_LOG(ubchist, 1429 "bp %p offset 0x%x bcount 0x%x blkno 0x%x", 1430 bp, offset, bp->b_bcount, bp->b_blkno); 1431 1432 VOP_STRATEGY(devvp, bp); 1433 } 1434 1435 loopdone: 1436 if (skipbytes) { 1437 UVMHIST_LOG(ubchist, "skipbytes %d", skipbytes, 0,0,0); 1438 } 1439 nestiobuf_done(mbp, skipbytes, error); 1440 if (async) { 1441 UVMHIST_LOG(ubchist, "returning 0 (async)", 0,0,0,0); 1442 return (0); 1443 } 1444 UVMHIST_LOG(ubchist, "waiting for mbp %p", mbp,0,0,0); 1445 error = biowait(mbp); 1446 s = splbio(); 1447 (*iodone)(mbp); 1448 splx(s); 1449 UVMHIST_LOG(ubchist, "returning, error %d", error,0,0,0); 1450 return (error); 1451 } 1452 1453 int 1454 genfs_compat_getpages(void *v) 1455 { 1456 struct vop_getpages_args /* { 1457 struct vnode *a_vp; 1458 voff_t a_offset; 1459 struct vm_page **a_m; 1460 int *a_count; 1461 int a_centeridx; 1462 vm_prot_t a_access_type; 1463 int a_advice; 1464 int a_flags; 1465 } */ *ap = v; 1466 1467 off_t origoffset; 1468 struct vnode *vp = ap->a_vp; 1469 struct uvm_object *uobj = &vp->v_uobj; 1470 struct vm_page *pg, **pgs; 1471 vaddr_t kva; 1472 int i, error, orignpages, npages; 1473 struct iovec iov; 1474 struct uio uio; 1475 kauth_cred_t cred = curlwp->l_cred; 1476 const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0; 1477 1478 error = 0; 1479 origoffset = ap->a_offset; 1480 orignpages = *ap->a_count; 1481 pgs = ap->a_m; 1482 1483 if (ap->a_flags & PGO_LOCKED) { 1484 uvn_findpages(uobj, origoffset, ap->a_count, ap->a_m, 1485 UFP_NOWAIT|UFP_NOALLOC| (memwrite ? UFP_NORDONLY : 0)); 1486 1487 error = ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0; 1488 if (error == 0 && memwrite) { 1489 genfs_markdirty(vp); 1490 } 1491 return error; 1492 } 1493 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= vp->v_size) { 1494 mutex_exit(&uobj->vmobjlock); 1495 return EINVAL; 1496 } 1497 if ((ap->a_flags & PGO_SYNCIO) == 0) { 1498 mutex_exit(&uobj->vmobjlock); 1499 return 0; 1500 } 1501 npages = orignpages; 1502 uvn_findpages(uobj, origoffset, &npages, pgs, UFP_ALL); 1503 mutex_exit(&uobj->vmobjlock); 1504 kva = uvm_pagermapin(pgs, npages, 1505 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 1506 for (i = 0; i < npages; i++) { 1507 pg = pgs[i]; 1508 if ((pg->flags & PG_FAKE) == 0) { 1509 continue; 1510 } 1511 iov.iov_base = (char *)kva + (i << PAGE_SHIFT); 1512 iov.iov_len = PAGE_SIZE; 1513 uio.uio_iov = &iov; 1514 uio.uio_iovcnt = 1; 1515 uio.uio_offset = origoffset + (i << PAGE_SHIFT); 1516 uio.uio_rw = UIO_READ; 1517 uio.uio_resid = PAGE_SIZE; 1518 UIO_SETUP_SYSSPACE(&uio); 1519 /* XXX vn_lock */ 1520 error = VOP_READ(vp, &uio, 0, cred); 1521 if (error) { 1522 break; 1523 } 1524 if (uio.uio_resid) { 1525 memset(iov.iov_base, 0, uio.uio_resid); 1526 } 1527 } 1528 uvm_pagermapout(kva, npages); 1529 mutex_enter(&uobj->vmobjlock); 1530 mutex_enter(&uvm_pageqlock); 1531 for (i = 0; i < npages; i++) { 1532 pg = pgs[i]; 1533 if (error && (pg->flags & PG_FAKE) != 0) { 1534 pg->flags |= PG_RELEASED; 1535 } else { 1536 pmap_clear_modify(pg); 1537 uvm_pageactivate(pg); 1538 } 1539 } 1540 if (error) { 1541 uvm_page_unbusy(pgs, npages); 1542 } 1543 mutex_exit(&uvm_pageqlock); 1544 if (error == 0 && memwrite) { 1545 genfs_markdirty(vp); 1546 } 1547 mutex_exit(&uobj->vmobjlock); 1548 return error; 1549 } 1550 1551 int 1552 genfs_compat_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, 1553 int flags) 1554 { 1555 off_t offset; 1556 struct iovec iov; 1557 struct uio uio; 1558 kauth_cred_t cred = curlwp->l_cred; 1559 struct buf *bp; 1560 vaddr_t kva; 1561 int error; 1562 1563 offset = pgs[0]->offset; 1564 kva = uvm_pagermapin(pgs, npages, 1565 UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK); 1566 1567 iov.iov_base = (void *)kva; 1568 iov.iov_len = npages << PAGE_SHIFT; 1569 uio.uio_iov = &iov; 1570 uio.uio_iovcnt = 1; 1571 uio.uio_offset = offset; 1572 uio.uio_rw = UIO_WRITE; 1573 uio.uio_resid = npages << PAGE_SHIFT; 1574 UIO_SETUP_SYSSPACE(&uio); 1575 /* XXX vn_lock */ 1576 error = VOP_WRITE(vp, &uio, 0, cred); 1577 1578 mutex_enter(&vp->v_interlock); 1579 vp->v_numoutput++; 1580 mutex_exit(&vp->v_interlock); 1581 1582 bp = getiobuf(vp, true); 1583 bp->b_cflags = BC_BUSY | BC_AGE; 1584 bp->b_lblkno = offset >> vp->v_mount->mnt_fs_bshift; 1585 bp->b_data = (char *)kva; 1586 bp->b_bcount = npages << PAGE_SHIFT; 1587 bp->b_bufsize = npages << PAGE_SHIFT; 1588 bp->b_resid = 0; 1589 bp->b_error = error; 1590 uvm_aio_aiodone(bp); 1591 return (error); 1592 } 1593 1594 /* 1595 * Process a uio using direct I/O. If we reach a part of the request 1596 * which cannot be processed in this fashion for some reason, just return. 1597 * The caller must handle some additional part of the request using 1598 * buffered I/O before trying direct I/O again. 1599 */ 1600 1601 void 1602 genfs_directio(struct vnode *vp, struct uio *uio, int ioflag) 1603 { 1604 struct vmspace *vs; 1605 struct iovec *iov; 1606 vaddr_t va; 1607 size_t len; 1608 const int mask = DEV_BSIZE - 1; 1609 int error; 1610 bool need_wapbl = (vp->v_mount && vp->v_mount->mnt_wapbl && 1611 (ioflag & IO_JOURNALLOCKED) == 0); 1612 1613 /* 1614 * We only support direct I/O to user space for now. 1615 */ 1616 1617 if (VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) { 1618 return; 1619 } 1620 1621 /* 1622 * If the vnode is mapped, we would need to get the getpages lock 1623 * to stabilize the bmap, but then we would get into trouble whil e 1624 * locking the pages if the pages belong to this same vnode (or a 1625 * multi-vnode cascade to the same effect). Just fall back to 1626 * buffered I/O if the vnode is mapped to avoid this mess. 1627 */ 1628 1629 if (vp->v_vflag & VV_MAPPED) { 1630 return; 1631 } 1632 1633 if (need_wapbl) { 1634 error = WAPBL_BEGIN(vp->v_mount); 1635 if (error) 1636 return; 1637 } 1638 1639 /* 1640 * Do as much of the uio as possible with direct I/O. 1641 */ 1642 1643 vs = uio->uio_vmspace; 1644 while (uio->uio_resid) { 1645 iov = uio->uio_iov; 1646 if (iov->iov_len == 0) { 1647 uio->uio_iov++; 1648 uio->uio_iovcnt--; 1649 continue; 1650 } 1651 va = (vaddr_t)iov->iov_base; 1652 len = MIN(iov->iov_len, genfs_maxdio); 1653 len &= ~mask; 1654 1655 /* 1656 * If the next chunk is smaller than DEV_BSIZE or extends past 1657 * the current EOF, then fall back to buffered I/O. 1658 */ 1659 1660 if (len == 0 || uio->uio_offset + len > vp->v_size) { 1661 break; 1662 } 1663 1664 /* 1665 * Check alignment. The file offset must be at least 1666 * sector-aligned. The exact constraint on memory alignment 1667 * is very hardware-dependent, but requiring sector-aligned 1668 * addresses there too is safe. 1669 */ 1670 1671 if (uio->uio_offset & mask || va & mask) { 1672 break; 1673 } 1674 error = genfs_do_directio(vs, va, len, vp, uio->uio_offset, 1675 uio->uio_rw); 1676 if (error) { 1677 break; 1678 } 1679 iov->iov_base = (char *)iov->iov_base + len; 1680 iov->iov_len -= len; 1681 uio->uio_offset += len; 1682 uio->uio_resid -= len; 1683 } 1684 1685 if (need_wapbl) 1686 WAPBL_END(vp->v_mount); 1687 } 1688 1689 /* 1690 * Iodone routine for direct I/O. We don't do much here since the request is 1691 * always synchronous, so the caller will do most of the work after biowait(). 1692 */ 1693 1694 static void 1695 genfs_dio_iodone(struct buf *bp) 1696 { 1697 1698 KASSERT((bp->b_flags & B_ASYNC) == 0); 1699 if ((bp->b_flags & B_READ) == 0 && (bp->b_cflags & BC_AGE) != 0) { 1700 mutex_enter(bp->b_objlock); 1701 vwakeup(bp); 1702 mutex_exit(bp->b_objlock); 1703 } 1704 putiobuf(bp); 1705 } 1706 1707 /* 1708 * Process one chunk of a direct I/O request. 1709 */ 1710 1711 static int 1712 genfs_do_directio(struct vmspace *vs, vaddr_t uva, size_t len, struct vnode *vp, 1713 off_t off, enum uio_rw rw) 1714 { 1715 struct vm_map *map; 1716 struct pmap *upm, *kpm; 1717 size_t klen = round_page(uva + len) - trunc_page(uva); 1718 off_t spoff, epoff; 1719 vaddr_t kva, puva; 1720 paddr_t pa; 1721 vm_prot_t prot; 1722 int error, rv, poff, koff; 1723 const int pgoflags = PGO_CLEANIT | PGO_SYNCIO | PGO_JOURNALLOCKED | 1724 (rw == UIO_WRITE ? PGO_FREE : 0); 1725 1726 /* 1727 * For writes, verify that this range of the file already has fully 1728 * allocated backing store. If there are any holes, just punt and 1729 * make the caller take the buffered write path. 1730 */ 1731 1732 if (rw == UIO_WRITE) { 1733 daddr_t lbn, elbn, blkno; 1734 int bsize, bshift, run; 1735 1736 bshift = vp->v_mount->mnt_fs_bshift; 1737 bsize = 1 << bshift; 1738 lbn = off >> bshift; 1739 elbn = (off + len + bsize - 1) >> bshift; 1740 while (lbn < elbn) { 1741 error = VOP_BMAP(vp, lbn, NULL, &blkno, &run); 1742 if (error) { 1743 return error; 1744 } 1745 if (blkno == (daddr_t)-1) { 1746 return ENOSPC; 1747 } 1748 lbn += 1 + run; 1749 } 1750 } 1751 1752 /* 1753 * Flush any cached pages for parts of the file that we're about to 1754 * access. If we're writing, invalidate pages as well. 1755 */ 1756 1757 spoff = trunc_page(off); 1758 epoff = round_page(off + len); 1759 mutex_enter(&vp->v_interlock); 1760 error = VOP_PUTPAGES(vp, spoff, epoff, pgoflags); 1761 if (error) { 1762 return error; 1763 } 1764 1765 /* 1766 * Wire the user pages and remap them into kernel memory. 1767 */ 1768 1769 prot = rw == UIO_READ ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_READ; 1770 error = uvm_vslock(vs, (void *)uva, len, prot); 1771 if (error) { 1772 return error; 1773 } 1774 1775 map = &vs->vm_map; 1776 upm = vm_map_pmap(map); 1777 kpm = vm_map_pmap(kernel_map); 1778 kva = uvm_km_alloc(kernel_map, klen, 0, 1779 UVM_KMF_VAONLY | UVM_KMF_WAITVA); 1780 puva = trunc_page(uva); 1781 for (poff = 0; poff < klen; poff += PAGE_SIZE) { 1782 rv = pmap_extract(upm, puva + poff, &pa); 1783 KASSERT(rv); 1784 pmap_enter(kpm, kva + poff, pa, prot, prot | PMAP_WIRED); 1785 } 1786 pmap_update(kpm); 1787 1788 /* 1789 * Do the I/O. 1790 */ 1791 1792 koff = uva - trunc_page(uva); 1793 error = genfs_do_io(vp, off, kva + koff, len, PGO_SYNCIO, rw, 1794 genfs_dio_iodone); 1795 1796 /* 1797 * Tear down the kernel mapping. 1798 */ 1799 1800 pmap_remove(kpm, kva, kva + klen); 1801 pmap_update(kpm); 1802 uvm_km_free(kernel_map, kva, klen, UVM_KMF_VAONLY); 1803 1804 /* 1805 * Unwire the user pages. 1806 */ 1807 1808 uvm_vsunlock(vs, (void *)uva, len); 1809 return error; 1810 } 1811 1812