1 /* $NetBSD: genfs_io.c,v 1.46 2010/12/06 10:22:43 uebayasi Exp $ */ 2 3 /* 4 * Copyright (c) 1982, 1986, 1989, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the University nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 */ 32 33 #include <sys/cdefs.h> 34 __KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.46 2010/12/06 10:22:43 uebayasi Exp $"); 35 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/proc.h> 39 #include <sys/kernel.h> 40 #include <sys/mount.h> 41 #include <sys/namei.h> 42 #include <sys/vnode.h> 43 #include <sys/fcntl.h> 44 #include <sys/kmem.h> 45 #include <sys/poll.h> 46 #include <sys/mman.h> 47 #include <sys/file.h> 48 #include <sys/kauth.h> 49 #include <sys/fstrans.h> 50 #include <sys/buf.h> 51 52 #include <miscfs/genfs/genfs.h> 53 #include <miscfs/genfs/genfs_node.h> 54 #include <miscfs/specfs/specdev.h> 55 56 #include <uvm/uvm.h> 57 #include <uvm/uvm_pager.h> 58 59 static int genfs_do_directio(struct vmspace *, vaddr_t, size_t, struct vnode *, 60 off_t, enum uio_rw); 61 static void genfs_dio_iodone(struct buf *); 62 63 static int genfs_do_io(struct vnode *, off_t, vaddr_t, size_t, int, enum uio_rw, 64 void (*)(struct buf *)); 65 static void genfs_rel_pages(struct vm_page **, int); 66 static void genfs_markdirty(struct vnode *); 67 68 int genfs_maxdio = MAXPHYS; 69 70 static void 71 genfs_rel_pages(struct vm_page **pgs, int npages) 72 { 73 int i; 74 75 for (i = 0; i < npages; i++) { 76 struct vm_page *pg = pgs[i]; 77 78 if (pg == NULL || pg == PGO_DONTCARE) 79 continue; 80 if (pg->flags & PG_FAKE) { 81 pg->flags |= PG_RELEASED; 82 } 83 } 84 mutex_enter(&uvm_pageqlock); 85 uvm_page_unbusy(pgs, npages); 86 mutex_exit(&uvm_pageqlock); 87 } 88 89 static void 90 genfs_markdirty(struct vnode *vp) 91 { 92 struct genfs_node * const gp = VTOG(vp); 93 94 KASSERT(mutex_owned(&vp->v_interlock)); 95 gp->g_dirtygen++; 96 if ((vp->v_iflag & VI_ONWORKLST) == 0) { 97 vn_syncer_add_to_worklist(vp, filedelay); 98 } 99 if ((vp->v_iflag & (VI_WRMAP|VI_WRMAPDIRTY)) == VI_WRMAP) { 100 vp->v_iflag |= VI_WRMAPDIRTY; 101 } 102 } 103 104 /* 105 * generic VM getpages routine. 106 * Return PG_BUSY pages for the given range, 107 * reading from backing store if necessary. 108 */ 109 110 int 111 genfs_getpages(void *v) 112 { 113 struct vop_getpages_args /* { 114 struct vnode *a_vp; 115 voff_t a_offset; 116 struct vm_page **a_m; 117 int *a_count; 118 int a_centeridx; 119 vm_prot_t a_access_type; 120 int a_advice; 121 int a_flags; 122 } */ * const ap = v; 123 124 off_t diskeof, memeof; 125 int i, error, npages; 126 const int flags = ap->a_flags; 127 struct vnode * const vp = ap->a_vp; 128 struct uvm_object * const uobj = &vp->v_uobj; 129 kauth_cred_t const cred = curlwp->l_cred; /* XXXUBC curlwp */ 130 const bool async = (flags & PGO_SYNCIO) == 0; 131 const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0; 132 const bool overwrite = (flags & PGO_OVERWRITE) != 0; 133 const bool blockalloc = memwrite && (flags & PGO_NOBLOCKALLOC) == 0; 134 const bool glocked = (flags & PGO_GLOCKHELD) != 0; 135 const bool need_wapbl = blockalloc && vp->v_mount->mnt_wapbl; 136 bool has_trans_wapbl = false; 137 UVMHIST_FUNC("genfs_getpages"); UVMHIST_CALLED(ubchist); 138 139 UVMHIST_LOG(ubchist, "vp %p off 0x%x/%x count %d", 140 vp, ap->a_offset >> 32, ap->a_offset, *ap->a_count); 141 142 KASSERT(vp->v_type == VREG || vp->v_type == VDIR || 143 vp->v_type == VLNK || vp->v_type == VBLK); 144 145 startover: 146 error = 0; 147 const voff_t origvsize = vp->v_size; 148 const off_t origoffset = ap->a_offset; 149 const int orignpages = *ap->a_count; 150 151 GOP_SIZE(vp, origvsize, &diskeof, 0); 152 if (flags & PGO_PASTEOF) { 153 off_t newsize; 154 #if defined(DIAGNOSTIC) 155 off_t writeeof; 156 #endif /* defined(DIAGNOSTIC) */ 157 158 newsize = MAX(origvsize, 159 origoffset + (orignpages << PAGE_SHIFT)); 160 GOP_SIZE(vp, newsize, &memeof, GOP_SIZE_MEM); 161 #if defined(DIAGNOSTIC) 162 GOP_SIZE(vp, vp->v_writesize, &writeeof, GOP_SIZE_MEM); 163 if (newsize > round_page(writeeof)) { 164 panic("%s: past eof: %" PRId64 " vs. %" PRId64, 165 __func__, newsize, round_page(writeeof)); 166 } 167 #endif /* defined(DIAGNOSTIC) */ 168 } else { 169 GOP_SIZE(vp, origvsize, &memeof, GOP_SIZE_MEM); 170 } 171 KASSERT(ap->a_centeridx >= 0 || ap->a_centeridx <= orignpages); 172 KASSERT((origoffset & (PAGE_SIZE - 1)) == 0 && origoffset >= 0); 173 KASSERT(orignpages > 0); 174 175 /* 176 * Bounds-check the request. 177 */ 178 179 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= memeof) { 180 if ((flags & PGO_LOCKED) == 0) { 181 mutex_exit(&uobj->vmobjlock); 182 } 183 UVMHIST_LOG(ubchist, "off 0x%x count %d goes past EOF 0x%x", 184 origoffset, *ap->a_count, memeof,0); 185 error = EINVAL; 186 goto out_err; 187 } 188 189 /* uobj is locked */ 190 191 if ((flags & PGO_NOTIMESTAMP) == 0 && 192 (vp->v_type != VBLK || 193 (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) { 194 int updflags = 0; 195 196 if ((vp->v_mount->mnt_flag & MNT_NOATIME) == 0) { 197 updflags = GOP_UPDATE_ACCESSED; 198 } 199 if (memwrite) { 200 updflags |= GOP_UPDATE_MODIFIED; 201 } 202 if (updflags != 0) { 203 GOP_MARKUPDATE(vp, updflags); 204 } 205 } 206 207 /* 208 * For PGO_LOCKED requests, just return whatever's in memory. 209 */ 210 211 if (flags & PGO_LOCKED) { 212 int nfound; 213 struct vm_page *pg; 214 215 KASSERT(!glocked); 216 npages = *ap->a_count; 217 #if defined(DEBUG) 218 for (i = 0; i < npages; i++) { 219 pg = ap->a_m[i]; 220 KASSERT(pg == NULL || pg == PGO_DONTCARE); 221 } 222 #endif /* defined(DEBUG) */ 223 nfound = uvn_findpages(uobj, origoffset, &npages, 224 ap->a_m, UFP_NOWAIT|UFP_NOALLOC|(memwrite ? UFP_NORDONLY : 0)); 225 KASSERT(npages == *ap->a_count); 226 if (nfound == 0) { 227 error = EBUSY; 228 goto out_err; 229 } 230 if (!genfs_node_rdtrylock(vp)) { 231 genfs_rel_pages(ap->a_m, npages); 232 233 /* 234 * restore the array. 235 */ 236 237 for (i = 0; i < npages; i++) { 238 pg = ap->a_m[i]; 239 240 if (pg != NULL && pg != PGO_DONTCARE) { 241 ap->a_m[i] = NULL; 242 } 243 KASSERT(ap->a_m[i] == NULL || 244 ap->a_m[i] == PGO_DONTCARE); 245 } 246 } else { 247 genfs_node_unlock(vp); 248 } 249 error = (ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0); 250 if (error == 0 && memwrite) { 251 genfs_markdirty(vp); 252 } 253 goto out_err; 254 } 255 mutex_exit(&uobj->vmobjlock); 256 257 /* 258 * find the requested pages and make some simple checks. 259 * leave space in the page array for a whole block. 260 */ 261 262 const int fs_bshift = (vp->v_type != VBLK) ? 263 vp->v_mount->mnt_fs_bshift : DEV_BSHIFT; 264 const int dev_bshift = (vp->v_type != VBLK) ? 265 vp->v_mount->mnt_dev_bshift : DEV_BSHIFT; 266 const int fs_bsize = 1 << fs_bshift; 267 #define blk_mask (fs_bsize - 1) 268 #define trunc_blk(x) ((x) & ~blk_mask) 269 #define round_blk(x) (((x) + blk_mask) & ~blk_mask) 270 271 const int orignmempages = MIN(orignpages, 272 round_page(memeof - origoffset) >> PAGE_SHIFT); 273 npages = orignmempages; 274 const off_t startoffset = trunc_blk(origoffset); 275 const off_t endoffset = MIN( 276 round_page(round_blk(origoffset + (npages << PAGE_SHIFT))), 277 round_page(memeof)); 278 const int ridx = (origoffset - startoffset) >> PAGE_SHIFT; 279 280 const int pgs_size = sizeof(struct vm_page *) * 281 ((endoffset - startoffset) >> PAGE_SHIFT); 282 struct vm_page **pgs, *pgs_onstack[UBC_MAX_PAGES]; 283 284 if (pgs_size > sizeof(pgs_onstack)) { 285 pgs = kmem_zalloc(pgs_size, async ? KM_NOSLEEP : KM_SLEEP); 286 if (pgs == NULL) { 287 pgs = pgs_onstack; 288 error = ENOMEM; 289 goto out_err; 290 } 291 } else { 292 pgs = pgs_onstack; 293 (void)memset(pgs, 0, pgs_size); 294 } 295 296 UVMHIST_LOG(ubchist, "ridx %d npages %d startoff %ld endoff %ld", 297 ridx, npages, startoffset, endoffset); 298 299 if (!has_trans_wapbl) { 300 fstrans_start(vp->v_mount, FSTRANS_SHARED); 301 /* 302 * XXX: This assumes that we come here only via 303 * the mmio path 304 */ 305 if (need_wapbl) { 306 error = WAPBL_BEGIN(vp->v_mount); 307 if (error) { 308 fstrans_done(vp->v_mount); 309 goto out_err_free; 310 } 311 } 312 has_trans_wapbl = true; 313 } 314 315 /* 316 * hold g_glock to prevent a race with truncate. 317 * 318 * check if our idea of v_size is still valid. 319 */ 320 321 KASSERT(!glocked || genfs_node_wrlocked(vp)); 322 if (!glocked) { 323 if (blockalloc) { 324 genfs_node_wrlock(vp); 325 } else { 326 genfs_node_rdlock(vp); 327 } 328 } 329 mutex_enter(&uobj->vmobjlock); 330 if (vp->v_size < origvsize) { 331 if (!glocked) { 332 genfs_node_unlock(vp); 333 } 334 if (pgs != pgs_onstack) 335 kmem_free(pgs, pgs_size); 336 goto startover; 337 } 338 339 if (uvn_findpages(uobj, origoffset, &npages, &pgs[ridx], 340 async ? UFP_NOWAIT : UFP_ALL) != orignmempages) { 341 if (!glocked) { 342 genfs_node_unlock(vp); 343 } 344 KASSERT(async != 0); 345 genfs_rel_pages(&pgs[ridx], orignmempages); 346 mutex_exit(&uobj->vmobjlock); 347 error = EBUSY; 348 goto out_err_free; 349 } 350 351 /* 352 * if the pages are already resident, just return them. 353 */ 354 355 for (i = 0; i < npages; i++) { 356 struct vm_page *pg = pgs[ridx + i]; 357 358 if ((pg->flags & PG_FAKE) || 359 (blockalloc && (pg->flags & PG_RDONLY))) { 360 break; 361 } 362 } 363 if (i == npages) { 364 if (!glocked) { 365 genfs_node_unlock(vp); 366 } 367 UVMHIST_LOG(ubchist, "returning cached pages", 0,0,0,0); 368 npages += ridx; 369 goto out; 370 } 371 372 /* 373 * if PGO_OVERWRITE is set, don't bother reading the pages. 374 */ 375 376 if (overwrite) { 377 if (!glocked) { 378 genfs_node_unlock(vp); 379 } 380 UVMHIST_LOG(ubchist, "PGO_OVERWRITE",0,0,0,0); 381 382 for (i = 0; i < npages; i++) { 383 struct vm_page *pg = pgs[ridx + i]; 384 385 pg->flags &= ~(PG_RDONLY|PG_CLEAN); 386 } 387 npages += ridx; 388 goto out; 389 } 390 391 /* 392 * the page wasn't resident and we're not overwriting, 393 * so we're going to have to do some i/o. 394 * find any additional pages needed to cover the expanded range. 395 */ 396 397 npages = (endoffset - startoffset) >> PAGE_SHIFT; 398 if (startoffset != origoffset || npages != orignmempages) { 399 int npgs; 400 401 /* 402 * we need to avoid deadlocks caused by locking 403 * additional pages at lower offsets than pages we 404 * already have locked. unlock them all and start over. 405 */ 406 407 genfs_rel_pages(&pgs[ridx], orignmempages); 408 memset(pgs, 0, pgs_size); 409 410 UVMHIST_LOG(ubchist, "reset npages start 0x%x end 0x%x", 411 startoffset, endoffset, 0,0); 412 npgs = npages; 413 if (uvn_findpages(uobj, startoffset, &npgs, pgs, 414 async ? UFP_NOWAIT : UFP_ALL) != npages) { 415 if (!glocked) { 416 genfs_node_unlock(vp); 417 } 418 KASSERT(async != 0); 419 genfs_rel_pages(pgs, npages); 420 mutex_exit(&uobj->vmobjlock); 421 error = EBUSY; 422 goto out_err_free; 423 } 424 } 425 426 mutex_exit(&uobj->vmobjlock); 427 428 { 429 size_t bytes, iobytes, tailstart, tailbytes, totalbytes, skipbytes; 430 vaddr_t kva; 431 struct buf *bp, *mbp; 432 bool sawhole = false; 433 434 /* 435 * read the desired page(s). 436 */ 437 438 totalbytes = npages << PAGE_SHIFT; 439 bytes = MIN(totalbytes, MAX(diskeof - startoffset, 0)); 440 tailbytes = totalbytes - bytes; 441 skipbytes = 0; 442 443 kva = uvm_pagermapin(pgs, npages, 444 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 445 446 mbp = getiobuf(vp, true); 447 mbp->b_bufsize = totalbytes; 448 mbp->b_data = (void *)kva; 449 mbp->b_resid = mbp->b_bcount = bytes; 450 mbp->b_cflags = BC_BUSY; 451 if (async) { 452 mbp->b_flags = B_READ | B_ASYNC; 453 mbp->b_iodone = uvm_aio_biodone; 454 } else { 455 mbp->b_flags = B_READ; 456 mbp->b_iodone = NULL; 457 } 458 if (async) 459 BIO_SETPRIO(mbp, BPRIO_TIMELIMITED); 460 else 461 BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL); 462 463 /* 464 * if EOF is in the middle of the range, zero the part past EOF. 465 * skip over pages which are not PG_FAKE since in that case they have 466 * valid data that we need to preserve. 467 */ 468 469 tailstart = bytes; 470 while (tailbytes > 0) { 471 const int len = PAGE_SIZE - (tailstart & PAGE_MASK); 472 473 KASSERT(len <= tailbytes); 474 if ((pgs[tailstart >> PAGE_SHIFT]->flags & PG_FAKE) != 0) { 475 memset((void *)(kva + tailstart), 0, len); 476 UVMHIST_LOG(ubchist, "tailbytes %p 0x%x 0x%x", 477 kva, tailstart, len, 0); 478 } 479 tailstart += len; 480 tailbytes -= len; 481 } 482 483 /* 484 * now loop over the pages, reading as needed. 485 */ 486 487 bp = NULL; 488 off_t offset; 489 for (offset = startoffset; 490 bytes > 0; 491 offset += iobytes, bytes -= iobytes) { 492 int run; 493 daddr_t lbn, blkno; 494 int pidx; 495 struct vnode *devvp; 496 497 /* 498 * skip pages which don't need to be read. 499 */ 500 501 pidx = (offset - startoffset) >> PAGE_SHIFT; 502 while ((pgs[pidx]->flags & PG_FAKE) == 0) { 503 size_t b; 504 505 KASSERT((offset & (PAGE_SIZE - 1)) == 0); 506 if ((pgs[pidx]->flags & PG_RDONLY)) { 507 sawhole = true; 508 } 509 b = MIN(PAGE_SIZE, bytes); 510 offset += b; 511 bytes -= b; 512 skipbytes += b; 513 pidx++; 514 UVMHIST_LOG(ubchist, "skipping, new offset 0x%x", 515 offset, 0,0,0); 516 if (bytes == 0) { 517 goto loopdone; 518 } 519 } 520 521 /* 522 * bmap the file to find out the blkno to read from and 523 * how much we can read in one i/o. if bmap returns an error, 524 * skip the rest of the top-level i/o. 525 */ 526 527 lbn = offset >> fs_bshift; 528 error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run); 529 if (error) { 530 UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%x -> %d\n", 531 lbn,error,0,0); 532 skipbytes += bytes; 533 bytes = 0; 534 goto loopdone; 535 } 536 537 /* 538 * see how many pages can be read with this i/o. 539 * reduce the i/o size if necessary to avoid 540 * overwriting pages with valid data. 541 */ 542 543 iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset, 544 bytes); 545 if (offset + iobytes > round_page(offset)) { 546 int pcount; 547 548 pcount = 1; 549 while (pidx + pcount < npages && 550 pgs[pidx + pcount]->flags & PG_FAKE) { 551 pcount++; 552 } 553 iobytes = MIN(iobytes, (pcount << PAGE_SHIFT) - 554 (offset - trunc_page(offset))); 555 } 556 557 /* 558 * if this block isn't allocated, zero it instead of 559 * reading it. unless we are going to allocate blocks, 560 * mark the pages we zeroed PG_RDONLY. 561 */ 562 563 if (blkno == (daddr_t)-1) { 564 int holepages = (round_page(offset + iobytes) - 565 trunc_page(offset)) >> PAGE_SHIFT; 566 UVMHIST_LOG(ubchist, "lbn 0x%x -> HOLE", lbn,0,0,0); 567 568 sawhole = true; 569 memset((char *)kva + (offset - startoffset), 0, 570 iobytes); 571 skipbytes += iobytes; 572 573 mutex_enter(&uobj->vmobjlock); 574 for (i = 0; i < holepages; i++) { 575 if (memwrite) { 576 pgs[pidx + i]->flags &= ~PG_CLEAN; 577 } 578 if (!blockalloc) { 579 pgs[pidx + i]->flags |= PG_RDONLY; 580 } 581 } 582 mutex_exit(&uobj->vmobjlock); 583 continue; 584 } 585 586 /* 587 * allocate a sub-buf for this piece of the i/o 588 * (or just use mbp if there's only 1 piece), 589 * and start it going. 590 */ 591 592 if (offset == startoffset && iobytes == bytes) { 593 bp = mbp; 594 } else { 595 UVMHIST_LOG(ubchist, "vp %p bp %p num now %d", 596 vp, bp, vp->v_numoutput, 0); 597 bp = getiobuf(vp, true); 598 nestiobuf_setup(mbp, bp, offset - startoffset, iobytes); 599 } 600 bp->b_lblkno = 0; 601 602 /* adjust physical blkno for partial blocks */ 603 bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >> 604 dev_bshift); 605 606 UVMHIST_LOG(ubchist, 607 "bp %p offset 0x%x bcount 0x%x blkno 0x%x", 608 bp, offset, bp->b_bcount, bp->b_blkno); 609 610 VOP_STRATEGY(devvp, bp); 611 } 612 613 loopdone: 614 nestiobuf_done(mbp, skipbytes, error); 615 if (async) { 616 UVMHIST_LOG(ubchist, "returning 0 (async)",0,0,0,0); 617 if (!glocked) { 618 genfs_node_unlock(vp); 619 } 620 error = 0; 621 goto out_err_free; 622 } 623 if (bp != NULL) { 624 error = biowait(mbp); 625 } 626 627 /* Remove the mapping (make KVA available as soon as possible) */ 628 uvm_pagermapout(kva, npages); 629 630 /* 631 * if this we encountered a hole then we have to do a little more work. 632 * for read faults, we marked the page PG_RDONLY so that future 633 * write accesses to the page will fault again. 634 * for write faults, we must make sure that the backing store for 635 * the page is completely allocated while the pages are locked. 636 */ 637 638 if (!error && sawhole && blockalloc) { 639 error = GOP_ALLOC(vp, startoffset, 640 npages << PAGE_SHIFT, 0, cred); 641 UVMHIST_LOG(ubchist, "gop_alloc off 0x%x/0x%x -> %d", 642 startoffset, npages << PAGE_SHIFT, error,0); 643 if (!error) { 644 mutex_enter(&uobj->vmobjlock); 645 for (i = 0; i < npages; i++) { 646 struct vm_page *pg = pgs[i]; 647 648 if (pg == NULL) { 649 continue; 650 } 651 pg->flags &= ~(PG_CLEAN|PG_RDONLY); 652 UVMHIST_LOG(ubchist, "mark dirty pg %p", 653 pg,0,0,0); 654 } 655 mutex_exit(&uobj->vmobjlock); 656 } 657 } 658 if (!glocked) { 659 genfs_node_unlock(vp); 660 } 661 662 putiobuf(mbp); 663 } 664 665 mutex_enter(&uobj->vmobjlock); 666 667 /* 668 * we're almost done! release the pages... 669 * for errors, we free the pages. 670 * otherwise we activate them and mark them as valid and clean. 671 * also, unbusy pages that were not actually requested. 672 */ 673 674 if (error) { 675 for (i = 0; i < npages; i++) { 676 struct vm_page *pg = pgs[i]; 677 678 if (pg == NULL) { 679 continue; 680 } 681 UVMHIST_LOG(ubchist, "examining pg %p flags 0x%x", 682 pg, pg->flags, 0,0); 683 if (pg->flags & PG_FAKE) { 684 pg->flags |= PG_RELEASED; 685 } 686 } 687 mutex_enter(&uvm_pageqlock); 688 uvm_page_unbusy(pgs, npages); 689 mutex_exit(&uvm_pageqlock); 690 mutex_exit(&uobj->vmobjlock); 691 UVMHIST_LOG(ubchist, "returning error %d", error,0,0,0); 692 goto out_err_free; 693 } 694 695 out: 696 UVMHIST_LOG(ubchist, "succeeding, npages %d", npages,0,0,0); 697 error = 0; 698 mutex_enter(&uvm_pageqlock); 699 for (i = 0; i < npages; i++) { 700 struct vm_page *pg = pgs[i]; 701 if (pg == NULL) { 702 continue; 703 } 704 UVMHIST_LOG(ubchist, "examining pg %p flags 0x%x", 705 pg, pg->flags, 0,0); 706 if (pg->flags & PG_FAKE && !overwrite) { 707 pg->flags &= ~(PG_FAKE); 708 pmap_clear_modify(pgs[i]); 709 } 710 KASSERT(!memwrite || !blockalloc || (pg->flags & PG_RDONLY) == 0); 711 if (i < ridx || i >= ridx + orignmempages || async) { 712 UVMHIST_LOG(ubchist, "unbusy pg %p offset 0x%x", 713 pg, pg->offset,0,0); 714 if (pg->flags & PG_WANTED) { 715 wakeup(pg); 716 } 717 if (pg->flags & PG_FAKE) { 718 KASSERT(overwrite); 719 uvm_pagezero(pg); 720 } 721 if (pg->flags & PG_RELEASED) { 722 uvm_pagefree(pg); 723 continue; 724 } 725 uvm_pageenqueue(pg); 726 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE); 727 UVM_PAGE_OWN(pg, NULL); 728 } 729 } 730 mutex_exit(&uvm_pageqlock); 731 if (memwrite) { 732 genfs_markdirty(vp); 733 } 734 mutex_exit(&uobj->vmobjlock); 735 if (ap->a_m != NULL) { 736 memcpy(ap->a_m, &pgs[ridx], 737 orignmempages * sizeof(struct vm_page *)); 738 } 739 740 out_err_free: 741 if (pgs != NULL && pgs != pgs_onstack) 742 kmem_free(pgs, pgs_size); 743 out_err: 744 if (has_trans_wapbl) { 745 if (need_wapbl) 746 WAPBL_END(vp->v_mount); 747 fstrans_done(vp->v_mount); 748 } 749 return error; 750 } 751 752 /* 753 * generic VM putpages routine. 754 * Write the given range of pages to backing store. 755 * 756 * => "offhi == 0" means flush all pages at or after "offlo". 757 * => object should be locked by caller. we return with the 758 * object unlocked. 759 * => if PGO_CLEANIT or PGO_SYNCIO is set, we may block (due to I/O). 760 * thus, a caller might want to unlock higher level resources 761 * (e.g. vm_map) before calling flush. 762 * => if neither PGO_CLEANIT nor PGO_SYNCIO is set, we will not block 763 * => if PGO_ALLPAGES is set, then all pages in the object will be processed. 764 * => NOTE: we rely on the fact that the object's memq is a TAILQ and 765 * that new pages are inserted on the tail end of the list. thus, 766 * we can make a complete pass through the object in one go by starting 767 * at the head and working towards the tail (new pages are put in 768 * front of us). 769 * => NOTE: we are allowed to lock the page queues, so the caller 770 * must not be holding the page queue lock. 771 * 772 * note on "cleaning" object and PG_BUSY pages: 773 * this routine is holding the lock on the object. the only time 774 * that it can run into a PG_BUSY page that it does not own is if 775 * some other process has started I/O on the page (e.g. either 776 * a pagein, or a pageout). if the PG_BUSY page is being paged 777 * in, then it can not be dirty (!PG_CLEAN) because no one has 778 * had a chance to modify it yet. if the PG_BUSY page is being 779 * paged out then it means that someone else has already started 780 * cleaning the page for us (how nice!). in this case, if we 781 * have syncio specified, then after we make our pass through the 782 * object we need to wait for the other PG_BUSY pages to clear 783 * off (i.e. we need to do an iosync). also note that once a 784 * page is PG_BUSY it must stay in its object until it is un-busyed. 785 * 786 * note on page traversal: 787 * we can traverse the pages in an object either by going down the 788 * linked list in "uobj->memq", or we can go over the address range 789 * by page doing hash table lookups for each address. depending 790 * on how many pages are in the object it may be cheaper to do one 791 * or the other. we set "by_list" to true if we are using memq. 792 * if the cost of a hash lookup was equal to the cost of the list 793 * traversal we could compare the number of pages in the start->stop 794 * range to the total number of pages in the object. however, it 795 * seems that a hash table lookup is more expensive than the linked 796 * list traversal, so we multiply the number of pages in the 797 * range by an estimate of the relatively higher cost of the hash lookup. 798 */ 799 800 int 801 genfs_putpages(void *v) 802 { 803 struct vop_putpages_args /* { 804 struct vnode *a_vp; 805 voff_t a_offlo; 806 voff_t a_offhi; 807 int a_flags; 808 } */ * const ap = v; 809 810 return genfs_do_putpages(ap->a_vp, ap->a_offlo, ap->a_offhi, 811 ap->a_flags, NULL); 812 } 813 814 int 815 genfs_do_putpages(struct vnode *vp, off_t startoff, off_t endoff, 816 int origflags, struct vm_page **busypg) 817 { 818 struct uvm_object * const uobj = &vp->v_uobj; 819 kmutex_t * const slock = &uobj->vmobjlock; 820 off_t off; 821 /* Even for strange MAXPHYS, the shift rounds down to a page */ 822 #define maxpages (MAXPHYS >> PAGE_SHIFT) 823 int i, error, npages, nback; 824 int freeflag; 825 struct vm_page *pgs[maxpages], *pg, *nextpg, *tpg, curmp, endmp; 826 bool wasclean, by_list, needs_clean, yld; 827 bool async = (origflags & PGO_SYNCIO) == 0; 828 bool pagedaemon = curlwp == uvm.pagedaemon_lwp; 829 struct lwp * const l = curlwp ? curlwp : &lwp0; 830 struct genfs_node * const gp = VTOG(vp); 831 int flags; 832 int dirtygen; 833 bool modified; 834 bool need_wapbl; 835 bool has_trans; 836 bool cleanall; 837 bool onworklst; 838 839 UVMHIST_FUNC("genfs_putpages"); UVMHIST_CALLED(ubchist); 840 841 KASSERT(origflags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)); 842 KASSERT((startoff & PAGE_MASK) == 0 && (endoff & PAGE_MASK) == 0); 843 KASSERT(startoff < endoff || endoff == 0); 844 845 UVMHIST_LOG(ubchist, "vp %p pages %d off 0x%x len 0x%x", 846 vp, uobj->uo_npages, startoff, endoff - startoff); 847 848 has_trans = false; 849 need_wapbl = (!pagedaemon && vp->v_mount && vp->v_mount->mnt_wapbl && 850 (origflags & PGO_JOURNALLOCKED) == 0); 851 852 retry: 853 modified = false; 854 flags = origflags; 855 KASSERT((vp->v_iflag & VI_ONWORKLST) != 0 || 856 (vp->v_iflag & VI_WRMAPDIRTY) == 0); 857 if (uobj->uo_npages == 0) { 858 if (vp->v_iflag & VI_ONWORKLST) { 859 vp->v_iflag &= ~VI_WRMAPDIRTY; 860 if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL) 861 vn_syncer_remove_from_worklist(vp); 862 } 863 if (has_trans) { 864 if (need_wapbl) 865 WAPBL_END(vp->v_mount); 866 fstrans_done(vp->v_mount); 867 } 868 mutex_exit(slock); 869 return (0); 870 } 871 872 /* 873 * the vnode has pages, set up to process the request. 874 */ 875 876 if (!has_trans && (flags & PGO_CLEANIT) != 0) { 877 mutex_exit(slock); 878 if (pagedaemon) { 879 error = fstrans_start_nowait(vp->v_mount, FSTRANS_LAZY); 880 if (error) 881 return error; 882 } else 883 fstrans_start(vp->v_mount, FSTRANS_LAZY); 884 if (need_wapbl) { 885 error = WAPBL_BEGIN(vp->v_mount); 886 if (error) { 887 fstrans_done(vp->v_mount); 888 return error; 889 } 890 } 891 has_trans = true; 892 mutex_enter(slock); 893 goto retry; 894 } 895 896 error = 0; 897 wasclean = (vp->v_numoutput == 0); 898 off = startoff; 899 if (endoff == 0 || flags & PGO_ALLPAGES) { 900 endoff = trunc_page(LLONG_MAX); 901 } 902 by_list = (uobj->uo_npages <= 903 ((endoff - startoff) >> PAGE_SHIFT) * UVM_PAGE_TREE_PENALTY); 904 905 #if !defined(DEBUG) 906 /* 907 * if this vnode is known not to have dirty pages, 908 * don't bother to clean it out. 909 */ 910 911 if ((vp->v_iflag & VI_ONWORKLST) == 0) { 912 if ((flags & (PGO_FREE|PGO_DEACTIVATE)) == 0) { 913 goto skip_scan; 914 } 915 flags &= ~PGO_CLEANIT; 916 } 917 #endif /* !defined(DEBUG) */ 918 919 /* 920 * start the loop. when scanning by list, hold the last page 921 * in the list before we start. pages allocated after we start 922 * will be added to the end of the list, so we can stop at the 923 * current last page. 924 */ 925 926 cleanall = (flags & PGO_CLEANIT) != 0 && wasclean && 927 startoff == 0 && endoff == trunc_page(LLONG_MAX) && 928 (vp->v_iflag & VI_ONWORKLST) != 0; 929 dirtygen = gp->g_dirtygen; 930 freeflag = pagedaemon ? PG_PAGEOUT : PG_RELEASED; 931 if (by_list) { 932 curmp.flags = PG_MARKER; 933 endmp.flags = PG_MARKER; 934 pg = TAILQ_FIRST(&uobj->memq); 935 TAILQ_INSERT_TAIL(&uobj->memq, &endmp, listq.queue); 936 } else { 937 pg = uvm_pagelookup(uobj, off); 938 } 939 nextpg = NULL; 940 while (by_list || off < endoff) { 941 942 /* 943 * if the current page is not interesting, move on to the next. 944 */ 945 946 KASSERT(pg == NULL || pg->uobject == uobj || 947 (pg->flags & PG_MARKER) != 0); 948 KASSERT(pg == NULL || 949 (pg->flags & (PG_RELEASED|PG_PAGEOUT)) == 0 || 950 (pg->flags & (PG_BUSY|PG_MARKER)) != 0); 951 if (by_list) { 952 if (pg == &endmp) { 953 break; 954 } 955 if (pg->flags & PG_MARKER) { 956 pg = TAILQ_NEXT(pg, listq.queue); 957 continue; 958 } 959 if (pg->offset < startoff || pg->offset >= endoff || 960 pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 961 if (pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 962 wasclean = false; 963 } 964 pg = TAILQ_NEXT(pg, listq.queue); 965 continue; 966 } 967 off = pg->offset; 968 } else if (pg == NULL || pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 969 if (pg != NULL) { 970 wasclean = false; 971 } 972 off += PAGE_SIZE; 973 if (off < endoff) { 974 pg = uvm_pagelookup(uobj, off); 975 } 976 continue; 977 } 978 979 /* 980 * if the current page needs to be cleaned and it's busy, 981 * wait for it to become unbusy. 982 */ 983 984 yld = (l->l_cpu->ci_schedstate.spc_flags & 985 SPCF_SHOULDYIELD) && !pagedaemon; 986 if (pg->flags & PG_BUSY || yld) { 987 UVMHIST_LOG(ubchist, "busy %p", pg,0,0,0); 988 if (flags & PGO_BUSYFAIL && pg->flags & PG_BUSY) { 989 UVMHIST_LOG(ubchist, "busyfail %p", pg, 0,0,0); 990 error = EDEADLK; 991 if (busypg != NULL) 992 *busypg = pg; 993 break; 994 } 995 if (pagedaemon) { 996 /* 997 * someone has taken the page while we 998 * dropped the lock for fstrans_start. 999 */ 1000 break; 1001 } 1002 if (by_list) { 1003 TAILQ_INSERT_BEFORE(pg, &curmp, listq.queue); 1004 UVMHIST_LOG(ubchist, "curmp next %p", 1005 TAILQ_NEXT(&curmp, listq.queue), 0,0,0); 1006 } 1007 if (yld) { 1008 mutex_exit(slock); 1009 preempt(); 1010 mutex_enter(slock); 1011 } else { 1012 pg->flags |= PG_WANTED; 1013 UVM_UNLOCK_AND_WAIT(pg, slock, 0, "genput", 0); 1014 mutex_enter(slock); 1015 } 1016 if (by_list) { 1017 UVMHIST_LOG(ubchist, "after next %p", 1018 TAILQ_NEXT(&curmp, listq.queue), 0,0,0); 1019 pg = TAILQ_NEXT(&curmp, listq.queue); 1020 TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue); 1021 } else { 1022 pg = uvm_pagelookup(uobj, off); 1023 } 1024 continue; 1025 } 1026 1027 /* 1028 * if we're freeing, remove all mappings of the page now. 1029 * if we're cleaning, check if the page is needs to be cleaned. 1030 */ 1031 1032 if (flags & PGO_FREE) { 1033 pmap_page_protect(pg, VM_PROT_NONE); 1034 } else if (flags & PGO_CLEANIT) { 1035 1036 /* 1037 * if we still have some hope to pull this vnode off 1038 * from the syncer queue, write-protect the page. 1039 */ 1040 1041 if (cleanall && wasclean && 1042 gp->g_dirtygen == dirtygen) { 1043 1044 /* 1045 * uobj pages get wired only by uvm_fault 1046 * where uobj is locked. 1047 */ 1048 1049 if (pg->wire_count == 0) { 1050 pmap_page_protect(pg, 1051 VM_PROT_READ|VM_PROT_EXECUTE); 1052 } else { 1053 cleanall = false; 1054 } 1055 } 1056 } 1057 1058 if (flags & PGO_CLEANIT) { 1059 needs_clean = pmap_clear_modify(pg) || 1060 (pg->flags & PG_CLEAN) == 0; 1061 pg->flags |= PG_CLEAN; 1062 } else { 1063 needs_clean = false; 1064 } 1065 1066 /* 1067 * if we're cleaning, build a cluster. 1068 * the cluster will consist of pages which are currently dirty, 1069 * but they will be returned to us marked clean. 1070 * if not cleaning, just operate on the one page. 1071 */ 1072 1073 if (needs_clean) { 1074 KDASSERT((vp->v_iflag & VI_ONWORKLST)); 1075 wasclean = false; 1076 memset(pgs, 0, sizeof(pgs)); 1077 pg->flags |= PG_BUSY; 1078 UVM_PAGE_OWN(pg, "genfs_putpages"); 1079 1080 /* 1081 * first look backward. 1082 */ 1083 1084 npages = MIN(maxpages >> 1, off >> PAGE_SHIFT); 1085 nback = npages; 1086 uvn_findpages(uobj, off - PAGE_SIZE, &nback, &pgs[0], 1087 UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY|UFP_BACKWARD); 1088 if (nback) { 1089 memmove(&pgs[0], &pgs[npages - nback], 1090 nback * sizeof(pgs[0])); 1091 if (npages - nback < nback) 1092 memset(&pgs[nback], 0, 1093 (npages - nback) * sizeof(pgs[0])); 1094 else 1095 memset(&pgs[npages - nback], 0, 1096 nback * sizeof(pgs[0])); 1097 } 1098 1099 /* 1100 * then plug in our page of interest. 1101 */ 1102 1103 pgs[nback] = pg; 1104 1105 /* 1106 * then look forward to fill in the remaining space in 1107 * the array of pages. 1108 */ 1109 1110 npages = maxpages - nback - 1; 1111 uvn_findpages(uobj, off + PAGE_SIZE, &npages, 1112 &pgs[nback + 1], 1113 UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY); 1114 npages += nback + 1; 1115 } else { 1116 pgs[0] = pg; 1117 npages = 1; 1118 nback = 0; 1119 } 1120 1121 /* 1122 * apply FREE or DEACTIVATE options if requested. 1123 */ 1124 1125 if (flags & (PGO_DEACTIVATE|PGO_FREE)) { 1126 mutex_enter(&uvm_pageqlock); 1127 } 1128 for (i = 0; i < npages; i++) { 1129 tpg = pgs[i]; 1130 KASSERT(tpg->uobject == uobj); 1131 if (by_list && tpg == TAILQ_NEXT(pg, listq.queue)) 1132 pg = tpg; 1133 if (tpg->offset < startoff || tpg->offset >= endoff) 1134 continue; 1135 if (flags & PGO_DEACTIVATE && tpg->wire_count == 0) { 1136 uvm_pagedeactivate(tpg); 1137 } else if (flags & PGO_FREE) { 1138 pmap_page_protect(tpg, VM_PROT_NONE); 1139 if (tpg->flags & PG_BUSY) { 1140 tpg->flags |= freeflag; 1141 if (pagedaemon) { 1142 uvm_pageout_start(1); 1143 uvm_pagedequeue(tpg); 1144 } 1145 } else { 1146 1147 /* 1148 * ``page is not busy'' 1149 * implies that npages is 1 1150 * and needs_clean is false. 1151 */ 1152 1153 nextpg = TAILQ_NEXT(tpg, listq.queue); 1154 uvm_pagefree(tpg); 1155 if (pagedaemon) 1156 uvmexp.pdfreed++; 1157 } 1158 } 1159 } 1160 if (flags & (PGO_DEACTIVATE|PGO_FREE)) { 1161 mutex_exit(&uvm_pageqlock); 1162 } 1163 if (needs_clean) { 1164 modified = true; 1165 1166 /* 1167 * start the i/o. if we're traversing by list, 1168 * keep our place in the list with a marker page. 1169 */ 1170 1171 if (by_list) { 1172 TAILQ_INSERT_AFTER(&uobj->memq, pg, &curmp, 1173 listq.queue); 1174 } 1175 mutex_exit(slock); 1176 error = GOP_WRITE(vp, pgs, npages, flags); 1177 mutex_enter(slock); 1178 if (by_list) { 1179 pg = TAILQ_NEXT(&curmp, listq.queue); 1180 TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue); 1181 } 1182 if (error) { 1183 break; 1184 } 1185 if (by_list) { 1186 continue; 1187 } 1188 } 1189 1190 /* 1191 * find the next page and continue if there was no error. 1192 */ 1193 1194 if (by_list) { 1195 if (nextpg) { 1196 pg = nextpg; 1197 nextpg = NULL; 1198 } else { 1199 pg = TAILQ_NEXT(pg, listq.queue); 1200 } 1201 } else { 1202 off += (npages - nback) << PAGE_SHIFT; 1203 if (off < endoff) { 1204 pg = uvm_pagelookup(uobj, off); 1205 } 1206 } 1207 } 1208 if (by_list) { 1209 TAILQ_REMOVE(&uobj->memq, &endmp, listq.queue); 1210 } 1211 1212 if (modified && (vp->v_iflag & VI_WRMAPDIRTY) != 0 && 1213 (vp->v_type != VBLK || 1214 (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) { 1215 GOP_MARKUPDATE(vp, GOP_UPDATE_MODIFIED); 1216 } 1217 1218 /* 1219 * if we're cleaning and there was nothing to clean, 1220 * take us off the syncer list. if we started any i/o 1221 * and we're doing sync i/o, wait for all writes to finish. 1222 */ 1223 1224 if (cleanall && wasclean && gp->g_dirtygen == dirtygen && 1225 (vp->v_iflag & VI_ONWORKLST) != 0) { 1226 #if defined(DEBUG) 1227 TAILQ_FOREACH(pg, &uobj->memq, listq.queue) { 1228 if ((pg->flags & (PG_FAKE | PG_MARKER)) != 0) { 1229 continue; 1230 } 1231 if ((pg->flags & PG_CLEAN) == 0) { 1232 printf("%s: %p: !CLEAN\n", __func__, pg); 1233 } 1234 if (pmap_is_modified(pg)) { 1235 printf("%s: %p: modified\n", __func__, pg); 1236 } 1237 } 1238 #endif /* defined(DEBUG) */ 1239 vp->v_iflag &= ~VI_WRMAPDIRTY; 1240 if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL) 1241 vn_syncer_remove_from_worklist(vp); 1242 } 1243 1244 #if !defined(DEBUG) 1245 skip_scan: 1246 #endif /* !defined(DEBUG) */ 1247 1248 /* Wait for output to complete. */ 1249 if (!wasclean && !async && vp->v_numoutput != 0) { 1250 while (vp->v_numoutput != 0) 1251 cv_wait(&vp->v_cv, slock); 1252 } 1253 onworklst = (vp->v_iflag & VI_ONWORKLST) != 0; 1254 mutex_exit(slock); 1255 1256 if ((flags & PGO_RECLAIM) != 0 && onworklst) { 1257 /* 1258 * in the case of PGO_RECLAIM, ensure to make the vnode clean. 1259 * retrying is not a big deal because, in many cases, 1260 * uobj->uo_npages is already 0 here. 1261 */ 1262 mutex_enter(slock); 1263 goto retry; 1264 } 1265 1266 if (has_trans) { 1267 if (need_wapbl) 1268 WAPBL_END(vp->v_mount); 1269 fstrans_done(vp->v_mount); 1270 } 1271 1272 return (error); 1273 } 1274 1275 int 1276 genfs_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, int flags) 1277 { 1278 off_t off; 1279 vaddr_t kva; 1280 size_t len; 1281 int error; 1282 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1283 1284 UVMHIST_LOG(ubchist, "vp %p pgs %p npages %d flags 0x%x", 1285 vp, pgs, npages, flags); 1286 1287 off = pgs[0]->offset; 1288 kva = uvm_pagermapin(pgs, npages, 1289 UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK); 1290 len = npages << PAGE_SHIFT; 1291 1292 error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE, 1293 uvm_aio_biodone); 1294 1295 return error; 1296 } 1297 1298 int 1299 genfs_gop_write_rwmap(struct vnode *vp, struct vm_page **pgs, int npages, int flags) 1300 { 1301 off_t off; 1302 vaddr_t kva; 1303 size_t len; 1304 int error; 1305 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1306 1307 UVMHIST_LOG(ubchist, "vp %p pgs %p npages %d flags 0x%x", 1308 vp, pgs, npages, flags); 1309 1310 off = pgs[0]->offset; 1311 kva = uvm_pagermapin(pgs, npages, 1312 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 1313 len = npages << PAGE_SHIFT; 1314 1315 error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE, 1316 uvm_aio_biodone); 1317 1318 return error; 1319 } 1320 1321 /* 1322 * Backend routine for doing I/O to vnode pages. Pages are already locked 1323 * and mapped into kernel memory. Here we just look up the underlying 1324 * device block addresses and call the strategy routine. 1325 */ 1326 1327 static int 1328 genfs_do_io(struct vnode *vp, off_t off, vaddr_t kva, size_t len, int flags, 1329 enum uio_rw rw, void (*iodone)(struct buf *)) 1330 { 1331 int s, error; 1332 int fs_bshift, dev_bshift; 1333 off_t eof, offset, startoffset; 1334 size_t bytes, iobytes, skipbytes; 1335 struct buf *mbp, *bp; 1336 const bool async = (flags & PGO_SYNCIO) == 0; 1337 const bool iowrite = rw == UIO_WRITE; 1338 const int brw = iowrite ? B_WRITE : B_READ; 1339 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1340 1341 UVMHIST_LOG(ubchist, "vp %p kva %p len 0x%x flags 0x%x", 1342 vp, kva, len, flags); 1343 1344 KASSERT(vp->v_size <= vp->v_writesize); 1345 GOP_SIZE(vp, vp->v_writesize, &eof, 0); 1346 if (vp->v_type != VBLK) { 1347 fs_bshift = vp->v_mount->mnt_fs_bshift; 1348 dev_bshift = vp->v_mount->mnt_dev_bshift; 1349 } else { 1350 fs_bshift = DEV_BSHIFT; 1351 dev_bshift = DEV_BSHIFT; 1352 } 1353 error = 0; 1354 startoffset = off; 1355 bytes = MIN(len, eof - startoffset); 1356 skipbytes = 0; 1357 KASSERT(bytes != 0); 1358 1359 if (iowrite) { 1360 mutex_enter(&vp->v_interlock); 1361 vp->v_numoutput += 2; 1362 mutex_exit(&vp->v_interlock); 1363 } 1364 mbp = getiobuf(vp, true); 1365 UVMHIST_LOG(ubchist, "vp %p mbp %p num now %d bytes 0x%x", 1366 vp, mbp, vp->v_numoutput, bytes); 1367 mbp->b_bufsize = len; 1368 mbp->b_data = (void *)kva; 1369 mbp->b_resid = mbp->b_bcount = bytes; 1370 mbp->b_cflags = BC_BUSY | BC_AGE; 1371 if (async) { 1372 mbp->b_flags = brw | B_ASYNC; 1373 mbp->b_iodone = iodone; 1374 } else { 1375 mbp->b_flags = brw; 1376 mbp->b_iodone = NULL; 1377 } 1378 if (curlwp == uvm.pagedaemon_lwp) 1379 BIO_SETPRIO(mbp, BPRIO_TIMELIMITED); 1380 else if (async) 1381 BIO_SETPRIO(mbp, BPRIO_TIMENONCRITICAL); 1382 else 1383 BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL); 1384 1385 bp = NULL; 1386 for (offset = startoffset; 1387 bytes > 0; 1388 offset += iobytes, bytes -= iobytes) { 1389 int run; 1390 daddr_t lbn, blkno; 1391 struct vnode *devvp; 1392 1393 /* 1394 * bmap the file to find out the blkno to read from and 1395 * how much we can read in one i/o. if bmap returns an error, 1396 * skip the rest of the top-level i/o. 1397 */ 1398 1399 lbn = offset >> fs_bshift; 1400 error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run); 1401 if (error) { 1402 UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%x -> %d\n", 1403 lbn,error,0,0); 1404 skipbytes += bytes; 1405 bytes = 0; 1406 goto loopdone; 1407 } 1408 1409 /* 1410 * see how many pages can be read with this i/o. 1411 * reduce the i/o size if necessary to avoid 1412 * overwriting pages with valid data. 1413 */ 1414 1415 iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset, 1416 bytes); 1417 1418 /* 1419 * if this block isn't allocated, zero it instead of 1420 * reading it. unless we are going to allocate blocks, 1421 * mark the pages we zeroed PG_RDONLY. 1422 */ 1423 1424 if (blkno == (daddr_t)-1) { 1425 if (!iowrite) { 1426 memset((char *)kva + (offset - startoffset), 0, 1427 iobytes); 1428 } 1429 skipbytes += iobytes; 1430 continue; 1431 } 1432 1433 /* 1434 * allocate a sub-buf for this piece of the i/o 1435 * (or just use mbp if there's only 1 piece), 1436 * and start it going. 1437 */ 1438 1439 if (offset == startoffset && iobytes == bytes) { 1440 bp = mbp; 1441 } else { 1442 UVMHIST_LOG(ubchist, "vp %p bp %p num now %d", 1443 vp, bp, vp->v_numoutput, 0); 1444 bp = getiobuf(vp, true); 1445 nestiobuf_setup(mbp, bp, offset - startoffset, iobytes); 1446 } 1447 bp->b_lblkno = 0; 1448 1449 /* adjust physical blkno for partial blocks */ 1450 bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >> 1451 dev_bshift); 1452 1453 UVMHIST_LOG(ubchist, 1454 "bp %p offset 0x%x bcount 0x%x blkno 0x%x", 1455 bp, offset, bp->b_bcount, bp->b_blkno); 1456 1457 VOP_STRATEGY(devvp, bp); 1458 } 1459 1460 loopdone: 1461 if (skipbytes) { 1462 UVMHIST_LOG(ubchist, "skipbytes %d", skipbytes, 0,0,0); 1463 } 1464 nestiobuf_done(mbp, skipbytes, error); 1465 if (async) { 1466 UVMHIST_LOG(ubchist, "returning 0 (async)", 0,0,0,0); 1467 return (0); 1468 } 1469 UVMHIST_LOG(ubchist, "waiting for mbp %p", mbp,0,0,0); 1470 error = biowait(mbp); 1471 s = splbio(); 1472 (*iodone)(mbp); 1473 splx(s); 1474 UVMHIST_LOG(ubchist, "returning, error %d", error,0,0,0); 1475 return (error); 1476 } 1477 1478 int 1479 genfs_compat_getpages(void *v) 1480 { 1481 struct vop_getpages_args /* { 1482 struct vnode *a_vp; 1483 voff_t a_offset; 1484 struct vm_page **a_m; 1485 int *a_count; 1486 int a_centeridx; 1487 vm_prot_t a_access_type; 1488 int a_advice; 1489 int a_flags; 1490 } */ *ap = v; 1491 1492 off_t origoffset; 1493 struct vnode *vp = ap->a_vp; 1494 struct uvm_object *uobj = &vp->v_uobj; 1495 struct vm_page *pg, **pgs; 1496 vaddr_t kva; 1497 int i, error, orignpages, npages; 1498 struct iovec iov; 1499 struct uio uio; 1500 kauth_cred_t cred = curlwp->l_cred; 1501 const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0; 1502 1503 error = 0; 1504 origoffset = ap->a_offset; 1505 orignpages = *ap->a_count; 1506 pgs = ap->a_m; 1507 1508 if (ap->a_flags & PGO_LOCKED) { 1509 uvn_findpages(uobj, origoffset, ap->a_count, ap->a_m, 1510 UFP_NOWAIT|UFP_NOALLOC| (memwrite ? UFP_NORDONLY : 0)); 1511 1512 error = ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0; 1513 if (error == 0 && memwrite) { 1514 genfs_markdirty(vp); 1515 } 1516 return error; 1517 } 1518 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= vp->v_size) { 1519 mutex_exit(&uobj->vmobjlock); 1520 return EINVAL; 1521 } 1522 if ((ap->a_flags & PGO_SYNCIO) == 0) { 1523 mutex_exit(&uobj->vmobjlock); 1524 return 0; 1525 } 1526 npages = orignpages; 1527 uvn_findpages(uobj, origoffset, &npages, pgs, UFP_ALL); 1528 mutex_exit(&uobj->vmobjlock); 1529 kva = uvm_pagermapin(pgs, npages, 1530 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 1531 for (i = 0; i < npages; i++) { 1532 pg = pgs[i]; 1533 if ((pg->flags & PG_FAKE) == 0) { 1534 continue; 1535 } 1536 iov.iov_base = (char *)kva + (i << PAGE_SHIFT); 1537 iov.iov_len = PAGE_SIZE; 1538 uio.uio_iov = &iov; 1539 uio.uio_iovcnt = 1; 1540 uio.uio_offset = origoffset + (i << PAGE_SHIFT); 1541 uio.uio_rw = UIO_READ; 1542 uio.uio_resid = PAGE_SIZE; 1543 UIO_SETUP_SYSSPACE(&uio); 1544 /* XXX vn_lock */ 1545 error = VOP_READ(vp, &uio, 0, cred); 1546 if (error) { 1547 break; 1548 } 1549 if (uio.uio_resid) { 1550 memset(iov.iov_base, 0, uio.uio_resid); 1551 } 1552 } 1553 uvm_pagermapout(kva, npages); 1554 mutex_enter(&uobj->vmobjlock); 1555 mutex_enter(&uvm_pageqlock); 1556 for (i = 0; i < npages; i++) { 1557 pg = pgs[i]; 1558 if (error && (pg->flags & PG_FAKE) != 0) { 1559 pg->flags |= PG_RELEASED; 1560 } else { 1561 pmap_clear_modify(pg); 1562 uvm_pageactivate(pg); 1563 } 1564 } 1565 if (error) { 1566 uvm_page_unbusy(pgs, npages); 1567 } 1568 mutex_exit(&uvm_pageqlock); 1569 if (error == 0 && memwrite) { 1570 genfs_markdirty(vp); 1571 } 1572 mutex_exit(&uobj->vmobjlock); 1573 return error; 1574 } 1575 1576 int 1577 genfs_compat_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, 1578 int flags) 1579 { 1580 off_t offset; 1581 struct iovec iov; 1582 struct uio uio; 1583 kauth_cred_t cred = curlwp->l_cred; 1584 struct buf *bp; 1585 vaddr_t kva; 1586 int error; 1587 1588 offset = pgs[0]->offset; 1589 kva = uvm_pagermapin(pgs, npages, 1590 UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK); 1591 1592 iov.iov_base = (void *)kva; 1593 iov.iov_len = npages << PAGE_SHIFT; 1594 uio.uio_iov = &iov; 1595 uio.uio_iovcnt = 1; 1596 uio.uio_offset = offset; 1597 uio.uio_rw = UIO_WRITE; 1598 uio.uio_resid = npages << PAGE_SHIFT; 1599 UIO_SETUP_SYSSPACE(&uio); 1600 /* XXX vn_lock */ 1601 error = VOP_WRITE(vp, &uio, 0, cred); 1602 1603 mutex_enter(&vp->v_interlock); 1604 vp->v_numoutput++; 1605 mutex_exit(&vp->v_interlock); 1606 1607 bp = getiobuf(vp, true); 1608 bp->b_cflags = BC_BUSY | BC_AGE; 1609 bp->b_lblkno = offset >> vp->v_mount->mnt_fs_bshift; 1610 bp->b_data = (char *)kva; 1611 bp->b_bcount = npages << PAGE_SHIFT; 1612 bp->b_bufsize = npages << PAGE_SHIFT; 1613 bp->b_resid = 0; 1614 bp->b_error = error; 1615 uvm_aio_aiodone(bp); 1616 return (error); 1617 } 1618 1619 /* 1620 * Process a uio using direct I/O. If we reach a part of the request 1621 * which cannot be processed in this fashion for some reason, just return. 1622 * The caller must handle some additional part of the request using 1623 * buffered I/O before trying direct I/O again. 1624 */ 1625 1626 void 1627 genfs_directio(struct vnode *vp, struct uio *uio, int ioflag) 1628 { 1629 struct vmspace *vs; 1630 struct iovec *iov; 1631 vaddr_t va; 1632 size_t len; 1633 const int mask = DEV_BSIZE - 1; 1634 int error; 1635 bool need_wapbl = (vp->v_mount && vp->v_mount->mnt_wapbl && 1636 (ioflag & IO_JOURNALLOCKED) == 0); 1637 1638 /* 1639 * We only support direct I/O to user space for now. 1640 */ 1641 1642 if (VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) { 1643 return; 1644 } 1645 1646 /* 1647 * If the vnode is mapped, we would need to get the getpages lock 1648 * to stabilize the bmap, but then we would get into trouble whil e 1649 * locking the pages if the pages belong to this same vnode (or a 1650 * multi-vnode cascade to the same effect). Just fall back to 1651 * buffered I/O if the vnode is mapped to avoid this mess. 1652 */ 1653 1654 if (vp->v_vflag & VV_MAPPED) { 1655 return; 1656 } 1657 1658 if (need_wapbl) { 1659 error = WAPBL_BEGIN(vp->v_mount); 1660 if (error) 1661 return; 1662 } 1663 1664 /* 1665 * Do as much of the uio as possible with direct I/O. 1666 */ 1667 1668 vs = uio->uio_vmspace; 1669 while (uio->uio_resid) { 1670 iov = uio->uio_iov; 1671 if (iov->iov_len == 0) { 1672 uio->uio_iov++; 1673 uio->uio_iovcnt--; 1674 continue; 1675 } 1676 va = (vaddr_t)iov->iov_base; 1677 len = MIN(iov->iov_len, genfs_maxdio); 1678 len &= ~mask; 1679 1680 /* 1681 * If the next chunk is smaller than DEV_BSIZE or extends past 1682 * the current EOF, then fall back to buffered I/O. 1683 */ 1684 1685 if (len == 0 || uio->uio_offset + len > vp->v_size) { 1686 break; 1687 } 1688 1689 /* 1690 * Check alignment. The file offset must be at least 1691 * sector-aligned. The exact constraint on memory alignment 1692 * is very hardware-dependent, but requiring sector-aligned 1693 * addresses there too is safe. 1694 */ 1695 1696 if (uio->uio_offset & mask || va & mask) { 1697 break; 1698 } 1699 error = genfs_do_directio(vs, va, len, vp, uio->uio_offset, 1700 uio->uio_rw); 1701 if (error) { 1702 break; 1703 } 1704 iov->iov_base = (char *)iov->iov_base + len; 1705 iov->iov_len -= len; 1706 uio->uio_offset += len; 1707 uio->uio_resid -= len; 1708 } 1709 1710 if (need_wapbl) 1711 WAPBL_END(vp->v_mount); 1712 } 1713 1714 /* 1715 * Iodone routine for direct I/O. We don't do much here since the request is 1716 * always synchronous, so the caller will do most of the work after biowait(). 1717 */ 1718 1719 static void 1720 genfs_dio_iodone(struct buf *bp) 1721 { 1722 1723 KASSERT((bp->b_flags & B_ASYNC) == 0); 1724 if ((bp->b_flags & B_READ) == 0 && (bp->b_cflags & BC_AGE) != 0) { 1725 mutex_enter(bp->b_objlock); 1726 vwakeup(bp); 1727 mutex_exit(bp->b_objlock); 1728 } 1729 putiobuf(bp); 1730 } 1731 1732 /* 1733 * Process one chunk of a direct I/O request. 1734 */ 1735 1736 static int 1737 genfs_do_directio(struct vmspace *vs, vaddr_t uva, size_t len, struct vnode *vp, 1738 off_t off, enum uio_rw rw) 1739 { 1740 struct vm_map *map; 1741 struct pmap *upm, *kpm; 1742 size_t klen = round_page(uva + len) - trunc_page(uva); 1743 off_t spoff, epoff; 1744 vaddr_t kva, puva; 1745 paddr_t pa; 1746 vm_prot_t prot; 1747 int error, rv, poff, koff; 1748 const int pgoflags = PGO_CLEANIT | PGO_SYNCIO | PGO_JOURNALLOCKED | 1749 (rw == UIO_WRITE ? PGO_FREE : 0); 1750 1751 /* 1752 * For writes, verify that this range of the file already has fully 1753 * allocated backing store. If there are any holes, just punt and 1754 * make the caller take the buffered write path. 1755 */ 1756 1757 if (rw == UIO_WRITE) { 1758 daddr_t lbn, elbn, blkno; 1759 int bsize, bshift, run; 1760 1761 bshift = vp->v_mount->mnt_fs_bshift; 1762 bsize = 1 << bshift; 1763 lbn = off >> bshift; 1764 elbn = (off + len + bsize - 1) >> bshift; 1765 while (lbn < elbn) { 1766 error = VOP_BMAP(vp, lbn, NULL, &blkno, &run); 1767 if (error) { 1768 return error; 1769 } 1770 if (blkno == (daddr_t)-1) { 1771 return ENOSPC; 1772 } 1773 lbn += 1 + run; 1774 } 1775 } 1776 1777 /* 1778 * Flush any cached pages for parts of the file that we're about to 1779 * access. If we're writing, invalidate pages as well. 1780 */ 1781 1782 spoff = trunc_page(off); 1783 epoff = round_page(off + len); 1784 mutex_enter(&vp->v_interlock); 1785 error = VOP_PUTPAGES(vp, spoff, epoff, pgoflags); 1786 if (error) { 1787 return error; 1788 } 1789 1790 /* 1791 * Wire the user pages and remap them into kernel memory. 1792 */ 1793 1794 prot = rw == UIO_READ ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_READ; 1795 error = uvm_vslock(vs, (void *)uva, len, prot); 1796 if (error) { 1797 return error; 1798 } 1799 1800 map = &vs->vm_map; 1801 upm = vm_map_pmap(map); 1802 kpm = vm_map_pmap(kernel_map); 1803 kva = uvm_km_alloc(kernel_map, klen, 0, 1804 UVM_KMF_VAONLY | UVM_KMF_WAITVA); 1805 puva = trunc_page(uva); 1806 for (poff = 0; poff < klen; poff += PAGE_SIZE) { 1807 rv = pmap_extract(upm, puva + poff, &pa); 1808 KASSERT(rv); 1809 pmap_enter(kpm, kva + poff, pa, prot, prot | PMAP_WIRED); 1810 } 1811 pmap_update(kpm); 1812 1813 /* 1814 * Do the I/O. 1815 */ 1816 1817 koff = uva - trunc_page(uva); 1818 error = genfs_do_io(vp, off, kva + koff, len, PGO_SYNCIO, rw, 1819 genfs_dio_iodone); 1820 1821 /* 1822 * Tear down the kernel mapping. 1823 */ 1824 1825 pmap_remove(kpm, kva, kva + klen); 1826 pmap_update(kpm); 1827 uvm_km_free(kernel_map, kva, klen, UVM_KMF_VAONLY); 1828 1829 /* 1830 * Unwire the user pages. 1831 */ 1832 1833 uvm_vsunlock(vs, (void *)uva, len); 1834 return error; 1835 } 1836 1837