1 /* $NetBSD: genfs_io.c,v 1.40 2010/09/01 16:56:19 chs Exp $ */ 2 3 /* 4 * Copyright (c) 1982, 1986, 1989, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the University nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 */ 32 33 #include <sys/cdefs.h> 34 __KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.40 2010/09/01 16:56:19 chs Exp $"); 35 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/proc.h> 39 #include <sys/kernel.h> 40 #include <sys/mount.h> 41 #include <sys/namei.h> 42 #include <sys/vnode.h> 43 #include <sys/fcntl.h> 44 #include <sys/kmem.h> 45 #include <sys/poll.h> 46 #include <sys/mman.h> 47 #include <sys/file.h> 48 #include <sys/kauth.h> 49 #include <sys/fstrans.h> 50 #include <sys/buf.h> 51 52 #include <miscfs/genfs/genfs.h> 53 #include <miscfs/genfs/genfs_node.h> 54 #include <miscfs/specfs/specdev.h> 55 56 #include <uvm/uvm.h> 57 #include <uvm/uvm_pager.h> 58 59 static int genfs_do_directio(struct vmspace *, vaddr_t, size_t, struct vnode *, 60 off_t, enum uio_rw); 61 static void genfs_dio_iodone(struct buf *); 62 63 static int genfs_do_io(struct vnode *, off_t, vaddr_t, size_t, int, enum uio_rw, 64 void (*)(struct buf *)); 65 static void genfs_rel_pages(struct vm_page **, int); 66 static void genfs_markdirty(struct vnode *); 67 68 int genfs_maxdio = MAXPHYS; 69 70 static void 71 genfs_rel_pages(struct vm_page **pgs, int npages) 72 { 73 int i; 74 75 for (i = 0; i < npages; i++) { 76 struct vm_page *pg = pgs[i]; 77 78 if (pg == NULL || pg == PGO_DONTCARE) 79 continue; 80 if (pg->flags & PG_FAKE) { 81 pg->flags |= PG_RELEASED; 82 } 83 } 84 mutex_enter(&uvm_pageqlock); 85 uvm_page_unbusy(pgs, npages); 86 mutex_exit(&uvm_pageqlock); 87 } 88 89 static void 90 genfs_markdirty(struct vnode *vp) 91 { 92 struct genfs_node * const gp = VTOG(vp); 93 94 KASSERT(mutex_owned(&vp->v_interlock)); 95 gp->g_dirtygen++; 96 if ((vp->v_iflag & VI_ONWORKLST) == 0) { 97 vn_syncer_add_to_worklist(vp, filedelay); 98 } 99 if ((vp->v_iflag & (VI_WRMAP|VI_WRMAPDIRTY)) == VI_WRMAP) { 100 vp->v_iflag |= VI_WRMAPDIRTY; 101 } 102 } 103 104 /* 105 * generic VM getpages routine. 106 * Return PG_BUSY pages for the given range, 107 * reading from backing store if necessary. 108 */ 109 110 int 111 genfs_getpages(void *v) 112 { 113 struct vop_getpages_args /* { 114 struct vnode *a_vp; 115 voff_t a_offset; 116 struct vm_page **a_m; 117 int *a_count; 118 int a_centeridx; 119 vm_prot_t a_access_type; 120 int a_advice; 121 int a_flags; 122 } */ * const ap = v; 123 124 off_t diskeof, memeof; 125 int i, error, npages; 126 const int flags = ap->a_flags; 127 struct vnode * const vp = ap->a_vp; 128 struct uvm_object * const uobj = &vp->v_uobj; 129 kauth_cred_t const cred = curlwp->l_cred; /* XXXUBC curlwp */ 130 const bool async = (flags & PGO_SYNCIO) == 0; 131 const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0; 132 bool has_trans = false; 133 const bool overwrite = (flags & PGO_OVERWRITE) != 0; 134 const bool blockalloc = memwrite && (flags & PGO_NOBLOCKALLOC) == 0; 135 const bool glocked = (flags & PGO_GLOCKHELD) != 0; 136 UVMHIST_FUNC("genfs_getpages"); UVMHIST_CALLED(ubchist); 137 138 UVMHIST_LOG(ubchist, "vp %p off 0x%x/%x count %d", 139 vp, ap->a_offset >> 32, ap->a_offset, *ap->a_count); 140 141 KASSERT(vp->v_type == VREG || vp->v_type == VDIR || 142 vp->v_type == VLNK || vp->v_type == VBLK); 143 144 startover: 145 error = 0; 146 const voff_t origvsize = vp->v_size; 147 const off_t origoffset = ap->a_offset; 148 const int orignpages = *ap->a_count; 149 150 GOP_SIZE(vp, origvsize, &diskeof, 0); 151 if (flags & PGO_PASTEOF) { 152 off_t newsize; 153 #if defined(DIAGNOSTIC) 154 off_t writeeof; 155 #endif /* defined(DIAGNOSTIC) */ 156 157 newsize = MAX(origvsize, 158 origoffset + (orignpages << PAGE_SHIFT)); 159 GOP_SIZE(vp, newsize, &memeof, GOP_SIZE_MEM); 160 #if defined(DIAGNOSTIC) 161 GOP_SIZE(vp, vp->v_writesize, &writeeof, GOP_SIZE_MEM); 162 if (newsize > round_page(writeeof)) { 163 panic("%s: past eof: %" PRId64 " vs. %" PRId64, 164 __func__, newsize, round_page(writeeof)); 165 } 166 #endif /* defined(DIAGNOSTIC) */ 167 } else { 168 GOP_SIZE(vp, origvsize, &memeof, GOP_SIZE_MEM); 169 } 170 KASSERT(ap->a_centeridx >= 0 || ap->a_centeridx <= orignpages); 171 KASSERT((origoffset & (PAGE_SIZE - 1)) == 0 && origoffset >= 0); 172 KASSERT(orignpages > 0); 173 174 /* 175 * Bounds-check the request. 176 */ 177 178 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= memeof) { 179 if ((flags & PGO_LOCKED) == 0) { 180 mutex_exit(&uobj->vmobjlock); 181 } 182 UVMHIST_LOG(ubchist, "off 0x%x count %d goes past EOF 0x%x", 183 origoffset, *ap->a_count, memeof,0); 184 error = EINVAL; 185 goto out_err; 186 } 187 188 /* uobj is locked */ 189 190 if ((flags & PGO_NOTIMESTAMP) == 0 && 191 (vp->v_type != VBLK || 192 (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) { 193 int updflags = 0; 194 195 if ((vp->v_mount->mnt_flag & MNT_NOATIME) == 0) { 196 updflags = GOP_UPDATE_ACCESSED; 197 } 198 if (memwrite) { 199 updflags |= GOP_UPDATE_MODIFIED; 200 } 201 if (updflags != 0) { 202 GOP_MARKUPDATE(vp, updflags); 203 } 204 } 205 206 /* 207 * For PGO_LOCKED requests, just return whatever's in memory. 208 */ 209 210 if (flags & PGO_LOCKED) { 211 int nfound; 212 struct vm_page *pg; 213 214 KASSERT(!glocked); 215 npages = *ap->a_count; 216 #if defined(DEBUG) 217 for (i = 0; i < npages; i++) { 218 pg = ap->a_m[i]; 219 KASSERT(pg == NULL || pg == PGO_DONTCARE); 220 } 221 #endif /* defined(DEBUG) */ 222 nfound = uvn_findpages(uobj, origoffset, &npages, 223 ap->a_m, UFP_NOWAIT|UFP_NOALLOC|(memwrite ? UFP_NORDONLY : 0)); 224 KASSERT(npages == *ap->a_count); 225 if (nfound == 0) { 226 error = EBUSY; 227 goto out_err; 228 } 229 if (!genfs_node_rdtrylock(vp)) { 230 genfs_rel_pages(ap->a_m, npages); 231 232 /* 233 * restore the array. 234 */ 235 236 for (i = 0; i < npages; i++) { 237 pg = ap->a_m[i]; 238 239 if (pg != NULL || pg != PGO_DONTCARE) { 240 ap->a_m[i] = NULL; 241 } 242 } 243 } else { 244 genfs_node_unlock(vp); 245 } 246 error = (ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0); 247 if (error == 0 && memwrite) { 248 genfs_markdirty(vp); 249 } 250 goto out_err; 251 } 252 mutex_exit(&uobj->vmobjlock); 253 254 /* 255 * find the requested pages and make some simple checks. 256 * leave space in the page array for a whole block. 257 */ 258 259 const int fs_bshift = (vp->v_type != VBLK) ? 260 vp->v_mount->mnt_fs_bshift : DEV_BSHIFT; 261 const int dev_bshift = (vp->v_type != VBLK) ? 262 vp->v_mount->mnt_dev_bshift : DEV_BSHIFT; 263 const int fs_bsize = 1 << fs_bshift; 264 #define blk_mask (fs_bsize - 1) 265 #define trunc_blk(x) ((x) & ~blk_mask) 266 #define round_blk(x) (((x) + blk_mask) & ~blk_mask) 267 268 const int orignmempages = MIN(orignpages, 269 round_page(memeof - origoffset) >> PAGE_SHIFT); 270 npages = orignmempages; 271 const off_t startoffset = trunc_blk(origoffset); 272 const off_t endoffset = MIN( 273 round_page(round_blk(origoffset + (npages << PAGE_SHIFT))), 274 round_page(memeof)); 275 const int ridx = (origoffset - startoffset) >> PAGE_SHIFT; 276 277 const int pgs_size = sizeof(struct vm_page *) * 278 ((endoffset - startoffset) >> PAGE_SHIFT); 279 struct vm_page **pgs, *pgs_onstack[UBC_MAX_PAGES]; 280 281 if (pgs_size > sizeof(pgs_onstack)) { 282 pgs = kmem_zalloc(pgs_size, async ? KM_NOSLEEP : KM_SLEEP); 283 if (pgs == NULL) { 284 pgs = pgs_onstack; 285 error = ENOMEM; 286 goto out_err; 287 } 288 } else { 289 pgs = pgs_onstack; 290 (void)memset(pgs, 0, pgs_size); 291 } 292 293 UVMHIST_LOG(ubchist, "ridx %d npages %d startoff %ld endoff %ld", 294 ridx, npages, startoffset, endoffset); 295 296 if (!has_trans) { 297 fstrans_start(vp->v_mount, FSTRANS_SHARED); 298 has_trans = true; 299 } 300 301 /* 302 * hold g_glock to prevent a race with truncate. 303 * 304 * check if our idea of v_size is still valid. 305 */ 306 307 KASSERT(!glocked || genfs_node_wrlocked(vp)); 308 if (!glocked) { 309 if (blockalloc) { 310 genfs_node_wrlock(vp); 311 } else { 312 genfs_node_rdlock(vp); 313 } 314 } 315 mutex_enter(&uobj->vmobjlock); 316 if (vp->v_size < origvsize) { 317 if (!glocked) { 318 genfs_node_unlock(vp); 319 } 320 if (pgs != pgs_onstack) 321 kmem_free(pgs, pgs_size); 322 goto startover; 323 } 324 325 if (uvn_findpages(uobj, origoffset, &npages, &pgs[ridx], 326 async ? UFP_NOWAIT : UFP_ALL) != orignmempages) { 327 if (!glocked) { 328 genfs_node_unlock(vp); 329 } 330 KASSERT(async != 0); 331 genfs_rel_pages(&pgs[ridx], orignmempages); 332 mutex_exit(&uobj->vmobjlock); 333 error = EBUSY; 334 goto out_err_free; 335 } 336 337 /* 338 * if the pages are already resident, just return them. 339 */ 340 341 for (i = 0; i < npages; i++) { 342 struct vm_page *pg = pgs[ridx + i]; 343 344 if ((pg->flags & PG_FAKE) || 345 (blockalloc && (pg->flags & PG_RDONLY))) { 346 break; 347 } 348 } 349 if (i == npages) { 350 if (!glocked) { 351 genfs_node_unlock(vp); 352 } 353 UVMHIST_LOG(ubchist, "returning cached pages", 0,0,0,0); 354 npages += ridx; 355 goto out; 356 } 357 358 /* 359 * if PGO_OVERWRITE is set, don't bother reading the pages. 360 */ 361 362 if (overwrite) { 363 if (!glocked) { 364 genfs_node_unlock(vp); 365 } 366 UVMHIST_LOG(ubchist, "PGO_OVERWRITE",0,0,0,0); 367 368 for (i = 0; i < npages; i++) { 369 struct vm_page *pg = pgs[ridx + i]; 370 371 pg->flags &= ~(PG_RDONLY|PG_CLEAN); 372 } 373 npages += ridx; 374 goto out; 375 } 376 377 /* 378 * the page wasn't resident and we're not overwriting, 379 * so we're going to have to do some i/o. 380 * find any additional pages needed to cover the expanded range. 381 */ 382 383 npages = (endoffset - startoffset) >> PAGE_SHIFT; 384 if (startoffset != origoffset || npages != orignmempages) { 385 int npgs; 386 387 /* 388 * we need to avoid deadlocks caused by locking 389 * additional pages at lower offsets than pages we 390 * already have locked. unlock them all and start over. 391 */ 392 393 genfs_rel_pages(&pgs[ridx], orignmempages); 394 memset(pgs, 0, pgs_size); 395 396 UVMHIST_LOG(ubchist, "reset npages start 0x%x end 0x%x", 397 startoffset, endoffset, 0,0); 398 npgs = npages; 399 if (uvn_findpages(uobj, startoffset, &npgs, pgs, 400 async ? UFP_NOWAIT : UFP_ALL) != npages) { 401 if (!glocked) { 402 genfs_node_unlock(vp); 403 } 404 KASSERT(async != 0); 405 genfs_rel_pages(pgs, npages); 406 mutex_exit(&uobj->vmobjlock); 407 error = EBUSY; 408 goto out_err_free; 409 } 410 } 411 412 mutex_exit(&uobj->vmobjlock); 413 414 { 415 size_t bytes, iobytes, tailstart, tailbytes, totalbytes, skipbytes; 416 vaddr_t kva; 417 struct buf *bp, *mbp; 418 bool sawhole = false; 419 420 /* 421 * read the desired page(s). 422 */ 423 424 totalbytes = npages << PAGE_SHIFT; 425 bytes = MIN(totalbytes, MAX(diskeof - startoffset, 0)); 426 tailbytes = totalbytes - bytes; 427 skipbytes = 0; 428 429 kva = uvm_pagermapin(pgs, npages, 430 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 431 432 mbp = getiobuf(vp, true); 433 mbp->b_bufsize = totalbytes; 434 mbp->b_data = (void *)kva; 435 mbp->b_resid = mbp->b_bcount = bytes; 436 mbp->b_cflags = BC_BUSY; 437 if (async) { 438 mbp->b_flags = B_READ | B_ASYNC; 439 mbp->b_iodone = uvm_aio_biodone; 440 } else { 441 mbp->b_flags = B_READ; 442 mbp->b_iodone = NULL; 443 } 444 if (async) 445 BIO_SETPRIO(mbp, BPRIO_TIMELIMITED); 446 else 447 BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL); 448 449 /* 450 * if EOF is in the middle of the range, zero the part past EOF. 451 * skip over pages which are not PG_FAKE since in that case they have 452 * valid data that we need to preserve. 453 */ 454 455 tailstart = bytes; 456 while (tailbytes > 0) { 457 const int len = PAGE_SIZE - (tailstart & PAGE_MASK); 458 459 KASSERT(len <= tailbytes); 460 if ((pgs[tailstart >> PAGE_SHIFT]->flags & PG_FAKE) != 0) { 461 memset((void *)(kva + tailstart), 0, len); 462 UVMHIST_LOG(ubchist, "tailbytes %p 0x%x 0x%x", 463 kva, tailstart, len, 0); 464 } 465 tailstart += len; 466 tailbytes -= len; 467 } 468 469 /* 470 * now loop over the pages, reading as needed. 471 */ 472 473 bp = NULL; 474 off_t offset; 475 for (offset = startoffset; 476 bytes > 0; 477 offset += iobytes, bytes -= iobytes) { 478 int run; 479 daddr_t lbn, blkno; 480 int pidx; 481 struct vnode *devvp; 482 483 /* 484 * skip pages which don't need to be read. 485 */ 486 487 pidx = (offset - startoffset) >> PAGE_SHIFT; 488 while ((pgs[pidx]->flags & PG_FAKE) == 0) { 489 size_t b; 490 491 KASSERT((offset & (PAGE_SIZE - 1)) == 0); 492 if ((pgs[pidx]->flags & PG_RDONLY)) { 493 sawhole = true; 494 } 495 b = MIN(PAGE_SIZE, bytes); 496 offset += b; 497 bytes -= b; 498 skipbytes += b; 499 pidx++; 500 UVMHIST_LOG(ubchist, "skipping, new offset 0x%x", 501 offset, 0,0,0); 502 if (bytes == 0) { 503 goto loopdone; 504 } 505 } 506 507 /* 508 * bmap the file to find out the blkno to read from and 509 * how much we can read in one i/o. if bmap returns an error, 510 * skip the rest of the top-level i/o. 511 */ 512 513 lbn = offset >> fs_bshift; 514 error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run); 515 if (error) { 516 UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%x -> %d\n", 517 lbn,error,0,0); 518 skipbytes += bytes; 519 bytes = 0; 520 goto loopdone; 521 } 522 523 /* 524 * see how many pages can be read with this i/o. 525 * reduce the i/o size if necessary to avoid 526 * overwriting pages with valid data. 527 */ 528 529 iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset, 530 bytes); 531 if (offset + iobytes > round_page(offset)) { 532 int pcount; 533 534 pcount = 1; 535 while (pidx + pcount < npages && 536 pgs[pidx + pcount]->flags & PG_FAKE) { 537 pcount++; 538 } 539 iobytes = MIN(iobytes, (pcount << PAGE_SHIFT) - 540 (offset - trunc_page(offset))); 541 } 542 543 /* 544 * if this block isn't allocated, zero it instead of 545 * reading it. unless we are going to allocate blocks, 546 * mark the pages we zeroed PG_RDONLY. 547 */ 548 549 if (blkno == (daddr_t)-1) { 550 int holepages = (round_page(offset + iobytes) - 551 trunc_page(offset)) >> PAGE_SHIFT; 552 UVMHIST_LOG(ubchist, "lbn 0x%x -> HOLE", lbn,0,0,0); 553 554 sawhole = true; 555 memset((char *)kva + (offset - startoffset), 0, 556 iobytes); 557 skipbytes += iobytes; 558 559 for (i = 0; i < holepages; i++) { 560 if (memwrite) { 561 pgs[pidx + i]->flags &= ~PG_CLEAN; 562 } 563 if (!blockalloc) { 564 pgs[pidx + i]->flags |= PG_RDONLY; 565 } 566 } 567 continue; 568 } 569 570 /* 571 * allocate a sub-buf for this piece of the i/o 572 * (or just use mbp if there's only 1 piece), 573 * and start it going. 574 */ 575 576 if (offset == startoffset && iobytes == bytes) { 577 bp = mbp; 578 } else { 579 UVMHIST_LOG(ubchist, "vp %p bp %p num now %d", 580 vp, bp, vp->v_numoutput, 0); 581 bp = getiobuf(vp, true); 582 nestiobuf_setup(mbp, bp, offset - startoffset, iobytes); 583 } 584 bp->b_lblkno = 0; 585 586 /* adjust physical blkno for partial blocks */ 587 bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >> 588 dev_bshift); 589 590 UVMHIST_LOG(ubchist, 591 "bp %p offset 0x%x bcount 0x%x blkno 0x%x", 592 bp, offset, bp->b_bcount, bp->b_blkno); 593 594 VOP_STRATEGY(devvp, bp); 595 } 596 597 loopdone: 598 nestiobuf_done(mbp, skipbytes, error); 599 if (async) { 600 UVMHIST_LOG(ubchist, "returning 0 (async)",0,0,0,0); 601 if (!glocked) { 602 genfs_node_unlock(vp); 603 } 604 error = 0; 605 goto out_err_free; 606 } 607 if (bp != NULL) { 608 error = biowait(mbp); 609 } 610 611 /* Remove the mapping (make KVA available as soon as possible) */ 612 uvm_pagermapout(kva, npages); 613 614 /* 615 * if this we encountered a hole then we have to do a little more work. 616 * for read faults, we marked the page PG_RDONLY so that future 617 * write accesses to the page will fault again. 618 * for write faults, we must make sure that the backing store for 619 * the page is completely allocated while the pages are locked. 620 */ 621 622 if (!error && sawhole && blockalloc) { 623 /* 624 * XXX: This assumes that we come here only via 625 * the mmio path 626 */ 627 if (vp->v_mount->mnt_wapbl) { 628 error = WAPBL_BEGIN(vp->v_mount); 629 } 630 631 if (!error) { 632 error = GOP_ALLOC(vp, startoffset, 633 npages << PAGE_SHIFT, 0, cred); 634 if (vp->v_mount->mnt_wapbl) { 635 WAPBL_END(vp->v_mount); 636 } 637 } 638 639 UVMHIST_LOG(ubchist, "gop_alloc off 0x%x/0x%x -> %d", 640 startoffset, npages << PAGE_SHIFT, error,0); 641 if (!error) { 642 for (i = 0; i < npages; i++) { 643 struct vm_page *pg = pgs[i]; 644 645 if (pg == NULL) { 646 continue; 647 } 648 pg->flags &= ~(PG_CLEAN|PG_RDONLY); 649 UVMHIST_LOG(ubchist, "mark dirty pg %p", 650 pg,0,0,0); 651 } 652 } 653 } 654 if (!glocked) { 655 genfs_node_unlock(vp); 656 } 657 658 putiobuf(mbp); 659 } 660 661 mutex_enter(&uobj->vmobjlock); 662 663 /* 664 * we're almost done! release the pages... 665 * for errors, we free the pages. 666 * otherwise we activate them and mark them as valid and clean. 667 * also, unbusy pages that were not actually requested. 668 */ 669 670 if (error) { 671 for (i = 0; i < npages; i++) { 672 struct vm_page *pg = pgs[i]; 673 674 if (pg == NULL) { 675 continue; 676 } 677 UVMHIST_LOG(ubchist, "examining pg %p flags 0x%x", 678 pg, pg->flags, 0,0); 679 if (pg->flags & PG_FAKE) { 680 pg->flags |= PG_RELEASED; 681 } 682 } 683 mutex_enter(&uvm_pageqlock); 684 uvm_page_unbusy(pgs, npages); 685 mutex_exit(&uvm_pageqlock); 686 mutex_exit(&uobj->vmobjlock); 687 UVMHIST_LOG(ubchist, "returning error %d", error,0,0,0); 688 goto out_err_free; 689 } 690 691 out: 692 UVMHIST_LOG(ubchist, "succeeding, npages %d", npages,0,0,0); 693 error = 0; 694 mutex_enter(&uvm_pageqlock); 695 for (i = 0; i < npages; i++) { 696 struct vm_page *pg = pgs[i]; 697 if (pg == NULL) { 698 continue; 699 } 700 UVMHIST_LOG(ubchist, "examining pg %p flags 0x%x", 701 pg, pg->flags, 0,0); 702 if (pg->flags & PG_FAKE && !overwrite) { 703 pg->flags &= ~(PG_FAKE); 704 pmap_clear_modify(pgs[i]); 705 } 706 KASSERT(!memwrite || !blockalloc || (pg->flags & PG_RDONLY) == 0); 707 if (i < ridx || i >= ridx + orignmempages || async) { 708 UVMHIST_LOG(ubchist, "unbusy pg %p offset 0x%x", 709 pg, pg->offset,0,0); 710 if (pg->flags & PG_WANTED) { 711 wakeup(pg); 712 } 713 if (pg->flags & PG_FAKE) { 714 KASSERT(overwrite); 715 uvm_pagezero(pg); 716 } 717 if (pg->flags & PG_RELEASED) { 718 uvm_pagefree(pg); 719 continue; 720 } 721 uvm_pageenqueue(pg); 722 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE); 723 UVM_PAGE_OWN(pg, NULL); 724 } 725 } 726 mutex_exit(&uvm_pageqlock); 727 if (memwrite) { 728 genfs_markdirty(vp); 729 } 730 mutex_exit(&uobj->vmobjlock); 731 if (ap->a_m != NULL) { 732 memcpy(ap->a_m, &pgs[ridx], 733 orignmempages * sizeof(struct vm_page *)); 734 } 735 736 out_err_free: 737 if (pgs != NULL && pgs != pgs_onstack) 738 kmem_free(pgs, pgs_size); 739 out_err: 740 if (has_trans) 741 fstrans_done(vp->v_mount); 742 return error; 743 } 744 745 /* 746 * generic VM putpages routine. 747 * Write the given range of pages to backing store. 748 * 749 * => "offhi == 0" means flush all pages at or after "offlo". 750 * => object should be locked by caller. we return with the 751 * object unlocked. 752 * => if PGO_CLEANIT or PGO_SYNCIO is set, we may block (due to I/O). 753 * thus, a caller might want to unlock higher level resources 754 * (e.g. vm_map) before calling flush. 755 * => if neither PGO_CLEANIT nor PGO_SYNCIO is set, we will not block 756 * => if PGO_ALLPAGES is set, then all pages in the object will be processed. 757 * => NOTE: we rely on the fact that the object's memq is a TAILQ and 758 * that new pages are inserted on the tail end of the list. thus, 759 * we can make a complete pass through the object in one go by starting 760 * at the head and working towards the tail (new pages are put in 761 * front of us). 762 * => NOTE: we are allowed to lock the page queues, so the caller 763 * must not be holding the page queue lock. 764 * 765 * note on "cleaning" object and PG_BUSY pages: 766 * this routine is holding the lock on the object. the only time 767 * that it can run into a PG_BUSY page that it does not own is if 768 * some other process has started I/O on the page (e.g. either 769 * a pagein, or a pageout). if the PG_BUSY page is being paged 770 * in, then it can not be dirty (!PG_CLEAN) because no one has 771 * had a chance to modify it yet. if the PG_BUSY page is being 772 * paged out then it means that someone else has already started 773 * cleaning the page for us (how nice!). in this case, if we 774 * have syncio specified, then after we make our pass through the 775 * object we need to wait for the other PG_BUSY pages to clear 776 * off (i.e. we need to do an iosync). also note that once a 777 * page is PG_BUSY it must stay in its object until it is un-busyed. 778 * 779 * note on page traversal: 780 * we can traverse the pages in an object either by going down the 781 * linked list in "uobj->memq", or we can go over the address range 782 * by page doing hash table lookups for each address. depending 783 * on how many pages are in the object it may be cheaper to do one 784 * or the other. we set "by_list" to true if we are using memq. 785 * if the cost of a hash lookup was equal to the cost of the list 786 * traversal we could compare the number of pages in the start->stop 787 * range to the total number of pages in the object. however, it 788 * seems that a hash table lookup is more expensive than the linked 789 * list traversal, so we multiply the number of pages in the 790 * range by an estimate of the relatively higher cost of the hash lookup. 791 */ 792 793 int 794 genfs_putpages(void *v) 795 { 796 struct vop_putpages_args /* { 797 struct vnode *a_vp; 798 voff_t a_offlo; 799 voff_t a_offhi; 800 int a_flags; 801 } */ * const ap = v; 802 803 return genfs_do_putpages(ap->a_vp, ap->a_offlo, ap->a_offhi, 804 ap->a_flags, NULL); 805 } 806 807 int 808 genfs_do_putpages(struct vnode *vp, off_t startoff, off_t endoff, 809 int origflags, struct vm_page **busypg) 810 { 811 struct uvm_object * const uobj = &vp->v_uobj; 812 kmutex_t * const slock = &uobj->vmobjlock; 813 off_t off; 814 /* Even for strange MAXPHYS, the shift rounds down to a page */ 815 #define maxpages (MAXPHYS >> PAGE_SHIFT) 816 int i, error, npages, nback; 817 int freeflag; 818 struct vm_page *pgs[maxpages], *pg, *nextpg, *tpg, curmp, endmp; 819 bool wasclean, by_list, needs_clean, yld; 820 bool async = (origflags & PGO_SYNCIO) == 0; 821 bool pagedaemon = curlwp == uvm.pagedaemon_lwp; 822 struct lwp * const l = curlwp ? curlwp : &lwp0; 823 struct genfs_node * const gp = VTOG(vp); 824 int flags; 825 int dirtygen; 826 bool modified; 827 bool need_wapbl; 828 bool has_trans; 829 bool cleanall; 830 bool onworklst; 831 832 UVMHIST_FUNC("genfs_putpages"); UVMHIST_CALLED(ubchist); 833 834 KASSERT(origflags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)); 835 KASSERT((startoff & PAGE_MASK) == 0 && (endoff & PAGE_MASK) == 0); 836 KASSERT(startoff < endoff || endoff == 0); 837 838 UVMHIST_LOG(ubchist, "vp %p pages %d off 0x%x len 0x%x", 839 vp, uobj->uo_npages, startoff, endoff - startoff); 840 841 has_trans = false; 842 need_wapbl = (!pagedaemon && vp->v_mount && vp->v_mount->mnt_wapbl && 843 (origflags & PGO_JOURNALLOCKED) == 0); 844 845 retry: 846 modified = false; 847 flags = origflags; 848 KASSERT((vp->v_iflag & VI_ONWORKLST) != 0 || 849 (vp->v_iflag & VI_WRMAPDIRTY) == 0); 850 if (uobj->uo_npages == 0) { 851 if (vp->v_iflag & VI_ONWORKLST) { 852 vp->v_iflag &= ~VI_WRMAPDIRTY; 853 if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL) 854 vn_syncer_remove_from_worklist(vp); 855 } 856 if (has_trans) { 857 if (need_wapbl) 858 WAPBL_END(vp->v_mount); 859 fstrans_done(vp->v_mount); 860 } 861 mutex_exit(slock); 862 return (0); 863 } 864 865 /* 866 * the vnode has pages, set up to process the request. 867 */ 868 869 if (!has_trans && (flags & PGO_CLEANIT) != 0) { 870 mutex_exit(slock); 871 if (pagedaemon) { 872 error = fstrans_start_nowait(vp->v_mount, FSTRANS_LAZY); 873 if (error) 874 return error; 875 } else 876 fstrans_start(vp->v_mount, FSTRANS_LAZY); 877 if (need_wapbl) { 878 error = WAPBL_BEGIN(vp->v_mount); 879 if (error) { 880 fstrans_done(vp->v_mount); 881 return error; 882 } 883 } 884 has_trans = true; 885 mutex_enter(slock); 886 goto retry; 887 } 888 889 error = 0; 890 wasclean = (vp->v_numoutput == 0); 891 off = startoff; 892 if (endoff == 0 || flags & PGO_ALLPAGES) { 893 endoff = trunc_page(LLONG_MAX); 894 } 895 by_list = (uobj->uo_npages <= 896 ((endoff - startoff) >> PAGE_SHIFT) * UVM_PAGE_TREE_PENALTY); 897 898 #if !defined(DEBUG) 899 /* 900 * if this vnode is known not to have dirty pages, 901 * don't bother to clean it out. 902 */ 903 904 if ((vp->v_iflag & VI_ONWORKLST) == 0) { 905 if ((flags & (PGO_FREE|PGO_DEACTIVATE)) == 0) { 906 goto skip_scan; 907 } 908 flags &= ~PGO_CLEANIT; 909 } 910 #endif /* !defined(DEBUG) */ 911 912 /* 913 * start the loop. when scanning by list, hold the last page 914 * in the list before we start. pages allocated after we start 915 * will be added to the end of the list, so we can stop at the 916 * current last page. 917 */ 918 919 cleanall = (flags & PGO_CLEANIT) != 0 && wasclean && 920 startoff == 0 && endoff == trunc_page(LLONG_MAX) && 921 (vp->v_iflag & VI_ONWORKLST) != 0; 922 dirtygen = gp->g_dirtygen; 923 freeflag = pagedaemon ? PG_PAGEOUT : PG_RELEASED; 924 if (by_list) { 925 curmp.flags = PG_MARKER; 926 endmp.flags = PG_MARKER; 927 pg = TAILQ_FIRST(&uobj->memq); 928 TAILQ_INSERT_TAIL(&uobj->memq, &endmp, listq.queue); 929 } else { 930 pg = uvm_pagelookup(uobj, off); 931 } 932 nextpg = NULL; 933 while (by_list || off < endoff) { 934 935 /* 936 * if the current page is not interesting, move on to the next. 937 */ 938 939 KASSERT(pg == NULL || pg->uobject == uobj || 940 (pg->flags & PG_MARKER) != 0); 941 KASSERT(pg == NULL || 942 (pg->flags & (PG_RELEASED|PG_PAGEOUT)) == 0 || 943 (pg->flags & (PG_BUSY|PG_MARKER)) != 0); 944 if (by_list) { 945 if (pg == &endmp) { 946 break; 947 } 948 if (pg->flags & PG_MARKER) { 949 pg = TAILQ_NEXT(pg, listq.queue); 950 continue; 951 } 952 if (pg->offset < startoff || pg->offset >= endoff || 953 pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 954 if (pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 955 wasclean = false; 956 } 957 pg = TAILQ_NEXT(pg, listq.queue); 958 continue; 959 } 960 off = pg->offset; 961 } else if (pg == NULL || pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 962 if (pg != NULL) { 963 wasclean = false; 964 } 965 off += PAGE_SIZE; 966 if (off < endoff) { 967 pg = uvm_pagelookup(uobj, off); 968 } 969 continue; 970 } 971 972 /* 973 * if the current page needs to be cleaned and it's busy, 974 * wait for it to become unbusy. 975 */ 976 977 yld = (l->l_cpu->ci_schedstate.spc_flags & 978 SPCF_SHOULDYIELD) && !pagedaemon; 979 if (pg->flags & PG_BUSY || yld) { 980 UVMHIST_LOG(ubchist, "busy %p", pg,0,0,0); 981 if (flags & PGO_BUSYFAIL && pg->flags & PG_BUSY) { 982 UVMHIST_LOG(ubchist, "busyfail %p", pg, 0,0,0); 983 error = EDEADLK; 984 if (busypg != NULL) 985 *busypg = pg; 986 break; 987 } 988 if (pagedaemon) { 989 /* 990 * someone has taken the page while we 991 * dropped the lock for fstrans_start. 992 */ 993 break; 994 } 995 if (by_list) { 996 TAILQ_INSERT_BEFORE(pg, &curmp, listq.queue); 997 UVMHIST_LOG(ubchist, "curmp next %p", 998 TAILQ_NEXT(&curmp, listq.queue), 0,0,0); 999 } 1000 if (yld) { 1001 mutex_exit(slock); 1002 preempt(); 1003 mutex_enter(slock); 1004 } else { 1005 pg->flags |= PG_WANTED; 1006 UVM_UNLOCK_AND_WAIT(pg, slock, 0, "genput", 0); 1007 mutex_enter(slock); 1008 } 1009 if (by_list) { 1010 UVMHIST_LOG(ubchist, "after next %p", 1011 TAILQ_NEXT(&curmp, listq.queue), 0,0,0); 1012 pg = TAILQ_NEXT(&curmp, listq.queue); 1013 TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue); 1014 } else { 1015 pg = uvm_pagelookup(uobj, off); 1016 } 1017 continue; 1018 } 1019 1020 /* 1021 * if we're freeing, remove all mappings of the page now. 1022 * if we're cleaning, check if the page is needs to be cleaned. 1023 */ 1024 1025 if (flags & PGO_FREE) { 1026 pmap_page_protect(pg, VM_PROT_NONE); 1027 } else if (flags & PGO_CLEANIT) { 1028 1029 /* 1030 * if we still have some hope to pull this vnode off 1031 * from the syncer queue, write-protect the page. 1032 */ 1033 1034 if (cleanall && wasclean && 1035 gp->g_dirtygen == dirtygen) { 1036 1037 /* 1038 * uobj pages get wired only by uvm_fault 1039 * where uobj is locked. 1040 */ 1041 1042 if (pg->wire_count == 0) { 1043 pmap_page_protect(pg, 1044 VM_PROT_READ|VM_PROT_EXECUTE); 1045 } else { 1046 cleanall = false; 1047 } 1048 } 1049 } 1050 1051 if (flags & PGO_CLEANIT) { 1052 needs_clean = pmap_clear_modify(pg) || 1053 (pg->flags & PG_CLEAN) == 0; 1054 pg->flags |= PG_CLEAN; 1055 } else { 1056 needs_clean = false; 1057 } 1058 1059 /* 1060 * if we're cleaning, build a cluster. 1061 * the cluster will consist of pages which are currently dirty, 1062 * but they will be returned to us marked clean. 1063 * if not cleaning, just operate on the one page. 1064 */ 1065 1066 if (needs_clean) { 1067 KDASSERT((vp->v_iflag & VI_ONWORKLST)); 1068 wasclean = false; 1069 memset(pgs, 0, sizeof(pgs)); 1070 pg->flags |= PG_BUSY; 1071 UVM_PAGE_OWN(pg, "genfs_putpages"); 1072 1073 /* 1074 * first look backward. 1075 */ 1076 1077 npages = MIN(maxpages >> 1, off >> PAGE_SHIFT); 1078 nback = npages; 1079 uvn_findpages(uobj, off - PAGE_SIZE, &nback, &pgs[0], 1080 UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY|UFP_BACKWARD); 1081 if (nback) { 1082 memmove(&pgs[0], &pgs[npages - nback], 1083 nback * sizeof(pgs[0])); 1084 if (npages - nback < nback) 1085 memset(&pgs[nback], 0, 1086 (npages - nback) * sizeof(pgs[0])); 1087 else 1088 memset(&pgs[npages - nback], 0, 1089 nback * sizeof(pgs[0])); 1090 } 1091 1092 /* 1093 * then plug in our page of interest. 1094 */ 1095 1096 pgs[nback] = pg; 1097 1098 /* 1099 * then look forward to fill in the remaining space in 1100 * the array of pages. 1101 */ 1102 1103 npages = maxpages - nback - 1; 1104 uvn_findpages(uobj, off + PAGE_SIZE, &npages, 1105 &pgs[nback + 1], 1106 UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY); 1107 npages += nback + 1; 1108 } else { 1109 pgs[0] = pg; 1110 npages = 1; 1111 nback = 0; 1112 } 1113 1114 /* 1115 * apply FREE or DEACTIVATE options if requested. 1116 */ 1117 1118 if (flags & (PGO_DEACTIVATE|PGO_FREE)) { 1119 mutex_enter(&uvm_pageqlock); 1120 } 1121 for (i = 0; i < npages; i++) { 1122 tpg = pgs[i]; 1123 KASSERT(tpg->uobject == uobj); 1124 if (by_list && tpg == TAILQ_NEXT(pg, listq.queue)) 1125 pg = tpg; 1126 if (tpg->offset < startoff || tpg->offset >= endoff) 1127 continue; 1128 if (flags & PGO_DEACTIVATE && tpg->wire_count == 0) { 1129 uvm_pagedeactivate(tpg); 1130 } else if (flags & PGO_FREE) { 1131 pmap_page_protect(tpg, VM_PROT_NONE); 1132 if (tpg->flags & PG_BUSY) { 1133 tpg->flags |= freeflag; 1134 if (pagedaemon) { 1135 uvm_pageout_start(1); 1136 uvm_pagedequeue(tpg); 1137 } 1138 } else { 1139 1140 /* 1141 * ``page is not busy'' 1142 * implies that npages is 1 1143 * and needs_clean is false. 1144 */ 1145 1146 nextpg = TAILQ_NEXT(tpg, listq.queue); 1147 uvm_pagefree(tpg); 1148 if (pagedaemon) 1149 uvmexp.pdfreed++; 1150 } 1151 } 1152 } 1153 if (flags & (PGO_DEACTIVATE|PGO_FREE)) { 1154 mutex_exit(&uvm_pageqlock); 1155 } 1156 if (needs_clean) { 1157 modified = true; 1158 1159 /* 1160 * start the i/o. if we're traversing by list, 1161 * keep our place in the list with a marker page. 1162 */ 1163 1164 if (by_list) { 1165 TAILQ_INSERT_AFTER(&uobj->memq, pg, &curmp, 1166 listq.queue); 1167 } 1168 mutex_exit(slock); 1169 error = GOP_WRITE(vp, pgs, npages, flags); 1170 mutex_enter(slock); 1171 if (by_list) { 1172 pg = TAILQ_NEXT(&curmp, listq.queue); 1173 TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue); 1174 } 1175 if (error) { 1176 break; 1177 } 1178 if (by_list) { 1179 continue; 1180 } 1181 } 1182 1183 /* 1184 * find the next page and continue if there was no error. 1185 */ 1186 1187 if (by_list) { 1188 if (nextpg) { 1189 pg = nextpg; 1190 nextpg = NULL; 1191 } else { 1192 pg = TAILQ_NEXT(pg, listq.queue); 1193 } 1194 } else { 1195 off += (npages - nback) << PAGE_SHIFT; 1196 if (off < endoff) { 1197 pg = uvm_pagelookup(uobj, off); 1198 } 1199 } 1200 } 1201 if (by_list) { 1202 TAILQ_REMOVE(&uobj->memq, &endmp, listq.queue); 1203 } 1204 1205 if (modified && (vp->v_iflag & VI_WRMAPDIRTY) != 0 && 1206 (vp->v_type != VBLK || 1207 (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) { 1208 GOP_MARKUPDATE(vp, GOP_UPDATE_MODIFIED); 1209 } 1210 1211 /* 1212 * if we're cleaning and there was nothing to clean, 1213 * take us off the syncer list. if we started any i/o 1214 * and we're doing sync i/o, wait for all writes to finish. 1215 */ 1216 1217 if (cleanall && wasclean && gp->g_dirtygen == dirtygen && 1218 (vp->v_iflag & VI_ONWORKLST) != 0) { 1219 #if defined(DEBUG) 1220 TAILQ_FOREACH(pg, &uobj->memq, listq.queue) { 1221 if ((pg->flags & PG_MARKER) != 0) { 1222 continue; 1223 } 1224 if ((pg->flags & PG_CLEAN) == 0) { 1225 printf("%s: %p: !CLEAN\n", __func__, pg); 1226 } 1227 if (pmap_is_modified(pg)) { 1228 printf("%s: %p: modified\n", __func__, pg); 1229 } 1230 } 1231 #endif /* defined(DEBUG) */ 1232 vp->v_iflag &= ~VI_WRMAPDIRTY; 1233 if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL) 1234 vn_syncer_remove_from_worklist(vp); 1235 } 1236 1237 #if !defined(DEBUG) 1238 skip_scan: 1239 #endif /* !defined(DEBUG) */ 1240 1241 /* Wait for output to complete. */ 1242 if (!wasclean && !async && vp->v_numoutput != 0) { 1243 while (vp->v_numoutput != 0) 1244 cv_wait(&vp->v_cv, slock); 1245 } 1246 onworklst = (vp->v_iflag & VI_ONWORKLST) != 0; 1247 mutex_exit(slock); 1248 1249 if ((flags & PGO_RECLAIM) != 0 && onworklst) { 1250 /* 1251 * in the case of PGO_RECLAIM, ensure to make the vnode clean. 1252 * retrying is not a big deal because, in many cases, 1253 * uobj->uo_npages is already 0 here. 1254 */ 1255 mutex_enter(slock); 1256 goto retry; 1257 } 1258 1259 if (has_trans) { 1260 if (need_wapbl) 1261 WAPBL_END(vp->v_mount); 1262 fstrans_done(vp->v_mount); 1263 } 1264 1265 return (error); 1266 } 1267 1268 int 1269 genfs_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, int flags) 1270 { 1271 off_t off; 1272 vaddr_t kva; 1273 size_t len; 1274 int error; 1275 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1276 1277 UVMHIST_LOG(ubchist, "vp %p pgs %p npages %d flags 0x%x", 1278 vp, pgs, npages, flags); 1279 1280 off = pgs[0]->offset; 1281 kva = uvm_pagermapin(pgs, npages, 1282 UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK); 1283 len = npages << PAGE_SHIFT; 1284 1285 error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE, 1286 uvm_aio_biodone); 1287 1288 return error; 1289 } 1290 1291 int 1292 genfs_gop_write_rwmap(struct vnode *vp, struct vm_page **pgs, int npages, int flags) 1293 { 1294 off_t off; 1295 vaddr_t kva; 1296 size_t len; 1297 int error; 1298 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1299 1300 UVMHIST_LOG(ubchist, "vp %p pgs %p npages %d flags 0x%x", 1301 vp, pgs, npages, flags); 1302 1303 off = pgs[0]->offset; 1304 kva = uvm_pagermapin(pgs, npages, 1305 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 1306 len = npages << PAGE_SHIFT; 1307 1308 error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE, 1309 uvm_aio_biodone); 1310 1311 return error; 1312 } 1313 1314 /* 1315 * Backend routine for doing I/O to vnode pages. Pages are already locked 1316 * and mapped into kernel memory. Here we just look up the underlying 1317 * device block addresses and call the strategy routine. 1318 */ 1319 1320 static int 1321 genfs_do_io(struct vnode *vp, off_t off, vaddr_t kva, size_t len, int flags, 1322 enum uio_rw rw, void (*iodone)(struct buf *)) 1323 { 1324 int s, error; 1325 int fs_bshift, dev_bshift; 1326 off_t eof, offset, startoffset; 1327 size_t bytes, iobytes, skipbytes; 1328 struct buf *mbp, *bp; 1329 const bool async = (flags & PGO_SYNCIO) == 0; 1330 const bool iowrite = rw == UIO_WRITE; 1331 const int brw = iowrite ? B_WRITE : B_READ; 1332 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1333 1334 UVMHIST_LOG(ubchist, "vp %p kva %p len 0x%x flags 0x%x", 1335 vp, kva, len, flags); 1336 1337 KASSERT(vp->v_size <= vp->v_writesize); 1338 GOP_SIZE(vp, vp->v_writesize, &eof, 0); 1339 if (vp->v_type != VBLK) { 1340 fs_bshift = vp->v_mount->mnt_fs_bshift; 1341 dev_bshift = vp->v_mount->mnt_dev_bshift; 1342 } else { 1343 fs_bshift = DEV_BSHIFT; 1344 dev_bshift = DEV_BSHIFT; 1345 } 1346 error = 0; 1347 startoffset = off; 1348 bytes = MIN(len, eof - startoffset); 1349 skipbytes = 0; 1350 KASSERT(bytes != 0); 1351 1352 if (iowrite) { 1353 mutex_enter(&vp->v_interlock); 1354 vp->v_numoutput += 2; 1355 mutex_exit(&vp->v_interlock); 1356 } 1357 mbp = getiobuf(vp, true); 1358 UVMHIST_LOG(ubchist, "vp %p mbp %p num now %d bytes 0x%x", 1359 vp, mbp, vp->v_numoutput, bytes); 1360 mbp->b_bufsize = len; 1361 mbp->b_data = (void *)kva; 1362 mbp->b_resid = mbp->b_bcount = bytes; 1363 mbp->b_cflags = BC_BUSY | BC_AGE; 1364 if (async) { 1365 mbp->b_flags = brw | B_ASYNC; 1366 mbp->b_iodone = iodone; 1367 } else { 1368 mbp->b_flags = brw; 1369 mbp->b_iodone = NULL; 1370 } 1371 if (curlwp == uvm.pagedaemon_lwp) 1372 BIO_SETPRIO(mbp, BPRIO_TIMELIMITED); 1373 else if (async) 1374 BIO_SETPRIO(mbp, BPRIO_TIMENONCRITICAL); 1375 else 1376 BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL); 1377 1378 bp = NULL; 1379 for (offset = startoffset; 1380 bytes > 0; 1381 offset += iobytes, bytes -= iobytes) { 1382 int run; 1383 daddr_t lbn, blkno; 1384 struct vnode *devvp; 1385 1386 /* 1387 * bmap the file to find out the blkno to read from and 1388 * how much we can read in one i/o. if bmap returns an error, 1389 * skip the rest of the top-level i/o. 1390 */ 1391 1392 lbn = offset >> fs_bshift; 1393 error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run); 1394 if (error) { 1395 UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%x -> %d\n", 1396 lbn,error,0,0); 1397 skipbytes += bytes; 1398 bytes = 0; 1399 goto loopdone; 1400 } 1401 1402 /* 1403 * see how many pages can be read with this i/o. 1404 * reduce the i/o size if necessary to avoid 1405 * overwriting pages with valid data. 1406 */ 1407 1408 iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset, 1409 bytes); 1410 1411 /* 1412 * if this block isn't allocated, zero it instead of 1413 * reading it. unless we are going to allocate blocks, 1414 * mark the pages we zeroed PG_RDONLY. 1415 */ 1416 1417 if (blkno == (daddr_t)-1) { 1418 if (!iowrite) { 1419 memset((char *)kva + (offset - startoffset), 0, 1420 iobytes); 1421 } 1422 skipbytes += iobytes; 1423 continue; 1424 } 1425 1426 /* 1427 * allocate a sub-buf for this piece of the i/o 1428 * (or just use mbp if there's only 1 piece), 1429 * and start it going. 1430 */ 1431 1432 if (offset == startoffset && iobytes == bytes) { 1433 bp = mbp; 1434 } else { 1435 UVMHIST_LOG(ubchist, "vp %p bp %p num now %d", 1436 vp, bp, vp->v_numoutput, 0); 1437 bp = getiobuf(vp, true); 1438 nestiobuf_setup(mbp, bp, offset - startoffset, iobytes); 1439 } 1440 bp->b_lblkno = 0; 1441 1442 /* adjust physical blkno for partial blocks */ 1443 bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >> 1444 dev_bshift); 1445 1446 UVMHIST_LOG(ubchist, 1447 "bp %p offset 0x%x bcount 0x%x blkno 0x%x", 1448 bp, offset, bp->b_bcount, bp->b_blkno); 1449 1450 VOP_STRATEGY(devvp, bp); 1451 } 1452 1453 loopdone: 1454 if (skipbytes) { 1455 UVMHIST_LOG(ubchist, "skipbytes %d", skipbytes, 0,0,0); 1456 } 1457 nestiobuf_done(mbp, skipbytes, error); 1458 if (async) { 1459 UVMHIST_LOG(ubchist, "returning 0 (async)", 0,0,0,0); 1460 return (0); 1461 } 1462 UVMHIST_LOG(ubchist, "waiting for mbp %p", mbp,0,0,0); 1463 error = biowait(mbp); 1464 s = splbio(); 1465 (*iodone)(mbp); 1466 splx(s); 1467 UVMHIST_LOG(ubchist, "returning, error %d", error,0,0,0); 1468 return (error); 1469 } 1470 1471 int 1472 genfs_compat_getpages(void *v) 1473 { 1474 struct vop_getpages_args /* { 1475 struct vnode *a_vp; 1476 voff_t a_offset; 1477 struct vm_page **a_m; 1478 int *a_count; 1479 int a_centeridx; 1480 vm_prot_t a_access_type; 1481 int a_advice; 1482 int a_flags; 1483 } */ *ap = v; 1484 1485 off_t origoffset; 1486 struct vnode *vp = ap->a_vp; 1487 struct uvm_object *uobj = &vp->v_uobj; 1488 struct vm_page *pg, **pgs; 1489 vaddr_t kva; 1490 int i, error, orignpages, npages; 1491 struct iovec iov; 1492 struct uio uio; 1493 kauth_cred_t cred = curlwp->l_cred; 1494 const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0; 1495 1496 error = 0; 1497 origoffset = ap->a_offset; 1498 orignpages = *ap->a_count; 1499 pgs = ap->a_m; 1500 1501 if (ap->a_flags & PGO_LOCKED) { 1502 uvn_findpages(uobj, origoffset, ap->a_count, ap->a_m, 1503 UFP_NOWAIT|UFP_NOALLOC| (memwrite ? UFP_NORDONLY : 0)); 1504 1505 error = ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0; 1506 if (error == 0 && memwrite) { 1507 genfs_markdirty(vp); 1508 } 1509 return error; 1510 } 1511 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= vp->v_size) { 1512 mutex_exit(&uobj->vmobjlock); 1513 return EINVAL; 1514 } 1515 if ((ap->a_flags & PGO_SYNCIO) == 0) { 1516 mutex_exit(&uobj->vmobjlock); 1517 return 0; 1518 } 1519 npages = orignpages; 1520 uvn_findpages(uobj, origoffset, &npages, pgs, UFP_ALL); 1521 mutex_exit(&uobj->vmobjlock); 1522 kva = uvm_pagermapin(pgs, npages, 1523 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 1524 for (i = 0; i < npages; i++) { 1525 pg = pgs[i]; 1526 if ((pg->flags & PG_FAKE) == 0) { 1527 continue; 1528 } 1529 iov.iov_base = (char *)kva + (i << PAGE_SHIFT); 1530 iov.iov_len = PAGE_SIZE; 1531 uio.uio_iov = &iov; 1532 uio.uio_iovcnt = 1; 1533 uio.uio_offset = origoffset + (i << PAGE_SHIFT); 1534 uio.uio_rw = UIO_READ; 1535 uio.uio_resid = PAGE_SIZE; 1536 UIO_SETUP_SYSSPACE(&uio); 1537 /* XXX vn_lock */ 1538 error = VOP_READ(vp, &uio, 0, cred); 1539 if (error) { 1540 break; 1541 } 1542 if (uio.uio_resid) { 1543 memset(iov.iov_base, 0, uio.uio_resid); 1544 } 1545 } 1546 uvm_pagermapout(kva, npages); 1547 mutex_enter(&uobj->vmobjlock); 1548 mutex_enter(&uvm_pageqlock); 1549 for (i = 0; i < npages; i++) { 1550 pg = pgs[i]; 1551 if (error && (pg->flags & PG_FAKE) != 0) { 1552 pg->flags |= PG_RELEASED; 1553 } else { 1554 pmap_clear_modify(pg); 1555 uvm_pageactivate(pg); 1556 } 1557 } 1558 if (error) { 1559 uvm_page_unbusy(pgs, npages); 1560 } 1561 mutex_exit(&uvm_pageqlock); 1562 if (error == 0 && memwrite) { 1563 genfs_markdirty(vp); 1564 } 1565 mutex_exit(&uobj->vmobjlock); 1566 return error; 1567 } 1568 1569 int 1570 genfs_compat_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, 1571 int flags) 1572 { 1573 off_t offset; 1574 struct iovec iov; 1575 struct uio uio; 1576 kauth_cred_t cred = curlwp->l_cred; 1577 struct buf *bp; 1578 vaddr_t kva; 1579 int error; 1580 1581 offset = pgs[0]->offset; 1582 kva = uvm_pagermapin(pgs, npages, 1583 UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK); 1584 1585 iov.iov_base = (void *)kva; 1586 iov.iov_len = npages << PAGE_SHIFT; 1587 uio.uio_iov = &iov; 1588 uio.uio_iovcnt = 1; 1589 uio.uio_offset = offset; 1590 uio.uio_rw = UIO_WRITE; 1591 uio.uio_resid = npages << PAGE_SHIFT; 1592 UIO_SETUP_SYSSPACE(&uio); 1593 /* XXX vn_lock */ 1594 error = VOP_WRITE(vp, &uio, 0, cred); 1595 1596 mutex_enter(&vp->v_interlock); 1597 vp->v_numoutput++; 1598 mutex_exit(&vp->v_interlock); 1599 1600 bp = getiobuf(vp, true); 1601 bp->b_cflags = BC_BUSY | BC_AGE; 1602 bp->b_lblkno = offset >> vp->v_mount->mnt_fs_bshift; 1603 bp->b_data = (char *)kva; 1604 bp->b_bcount = npages << PAGE_SHIFT; 1605 bp->b_bufsize = npages << PAGE_SHIFT; 1606 bp->b_resid = 0; 1607 bp->b_error = error; 1608 uvm_aio_aiodone(bp); 1609 return (error); 1610 } 1611 1612 /* 1613 * Process a uio using direct I/O. If we reach a part of the request 1614 * which cannot be processed in this fashion for some reason, just return. 1615 * The caller must handle some additional part of the request using 1616 * buffered I/O before trying direct I/O again. 1617 */ 1618 1619 void 1620 genfs_directio(struct vnode *vp, struct uio *uio, int ioflag) 1621 { 1622 struct vmspace *vs; 1623 struct iovec *iov; 1624 vaddr_t va; 1625 size_t len; 1626 const int mask = DEV_BSIZE - 1; 1627 int error; 1628 bool need_wapbl = (vp->v_mount && vp->v_mount->mnt_wapbl && 1629 (ioflag & IO_JOURNALLOCKED) == 0); 1630 1631 /* 1632 * We only support direct I/O to user space for now. 1633 */ 1634 1635 if (VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) { 1636 return; 1637 } 1638 1639 /* 1640 * If the vnode is mapped, we would need to get the getpages lock 1641 * to stabilize the bmap, but then we would get into trouble whil e 1642 * locking the pages if the pages belong to this same vnode (or a 1643 * multi-vnode cascade to the same effect). Just fall back to 1644 * buffered I/O if the vnode is mapped to avoid this mess. 1645 */ 1646 1647 if (vp->v_vflag & VV_MAPPED) { 1648 return; 1649 } 1650 1651 if (need_wapbl) { 1652 error = WAPBL_BEGIN(vp->v_mount); 1653 if (error) 1654 return; 1655 } 1656 1657 /* 1658 * Do as much of the uio as possible with direct I/O. 1659 */ 1660 1661 vs = uio->uio_vmspace; 1662 while (uio->uio_resid) { 1663 iov = uio->uio_iov; 1664 if (iov->iov_len == 0) { 1665 uio->uio_iov++; 1666 uio->uio_iovcnt--; 1667 continue; 1668 } 1669 va = (vaddr_t)iov->iov_base; 1670 len = MIN(iov->iov_len, genfs_maxdio); 1671 len &= ~mask; 1672 1673 /* 1674 * If the next chunk is smaller than DEV_BSIZE or extends past 1675 * the current EOF, then fall back to buffered I/O. 1676 */ 1677 1678 if (len == 0 || uio->uio_offset + len > vp->v_size) { 1679 break; 1680 } 1681 1682 /* 1683 * Check alignment. The file offset must be at least 1684 * sector-aligned. The exact constraint on memory alignment 1685 * is very hardware-dependent, but requiring sector-aligned 1686 * addresses there too is safe. 1687 */ 1688 1689 if (uio->uio_offset & mask || va & mask) { 1690 break; 1691 } 1692 error = genfs_do_directio(vs, va, len, vp, uio->uio_offset, 1693 uio->uio_rw); 1694 if (error) { 1695 break; 1696 } 1697 iov->iov_base = (char *)iov->iov_base + len; 1698 iov->iov_len -= len; 1699 uio->uio_offset += len; 1700 uio->uio_resid -= len; 1701 } 1702 1703 if (need_wapbl) 1704 WAPBL_END(vp->v_mount); 1705 } 1706 1707 /* 1708 * Iodone routine for direct I/O. We don't do much here since the request is 1709 * always synchronous, so the caller will do most of the work after biowait(). 1710 */ 1711 1712 static void 1713 genfs_dio_iodone(struct buf *bp) 1714 { 1715 1716 KASSERT((bp->b_flags & B_ASYNC) == 0); 1717 if ((bp->b_flags & B_READ) == 0 && (bp->b_cflags & BC_AGE) != 0) { 1718 mutex_enter(bp->b_objlock); 1719 vwakeup(bp); 1720 mutex_exit(bp->b_objlock); 1721 } 1722 putiobuf(bp); 1723 } 1724 1725 /* 1726 * Process one chunk of a direct I/O request. 1727 */ 1728 1729 static int 1730 genfs_do_directio(struct vmspace *vs, vaddr_t uva, size_t len, struct vnode *vp, 1731 off_t off, enum uio_rw rw) 1732 { 1733 struct vm_map *map; 1734 struct pmap *upm, *kpm; 1735 size_t klen = round_page(uva + len) - trunc_page(uva); 1736 off_t spoff, epoff; 1737 vaddr_t kva, puva; 1738 paddr_t pa; 1739 vm_prot_t prot; 1740 int error, rv, poff, koff; 1741 const int pgoflags = PGO_CLEANIT | PGO_SYNCIO | PGO_JOURNALLOCKED | 1742 (rw == UIO_WRITE ? PGO_FREE : 0); 1743 1744 /* 1745 * For writes, verify that this range of the file already has fully 1746 * allocated backing store. If there are any holes, just punt and 1747 * make the caller take the buffered write path. 1748 */ 1749 1750 if (rw == UIO_WRITE) { 1751 daddr_t lbn, elbn, blkno; 1752 int bsize, bshift, run; 1753 1754 bshift = vp->v_mount->mnt_fs_bshift; 1755 bsize = 1 << bshift; 1756 lbn = off >> bshift; 1757 elbn = (off + len + bsize - 1) >> bshift; 1758 while (lbn < elbn) { 1759 error = VOP_BMAP(vp, lbn, NULL, &blkno, &run); 1760 if (error) { 1761 return error; 1762 } 1763 if (blkno == (daddr_t)-1) { 1764 return ENOSPC; 1765 } 1766 lbn += 1 + run; 1767 } 1768 } 1769 1770 /* 1771 * Flush any cached pages for parts of the file that we're about to 1772 * access. If we're writing, invalidate pages as well. 1773 */ 1774 1775 spoff = trunc_page(off); 1776 epoff = round_page(off + len); 1777 mutex_enter(&vp->v_interlock); 1778 error = VOP_PUTPAGES(vp, spoff, epoff, pgoflags); 1779 if (error) { 1780 return error; 1781 } 1782 1783 /* 1784 * Wire the user pages and remap them into kernel memory. 1785 */ 1786 1787 prot = rw == UIO_READ ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_READ; 1788 error = uvm_vslock(vs, (void *)uva, len, prot); 1789 if (error) { 1790 return error; 1791 } 1792 1793 map = &vs->vm_map; 1794 upm = vm_map_pmap(map); 1795 kpm = vm_map_pmap(kernel_map); 1796 kva = uvm_km_alloc(kernel_map, klen, 0, 1797 UVM_KMF_VAONLY | UVM_KMF_WAITVA); 1798 puva = trunc_page(uva); 1799 for (poff = 0; poff < klen; poff += PAGE_SIZE) { 1800 rv = pmap_extract(upm, puva + poff, &pa); 1801 KASSERT(rv); 1802 pmap_enter(kpm, kva + poff, pa, prot, prot | PMAP_WIRED); 1803 } 1804 pmap_update(kpm); 1805 1806 /* 1807 * Do the I/O. 1808 */ 1809 1810 koff = uva - trunc_page(uva); 1811 error = genfs_do_io(vp, off, kva + koff, len, PGO_SYNCIO, rw, 1812 genfs_dio_iodone); 1813 1814 /* 1815 * Tear down the kernel mapping. 1816 */ 1817 1818 pmap_remove(kpm, kva, kva + klen); 1819 pmap_update(kpm); 1820 uvm_km_free(kernel_map, kva, klen, UVM_KMF_VAONLY); 1821 1822 /* 1823 * Unwire the user pages. 1824 */ 1825 1826 uvm_vsunlock(vs, (void *)uva, len); 1827 return error; 1828 } 1829 1830