1 /* $NetBSD: genfs_io.c,v 1.55 2012/05/22 14:20:39 yamt Exp $ */ 2 3 /* 4 * Copyright (c) 1982, 1986, 1989, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the University nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 */ 32 33 #include <sys/cdefs.h> 34 __KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.55 2012/05/22 14:20:39 yamt Exp $"); 35 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/proc.h> 39 #include <sys/kernel.h> 40 #include <sys/mount.h> 41 #include <sys/vnode.h> 42 #include <sys/kmem.h> 43 #include <sys/kauth.h> 44 #include <sys/fstrans.h> 45 #include <sys/buf.h> 46 47 #include <miscfs/genfs/genfs.h> 48 #include <miscfs/genfs/genfs_node.h> 49 #include <miscfs/specfs/specdev.h> 50 #include <miscfs/syncfs/syncfs.h> 51 52 #include <uvm/uvm.h> 53 #include <uvm/uvm_pager.h> 54 55 static int genfs_do_directio(struct vmspace *, vaddr_t, size_t, struct vnode *, 56 off_t, enum uio_rw); 57 static void genfs_dio_iodone(struct buf *); 58 59 static int genfs_do_io(struct vnode *, off_t, vaddr_t, size_t, int, enum uio_rw, 60 void (*)(struct buf *)); 61 static void genfs_rel_pages(struct vm_page **, unsigned int); 62 static void genfs_markdirty(struct vnode *); 63 64 int genfs_maxdio = MAXPHYS; 65 66 static void 67 genfs_rel_pages(struct vm_page **pgs, unsigned int npages) 68 { 69 unsigned int i; 70 71 for (i = 0; i < npages; i++) { 72 struct vm_page *pg = pgs[i]; 73 74 if (pg == NULL || pg == PGO_DONTCARE) 75 continue; 76 KASSERT(uvm_page_locked_p(pg)); 77 if (pg->flags & PG_FAKE) { 78 pg->flags |= PG_RELEASED; 79 } 80 } 81 mutex_enter(&uvm_pageqlock); 82 uvm_page_unbusy(pgs, npages); 83 mutex_exit(&uvm_pageqlock); 84 } 85 86 static void 87 genfs_markdirty(struct vnode *vp) 88 { 89 struct genfs_node * const gp = VTOG(vp); 90 91 KASSERT(mutex_owned(vp->v_interlock)); 92 gp->g_dirtygen++; 93 if ((vp->v_iflag & VI_ONWORKLST) == 0) { 94 vn_syncer_add_to_worklist(vp, filedelay); 95 } 96 if ((vp->v_iflag & (VI_WRMAP|VI_WRMAPDIRTY)) == VI_WRMAP) { 97 vp->v_iflag |= VI_WRMAPDIRTY; 98 } 99 } 100 101 /* 102 * generic VM getpages routine. 103 * Return PG_BUSY pages for the given range, 104 * reading from backing store if necessary. 105 */ 106 107 int 108 genfs_getpages(void *v) 109 { 110 struct vop_getpages_args /* { 111 struct vnode *a_vp; 112 voff_t a_offset; 113 struct vm_page **a_m; 114 int *a_count; 115 int a_centeridx; 116 vm_prot_t a_access_type; 117 int a_advice; 118 int a_flags; 119 } */ * const ap = v; 120 121 off_t diskeof, memeof; 122 int i, error, npages; 123 const int flags = ap->a_flags; 124 struct vnode * const vp = ap->a_vp; 125 struct uvm_object * const uobj = &vp->v_uobj; 126 kauth_cred_t const cred = curlwp->l_cred; /* XXXUBC curlwp */ 127 const bool async = (flags & PGO_SYNCIO) == 0; 128 const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0; 129 const bool overwrite = (flags & PGO_OVERWRITE) != 0; 130 const bool blockalloc = memwrite && (flags & PGO_NOBLOCKALLOC) == 0; 131 const bool glocked = (flags & PGO_GLOCKHELD) != 0; 132 const bool need_wapbl = blockalloc && vp->v_mount->mnt_wapbl; 133 bool has_trans_wapbl = false; 134 UVMHIST_FUNC("genfs_getpages"); UVMHIST_CALLED(ubchist); 135 136 UVMHIST_LOG(ubchist, "vp %p off 0x%x/%x count %d", 137 vp, ap->a_offset >> 32, ap->a_offset, *ap->a_count); 138 139 KASSERT(vp->v_type == VREG || vp->v_type == VDIR || 140 vp->v_type == VLNK || vp->v_type == VBLK); 141 142 startover: 143 error = 0; 144 const voff_t origvsize = vp->v_size; 145 const off_t origoffset = ap->a_offset; 146 const int orignpages = *ap->a_count; 147 148 GOP_SIZE(vp, origvsize, &diskeof, 0); 149 if (flags & PGO_PASTEOF) { 150 off_t newsize; 151 #if defined(DIAGNOSTIC) 152 off_t writeeof; 153 #endif /* defined(DIAGNOSTIC) */ 154 155 newsize = MAX(origvsize, 156 origoffset + (orignpages << PAGE_SHIFT)); 157 GOP_SIZE(vp, newsize, &memeof, GOP_SIZE_MEM); 158 #if defined(DIAGNOSTIC) 159 GOP_SIZE(vp, vp->v_writesize, &writeeof, GOP_SIZE_MEM); 160 if (newsize > round_page(writeeof)) { 161 panic("%s: past eof: %" PRId64 " vs. %" PRId64, 162 __func__, newsize, round_page(writeeof)); 163 } 164 #endif /* defined(DIAGNOSTIC) */ 165 } else { 166 GOP_SIZE(vp, origvsize, &memeof, GOP_SIZE_MEM); 167 } 168 KASSERT(ap->a_centeridx >= 0 || ap->a_centeridx <= orignpages); 169 KASSERT((origoffset & (PAGE_SIZE - 1)) == 0 && origoffset >= 0); 170 KASSERT(orignpages > 0); 171 172 /* 173 * Bounds-check the request. 174 */ 175 176 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= memeof) { 177 if ((flags & PGO_LOCKED) == 0) { 178 mutex_exit(uobj->vmobjlock); 179 } 180 UVMHIST_LOG(ubchist, "off 0x%x count %d goes past EOF 0x%x", 181 origoffset, *ap->a_count, memeof,0); 182 error = EINVAL; 183 goto out_err; 184 } 185 186 /* uobj is locked */ 187 188 if ((flags & PGO_NOTIMESTAMP) == 0 && 189 (vp->v_type != VBLK || 190 (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) { 191 int updflags = 0; 192 193 if ((vp->v_mount->mnt_flag & MNT_NOATIME) == 0) { 194 updflags = GOP_UPDATE_ACCESSED; 195 } 196 if (memwrite) { 197 updflags |= GOP_UPDATE_MODIFIED; 198 } 199 if (updflags != 0) { 200 GOP_MARKUPDATE(vp, updflags); 201 } 202 } 203 204 /* 205 * For PGO_LOCKED requests, just return whatever's in memory. 206 */ 207 208 if (flags & PGO_LOCKED) { 209 int nfound; 210 struct vm_page *pg; 211 212 KASSERT(!glocked); 213 npages = *ap->a_count; 214 #if defined(DEBUG) 215 for (i = 0; i < npages; i++) { 216 pg = ap->a_m[i]; 217 KASSERT(pg == NULL || pg == PGO_DONTCARE); 218 } 219 #endif /* defined(DEBUG) */ 220 nfound = uvn_findpages(uobj, origoffset, &npages, 221 ap->a_m, UFP_NOWAIT|UFP_NOALLOC|(memwrite ? UFP_NORDONLY : 0)); 222 KASSERT(npages == *ap->a_count); 223 if (nfound == 0) { 224 error = EBUSY; 225 goto out_err; 226 } 227 if (!genfs_node_rdtrylock(vp)) { 228 genfs_rel_pages(ap->a_m, npages); 229 230 /* 231 * restore the array. 232 */ 233 234 for (i = 0; i < npages; i++) { 235 pg = ap->a_m[i]; 236 237 if (pg != NULL && pg != PGO_DONTCARE) { 238 ap->a_m[i] = NULL; 239 } 240 KASSERT(ap->a_m[i] == NULL || 241 ap->a_m[i] == PGO_DONTCARE); 242 } 243 } else { 244 genfs_node_unlock(vp); 245 } 246 error = (ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0); 247 if (error == 0 && memwrite) { 248 genfs_markdirty(vp); 249 } 250 goto out_err; 251 } 252 mutex_exit(uobj->vmobjlock); 253 254 /* 255 * find the requested pages and make some simple checks. 256 * leave space in the page array for a whole block. 257 */ 258 259 const int fs_bshift = (vp->v_type != VBLK) ? 260 vp->v_mount->mnt_fs_bshift : DEV_BSHIFT; 261 const int dev_bshift = (vp->v_type != VBLK) ? 262 vp->v_mount->mnt_dev_bshift : DEV_BSHIFT; 263 const int fs_bsize = 1 << fs_bshift; 264 #define blk_mask (fs_bsize - 1) 265 #define trunc_blk(x) ((x) & ~blk_mask) 266 #define round_blk(x) (((x) + blk_mask) & ~blk_mask) 267 268 const int orignmempages = MIN(orignpages, 269 round_page(memeof - origoffset) >> PAGE_SHIFT); 270 npages = orignmempages; 271 const off_t startoffset = trunc_blk(origoffset); 272 const off_t endoffset = MIN( 273 round_page(round_blk(origoffset + (npages << PAGE_SHIFT))), 274 round_page(memeof)); 275 const int ridx = (origoffset - startoffset) >> PAGE_SHIFT; 276 277 const int pgs_size = sizeof(struct vm_page *) * 278 ((endoffset - startoffset) >> PAGE_SHIFT); 279 struct vm_page **pgs, *pgs_onstack[UBC_MAX_PAGES]; 280 281 if (pgs_size > sizeof(pgs_onstack)) { 282 pgs = kmem_zalloc(pgs_size, async ? KM_NOSLEEP : KM_SLEEP); 283 if (pgs == NULL) { 284 pgs = pgs_onstack; 285 error = ENOMEM; 286 goto out_err; 287 } 288 } else { 289 pgs = pgs_onstack; 290 (void)memset(pgs, 0, pgs_size); 291 } 292 293 UVMHIST_LOG(ubchist, "ridx %d npages %d startoff %ld endoff %ld", 294 ridx, npages, startoffset, endoffset); 295 296 if (!has_trans_wapbl) { 297 fstrans_start(vp->v_mount, FSTRANS_SHARED); 298 /* 299 * XXX: This assumes that we come here only via 300 * the mmio path 301 */ 302 if (need_wapbl) { 303 error = WAPBL_BEGIN(vp->v_mount); 304 if (error) { 305 fstrans_done(vp->v_mount); 306 goto out_err_free; 307 } 308 } 309 has_trans_wapbl = true; 310 } 311 312 /* 313 * hold g_glock to prevent a race with truncate. 314 * 315 * check if our idea of v_size is still valid. 316 */ 317 318 KASSERT(!glocked || genfs_node_wrlocked(vp)); 319 if (!glocked) { 320 if (blockalloc) { 321 genfs_node_wrlock(vp); 322 } else { 323 genfs_node_rdlock(vp); 324 } 325 } 326 mutex_enter(uobj->vmobjlock); 327 if (vp->v_size < origvsize) { 328 if (!glocked) { 329 genfs_node_unlock(vp); 330 } 331 if (pgs != pgs_onstack) 332 kmem_free(pgs, pgs_size); 333 goto startover; 334 } 335 336 if (uvn_findpages(uobj, origoffset, &npages, &pgs[ridx], 337 async ? UFP_NOWAIT : UFP_ALL) != orignmempages) { 338 if (!glocked) { 339 genfs_node_unlock(vp); 340 } 341 KASSERT(async != 0); 342 genfs_rel_pages(&pgs[ridx], orignmempages); 343 mutex_exit(uobj->vmobjlock); 344 error = EBUSY; 345 goto out_err_free; 346 } 347 348 /* 349 * if the pages are already resident, just return them. 350 */ 351 352 for (i = 0; i < npages; i++) { 353 struct vm_page *pg = pgs[ridx + i]; 354 355 if ((pg->flags & PG_FAKE) || 356 (blockalloc && (pg->flags & PG_RDONLY))) { 357 break; 358 } 359 } 360 if (i == npages) { 361 if (!glocked) { 362 genfs_node_unlock(vp); 363 } 364 UVMHIST_LOG(ubchist, "returning cached pages", 0,0,0,0); 365 npages += ridx; 366 goto out; 367 } 368 369 /* 370 * if PGO_OVERWRITE is set, don't bother reading the pages. 371 */ 372 373 if (overwrite) { 374 if (!glocked) { 375 genfs_node_unlock(vp); 376 } 377 UVMHIST_LOG(ubchist, "PGO_OVERWRITE",0,0,0,0); 378 379 for (i = 0; i < npages; i++) { 380 struct vm_page *pg = pgs[ridx + i]; 381 382 pg->flags &= ~(PG_RDONLY|PG_CLEAN); 383 } 384 npages += ridx; 385 goto out; 386 } 387 388 /* 389 * the page wasn't resident and we're not overwriting, 390 * so we're going to have to do some i/o. 391 * find any additional pages needed to cover the expanded range. 392 */ 393 394 npages = (endoffset - startoffset) >> PAGE_SHIFT; 395 if (startoffset != origoffset || npages != orignmempages) { 396 int npgs; 397 398 /* 399 * we need to avoid deadlocks caused by locking 400 * additional pages at lower offsets than pages we 401 * already have locked. unlock them all and start over. 402 */ 403 404 genfs_rel_pages(&pgs[ridx], orignmempages); 405 memset(pgs, 0, pgs_size); 406 407 UVMHIST_LOG(ubchist, "reset npages start 0x%x end 0x%x", 408 startoffset, endoffset, 0,0); 409 npgs = npages; 410 if (uvn_findpages(uobj, startoffset, &npgs, pgs, 411 async ? UFP_NOWAIT : UFP_ALL) != npages) { 412 if (!glocked) { 413 genfs_node_unlock(vp); 414 } 415 KASSERT(async != 0); 416 genfs_rel_pages(pgs, npages); 417 mutex_exit(uobj->vmobjlock); 418 error = EBUSY; 419 goto out_err_free; 420 } 421 } 422 423 mutex_exit(uobj->vmobjlock); 424 425 { 426 size_t bytes, iobytes, tailstart, tailbytes, totalbytes, skipbytes; 427 vaddr_t kva; 428 struct buf *bp, *mbp; 429 bool sawhole = false; 430 431 /* 432 * read the desired page(s). 433 */ 434 435 totalbytes = npages << PAGE_SHIFT; 436 bytes = MIN(totalbytes, MAX(diskeof - startoffset, 0)); 437 tailbytes = totalbytes - bytes; 438 skipbytes = 0; 439 440 kva = uvm_pagermapin(pgs, npages, 441 UVMPAGER_MAPIN_READ | (async ? 0 : UVMPAGER_MAPIN_WAITOK)); 442 if (kva == 0) { 443 error = EBUSY; 444 goto mapin_fail; 445 } 446 447 mbp = getiobuf(vp, true); 448 mbp->b_bufsize = totalbytes; 449 mbp->b_data = (void *)kva; 450 mbp->b_resid = mbp->b_bcount = bytes; 451 mbp->b_cflags = BC_BUSY; 452 if (async) { 453 mbp->b_flags = B_READ | B_ASYNC; 454 mbp->b_iodone = uvm_aio_biodone; 455 } else { 456 mbp->b_flags = B_READ; 457 mbp->b_iodone = NULL; 458 } 459 if (async) 460 BIO_SETPRIO(mbp, BPRIO_TIMELIMITED); 461 else 462 BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL); 463 464 /* 465 * if EOF is in the middle of the range, zero the part past EOF. 466 * skip over pages which are not PG_FAKE since in that case they have 467 * valid data that we need to preserve. 468 */ 469 470 tailstart = bytes; 471 while (tailbytes > 0) { 472 const int len = PAGE_SIZE - (tailstart & PAGE_MASK); 473 474 KASSERT(len <= tailbytes); 475 if ((pgs[tailstart >> PAGE_SHIFT]->flags & PG_FAKE) != 0) { 476 memset((void *)(kva + tailstart), 0, len); 477 UVMHIST_LOG(ubchist, "tailbytes %p 0x%x 0x%x", 478 kva, tailstart, len, 0); 479 } 480 tailstart += len; 481 tailbytes -= len; 482 } 483 484 /* 485 * now loop over the pages, reading as needed. 486 */ 487 488 bp = NULL; 489 off_t offset; 490 for (offset = startoffset; 491 bytes > 0; 492 offset += iobytes, bytes -= iobytes) { 493 int run; 494 daddr_t lbn, blkno; 495 int pidx; 496 struct vnode *devvp; 497 498 /* 499 * skip pages which don't need to be read. 500 */ 501 502 pidx = (offset - startoffset) >> PAGE_SHIFT; 503 while ((pgs[pidx]->flags & PG_FAKE) == 0) { 504 size_t b; 505 506 KASSERT((offset & (PAGE_SIZE - 1)) == 0); 507 if ((pgs[pidx]->flags & PG_RDONLY)) { 508 sawhole = true; 509 } 510 b = MIN(PAGE_SIZE, bytes); 511 offset += b; 512 bytes -= b; 513 skipbytes += b; 514 pidx++; 515 UVMHIST_LOG(ubchist, "skipping, new offset 0x%x", 516 offset, 0,0,0); 517 if (bytes == 0) { 518 goto loopdone; 519 } 520 } 521 522 /* 523 * bmap the file to find out the blkno to read from and 524 * how much we can read in one i/o. if bmap returns an error, 525 * skip the rest of the top-level i/o. 526 */ 527 528 lbn = offset >> fs_bshift; 529 error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run); 530 if (error) { 531 UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%x -> %d\n", 532 lbn,error,0,0); 533 skipbytes += bytes; 534 bytes = 0; 535 goto loopdone; 536 } 537 538 /* 539 * see how many pages can be read with this i/o. 540 * reduce the i/o size if necessary to avoid 541 * overwriting pages with valid data. 542 */ 543 544 iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset, 545 bytes); 546 if (offset + iobytes > round_page(offset)) { 547 int pcount; 548 549 pcount = 1; 550 while (pidx + pcount < npages && 551 pgs[pidx + pcount]->flags & PG_FAKE) { 552 pcount++; 553 } 554 iobytes = MIN(iobytes, (pcount << PAGE_SHIFT) - 555 (offset - trunc_page(offset))); 556 } 557 558 /* 559 * if this block isn't allocated, zero it instead of 560 * reading it. unless we are going to allocate blocks, 561 * mark the pages we zeroed PG_RDONLY. 562 */ 563 564 if (blkno == (daddr_t)-1) { 565 int holepages = (round_page(offset + iobytes) - 566 trunc_page(offset)) >> PAGE_SHIFT; 567 UVMHIST_LOG(ubchist, "lbn 0x%x -> HOLE", lbn,0,0,0); 568 569 sawhole = true; 570 memset((char *)kva + (offset - startoffset), 0, 571 iobytes); 572 skipbytes += iobytes; 573 574 mutex_enter(uobj->vmobjlock); 575 for (i = 0; i < holepages; i++) { 576 if (memwrite) { 577 pgs[pidx + i]->flags &= ~PG_CLEAN; 578 } 579 if (!blockalloc) { 580 pgs[pidx + i]->flags |= PG_RDONLY; 581 } 582 } 583 mutex_exit(uobj->vmobjlock); 584 continue; 585 } 586 587 /* 588 * allocate a sub-buf for this piece of the i/o 589 * (or just use mbp if there's only 1 piece), 590 * and start it going. 591 */ 592 593 if (offset == startoffset && iobytes == bytes) { 594 bp = mbp; 595 } else { 596 UVMHIST_LOG(ubchist, "vp %p bp %p num now %d", 597 vp, bp, vp->v_numoutput, 0); 598 bp = getiobuf(vp, true); 599 nestiobuf_setup(mbp, bp, offset - startoffset, iobytes); 600 } 601 bp->b_lblkno = 0; 602 603 /* adjust physical blkno for partial blocks */ 604 bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >> 605 dev_bshift); 606 607 UVMHIST_LOG(ubchist, 608 "bp %p offset 0x%x bcount 0x%x blkno 0x%x", 609 bp, offset, bp->b_bcount, bp->b_blkno); 610 611 VOP_STRATEGY(devvp, bp); 612 } 613 614 loopdone: 615 nestiobuf_done(mbp, skipbytes, error); 616 if (async) { 617 UVMHIST_LOG(ubchist, "returning 0 (async)",0,0,0,0); 618 if (!glocked) { 619 genfs_node_unlock(vp); 620 } 621 error = 0; 622 goto out_err_free; 623 } 624 if (bp != NULL) { 625 error = biowait(mbp); 626 } 627 628 /* Remove the mapping (make KVA available as soon as possible) */ 629 uvm_pagermapout(kva, npages); 630 631 /* 632 * if this we encountered a hole then we have to do a little more work. 633 * for read faults, we marked the page PG_RDONLY so that future 634 * write accesses to the page will fault again. 635 * for write faults, we must make sure that the backing store for 636 * the page is completely allocated while the pages are locked. 637 */ 638 639 if (!error && sawhole && blockalloc) { 640 error = GOP_ALLOC(vp, startoffset, 641 npages << PAGE_SHIFT, 0, cred); 642 UVMHIST_LOG(ubchist, "gop_alloc off 0x%x/0x%x -> %d", 643 startoffset, npages << PAGE_SHIFT, error,0); 644 if (!error) { 645 mutex_enter(uobj->vmobjlock); 646 for (i = 0; i < npages; i++) { 647 struct vm_page *pg = pgs[i]; 648 649 if (pg == NULL) { 650 continue; 651 } 652 pg->flags &= ~(PG_CLEAN|PG_RDONLY); 653 UVMHIST_LOG(ubchist, "mark dirty pg %p", 654 pg,0,0,0); 655 } 656 mutex_exit(uobj->vmobjlock); 657 } 658 } 659 660 putiobuf(mbp); 661 } 662 663 mapin_fail: 664 if (!glocked) { 665 genfs_node_unlock(vp); 666 } 667 mutex_enter(uobj->vmobjlock); 668 669 /* 670 * we're almost done! release the pages... 671 * for errors, we free the pages. 672 * otherwise we activate them and mark them as valid and clean. 673 * also, unbusy pages that were not actually requested. 674 */ 675 676 if (error) { 677 genfs_rel_pages(pgs, npages); 678 mutex_exit(uobj->vmobjlock); 679 UVMHIST_LOG(ubchist, "returning error %d", error,0,0,0); 680 goto out_err_free; 681 } 682 683 out: 684 UVMHIST_LOG(ubchist, "succeeding, npages %d", npages,0,0,0); 685 error = 0; 686 mutex_enter(&uvm_pageqlock); 687 for (i = 0; i < npages; i++) { 688 struct vm_page *pg = pgs[i]; 689 if (pg == NULL) { 690 continue; 691 } 692 UVMHIST_LOG(ubchist, "examining pg %p flags 0x%x", 693 pg, pg->flags, 0,0); 694 if (pg->flags & PG_FAKE && !overwrite) { 695 pg->flags &= ~(PG_FAKE); 696 pmap_clear_modify(pgs[i]); 697 } 698 KASSERT(!memwrite || !blockalloc || (pg->flags & PG_RDONLY) == 0); 699 if (i < ridx || i >= ridx + orignmempages || async) { 700 UVMHIST_LOG(ubchist, "unbusy pg %p offset 0x%x", 701 pg, pg->offset,0,0); 702 if (pg->flags & PG_WANTED) { 703 wakeup(pg); 704 } 705 if (pg->flags & PG_FAKE) { 706 KASSERT(overwrite); 707 uvm_pagezero(pg); 708 } 709 if (pg->flags & PG_RELEASED) { 710 uvm_pagefree(pg); 711 continue; 712 } 713 uvm_pageenqueue(pg); 714 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE); 715 UVM_PAGE_OWN(pg, NULL); 716 } 717 } 718 mutex_exit(&uvm_pageqlock); 719 if (memwrite) { 720 genfs_markdirty(vp); 721 } 722 mutex_exit(uobj->vmobjlock); 723 if (ap->a_m != NULL) { 724 memcpy(ap->a_m, &pgs[ridx], 725 orignmempages * sizeof(struct vm_page *)); 726 } 727 728 out_err_free: 729 if (pgs != NULL && pgs != pgs_onstack) 730 kmem_free(pgs, pgs_size); 731 out_err: 732 if (has_trans_wapbl) { 733 if (need_wapbl) 734 WAPBL_END(vp->v_mount); 735 fstrans_done(vp->v_mount); 736 } 737 return error; 738 } 739 740 /* 741 * generic VM putpages routine. 742 * Write the given range of pages to backing store. 743 * 744 * => "offhi == 0" means flush all pages at or after "offlo". 745 * => object should be locked by caller. we return with the 746 * object unlocked. 747 * => if PGO_CLEANIT or PGO_SYNCIO is set, we may block (due to I/O). 748 * thus, a caller might want to unlock higher level resources 749 * (e.g. vm_map) before calling flush. 750 * => if neither PGO_CLEANIT nor PGO_SYNCIO is set, we will not block 751 * => if PGO_ALLPAGES is set, then all pages in the object will be processed. 752 * => NOTE: we rely on the fact that the object's memq is a TAILQ and 753 * that new pages are inserted on the tail end of the list. thus, 754 * we can make a complete pass through the object in one go by starting 755 * at the head and working towards the tail (new pages are put in 756 * front of us). 757 * => NOTE: we are allowed to lock the page queues, so the caller 758 * must not be holding the page queue lock. 759 * 760 * note on "cleaning" object and PG_BUSY pages: 761 * this routine is holding the lock on the object. the only time 762 * that it can run into a PG_BUSY page that it does not own is if 763 * some other process has started I/O on the page (e.g. either 764 * a pagein, or a pageout). if the PG_BUSY page is being paged 765 * in, then it can not be dirty (!PG_CLEAN) because no one has 766 * had a chance to modify it yet. if the PG_BUSY page is being 767 * paged out then it means that someone else has already started 768 * cleaning the page for us (how nice!). in this case, if we 769 * have syncio specified, then after we make our pass through the 770 * object we need to wait for the other PG_BUSY pages to clear 771 * off (i.e. we need to do an iosync). also note that once a 772 * page is PG_BUSY it must stay in its object until it is un-busyed. 773 * 774 * note on page traversal: 775 * we can traverse the pages in an object either by going down the 776 * linked list in "uobj->memq", or we can go over the address range 777 * by page doing hash table lookups for each address. depending 778 * on how many pages are in the object it may be cheaper to do one 779 * or the other. we set "by_list" to true if we are using memq. 780 * if the cost of a hash lookup was equal to the cost of the list 781 * traversal we could compare the number of pages in the start->stop 782 * range to the total number of pages in the object. however, it 783 * seems that a hash table lookup is more expensive than the linked 784 * list traversal, so we multiply the number of pages in the 785 * range by an estimate of the relatively higher cost of the hash lookup. 786 */ 787 788 int 789 genfs_putpages(void *v) 790 { 791 struct vop_putpages_args /* { 792 struct vnode *a_vp; 793 voff_t a_offlo; 794 voff_t a_offhi; 795 int a_flags; 796 } */ * const ap = v; 797 798 return genfs_do_putpages(ap->a_vp, ap->a_offlo, ap->a_offhi, 799 ap->a_flags, NULL); 800 } 801 802 int 803 genfs_do_putpages(struct vnode *vp, off_t startoff, off_t endoff, 804 int origflags, struct vm_page **busypg) 805 { 806 struct uvm_object * const uobj = &vp->v_uobj; 807 kmutex_t * const slock = uobj->vmobjlock; 808 off_t off; 809 /* Even for strange MAXPHYS, the shift rounds down to a page */ 810 #define maxpages (MAXPHYS >> PAGE_SHIFT) 811 int i, error, npages, nback; 812 int freeflag; 813 struct vm_page *pgs[maxpages], *pg, *nextpg, *tpg, curmp, endmp; 814 bool wasclean, by_list, needs_clean, yld; 815 bool async = (origflags & PGO_SYNCIO) == 0; 816 bool pagedaemon = curlwp == uvm.pagedaemon_lwp; 817 struct lwp * const l = curlwp ? curlwp : &lwp0; 818 struct genfs_node * const gp = VTOG(vp); 819 int flags; 820 int dirtygen; 821 bool modified; 822 bool need_wapbl; 823 bool has_trans; 824 bool cleanall; 825 bool onworklst; 826 827 UVMHIST_FUNC("genfs_putpages"); UVMHIST_CALLED(ubchist); 828 829 KASSERT(origflags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)); 830 KASSERT((startoff & PAGE_MASK) == 0 && (endoff & PAGE_MASK) == 0); 831 KASSERT(startoff < endoff || endoff == 0); 832 833 UVMHIST_LOG(ubchist, "vp %p pages %d off 0x%x len 0x%x", 834 vp, uobj->uo_npages, startoff, endoff - startoff); 835 836 has_trans = false; 837 need_wapbl = (!pagedaemon && vp->v_mount && vp->v_mount->mnt_wapbl && 838 (origflags & PGO_JOURNALLOCKED) == 0); 839 840 retry: 841 modified = false; 842 flags = origflags; 843 KASSERT((vp->v_iflag & VI_ONWORKLST) != 0 || 844 (vp->v_iflag & VI_WRMAPDIRTY) == 0); 845 if (uobj->uo_npages == 0) { 846 if (vp->v_iflag & VI_ONWORKLST) { 847 vp->v_iflag &= ~VI_WRMAPDIRTY; 848 if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL) 849 vn_syncer_remove_from_worklist(vp); 850 } 851 if (has_trans) { 852 if (need_wapbl) 853 WAPBL_END(vp->v_mount); 854 fstrans_done(vp->v_mount); 855 } 856 mutex_exit(slock); 857 return (0); 858 } 859 860 /* 861 * the vnode has pages, set up to process the request. 862 */ 863 864 if (!has_trans && (flags & PGO_CLEANIT) != 0) { 865 mutex_exit(slock); 866 if (pagedaemon) { 867 error = fstrans_start_nowait(vp->v_mount, FSTRANS_LAZY); 868 if (error) 869 return error; 870 } else 871 fstrans_start(vp->v_mount, FSTRANS_LAZY); 872 if (need_wapbl) { 873 error = WAPBL_BEGIN(vp->v_mount); 874 if (error) { 875 fstrans_done(vp->v_mount); 876 return error; 877 } 878 } 879 has_trans = true; 880 mutex_enter(slock); 881 goto retry; 882 } 883 884 error = 0; 885 wasclean = (vp->v_numoutput == 0); 886 off = startoff; 887 if (endoff == 0 || flags & PGO_ALLPAGES) { 888 endoff = trunc_page(LLONG_MAX); 889 } 890 by_list = (uobj->uo_npages <= 891 ((endoff - startoff) >> PAGE_SHIFT) * UVM_PAGE_TREE_PENALTY); 892 893 /* 894 * if this vnode is known not to have dirty pages, 895 * don't bother to clean it out. 896 */ 897 898 if ((vp->v_iflag & VI_ONWORKLST) == 0) { 899 #if !defined(DEBUG) 900 if ((flags & (PGO_FREE|PGO_DEACTIVATE)) == 0) { 901 goto skip_scan; 902 } 903 #endif /* !defined(DEBUG) */ 904 flags &= ~PGO_CLEANIT; 905 } 906 907 /* 908 * start the loop. when scanning by list, hold the last page 909 * in the list before we start. pages allocated after we start 910 * will be added to the end of the list, so we can stop at the 911 * current last page. 912 */ 913 914 cleanall = (flags & PGO_CLEANIT) != 0 && wasclean && 915 startoff == 0 && endoff == trunc_page(LLONG_MAX) && 916 (vp->v_iflag & VI_ONWORKLST) != 0; 917 dirtygen = gp->g_dirtygen; 918 freeflag = pagedaemon ? PG_PAGEOUT : PG_RELEASED; 919 if (by_list) { 920 curmp.flags = PG_MARKER; 921 endmp.flags = PG_MARKER; 922 pg = TAILQ_FIRST(&uobj->memq); 923 TAILQ_INSERT_TAIL(&uobj->memq, &endmp, listq.queue); 924 } else { 925 pg = uvm_pagelookup(uobj, off); 926 } 927 nextpg = NULL; 928 while (by_list || off < endoff) { 929 930 /* 931 * if the current page is not interesting, move on to the next. 932 */ 933 934 KASSERT(pg == NULL || pg->uobject == uobj || 935 (pg->flags & PG_MARKER) != 0); 936 KASSERT(pg == NULL || 937 (pg->flags & (PG_RELEASED|PG_PAGEOUT)) == 0 || 938 (pg->flags & (PG_BUSY|PG_MARKER)) != 0); 939 if (by_list) { 940 if (pg == &endmp) { 941 break; 942 } 943 if (pg->flags & PG_MARKER) { 944 pg = TAILQ_NEXT(pg, listq.queue); 945 continue; 946 } 947 if (pg->offset < startoff || pg->offset >= endoff || 948 pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 949 if (pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 950 wasclean = false; 951 } 952 pg = TAILQ_NEXT(pg, listq.queue); 953 continue; 954 } 955 off = pg->offset; 956 } else if (pg == NULL || pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 957 if (pg != NULL) { 958 wasclean = false; 959 } 960 off += PAGE_SIZE; 961 if (off < endoff) { 962 pg = uvm_pagelookup(uobj, off); 963 } 964 continue; 965 } 966 967 /* 968 * if the current page needs to be cleaned and it's busy, 969 * wait for it to become unbusy. 970 */ 971 972 yld = (l->l_cpu->ci_schedstate.spc_flags & 973 SPCF_SHOULDYIELD) && !pagedaemon; 974 if (pg->flags & PG_BUSY || yld) { 975 UVMHIST_LOG(ubchist, "busy %p", pg,0,0,0); 976 if (flags & PGO_BUSYFAIL && pg->flags & PG_BUSY) { 977 UVMHIST_LOG(ubchist, "busyfail %p", pg, 0,0,0); 978 error = EDEADLK; 979 if (busypg != NULL) 980 *busypg = pg; 981 break; 982 } 983 if (pagedaemon) { 984 /* 985 * someone has taken the page while we 986 * dropped the lock for fstrans_start. 987 */ 988 break; 989 } 990 if (by_list) { 991 TAILQ_INSERT_BEFORE(pg, &curmp, listq.queue); 992 UVMHIST_LOG(ubchist, "curmp next %p", 993 TAILQ_NEXT(&curmp, listq.queue), 0,0,0); 994 } 995 if (yld) { 996 mutex_exit(slock); 997 preempt(); 998 mutex_enter(slock); 999 } else { 1000 pg->flags |= PG_WANTED; 1001 UVM_UNLOCK_AND_WAIT(pg, slock, 0, "genput", 0); 1002 mutex_enter(slock); 1003 } 1004 if (by_list) { 1005 UVMHIST_LOG(ubchist, "after next %p", 1006 TAILQ_NEXT(&curmp, listq.queue), 0,0,0); 1007 pg = TAILQ_NEXT(&curmp, listq.queue); 1008 TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue); 1009 } else { 1010 pg = uvm_pagelookup(uobj, off); 1011 } 1012 continue; 1013 } 1014 1015 /* 1016 * if we're freeing, remove all mappings of the page now. 1017 * if we're cleaning, check if the page is needs to be cleaned. 1018 */ 1019 1020 if (flags & PGO_FREE) { 1021 pmap_page_protect(pg, VM_PROT_NONE); 1022 } else if (flags & PGO_CLEANIT) { 1023 1024 /* 1025 * if we still have some hope to pull this vnode off 1026 * from the syncer queue, write-protect the page. 1027 */ 1028 1029 if (cleanall && wasclean && 1030 gp->g_dirtygen == dirtygen) { 1031 1032 /* 1033 * uobj pages get wired only by uvm_fault 1034 * where uobj is locked. 1035 */ 1036 1037 if (pg->wire_count == 0) { 1038 pmap_page_protect(pg, 1039 VM_PROT_READ|VM_PROT_EXECUTE); 1040 } else { 1041 cleanall = false; 1042 } 1043 } 1044 } 1045 1046 if (flags & PGO_CLEANIT) { 1047 needs_clean = pmap_clear_modify(pg) || 1048 (pg->flags & PG_CLEAN) == 0; 1049 pg->flags |= PG_CLEAN; 1050 } else { 1051 needs_clean = false; 1052 } 1053 1054 /* 1055 * if we're cleaning, build a cluster. 1056 * the cluster will consist of pages which are currently dirty, 1057 * but they will be returned to us marked clean. 1058 * if not cleaning, just operate on the one page. 1059 */ 1060 1061 if (needs_clean) { 1062 KDASSERT((vp->v_iflag & VI_ONWORKLST)); 1063 wasclean = false; 1064 memset(pgs, 0, sizeof(pgs)); 1065 pg->flags |= PG_BUSY; 1066 UVM_PAGE_OWN(pg, "genfs_putpages"); 1067 1068 /* 1069 * first look backward. 1070 */ 1071 1072 npages = MIN(maxpages >> 1, off >> PAGE_SHIFT); 1073 nback = npages; 1074 uvn_findpages(uobj, off - PAGE_SIZE, &nback, &pgs[0], 1075 UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY|UFP_BACKWARD); 1076 if (nback) { 1077 memmove(&pgs[0], &pgs[npages - nback], 1078 nback * sizeof(pgs[0])); 1079 if (npages - nback < nback) 1080 memset(&pgs[nback], 0, 1081 (npages - nback) * sizeof(pgs[0])); 1082 else 1083 memset(&pgs[npages - nback], 0, 1084 nback * sizeof(pgs[0])); 1085 } 1086 1087 /* 1088 * then plug in our page of interest. 1089 */ 1090 1091 pgs[nback] = pg; 1092 1093 /* 1094 * then look forward to fill in the remaining space in 1095 * the array of pages. 1096 */ 1097 1098 npages = maxpages - nback - 1; 1099 uvn_findpages(uobj, off + PAGE_SIZE, &npages, 1100 &pgs[nback + 1], 1101 UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY); 1102 npages += nback + 1; 1103 } else { 1104 pgs[0] = pg; 1105 npages = 1; 1106 nback = 0; 1107 } 1108 1109 /* 1110 * apply FREE or DEACTIVATE options if requested. 1111 */ 1112 1113 if (flags & (PGO_DEACTIVATE|PGO_FREE)) { 1114 mutex_enter(&uvm_pageqlock); 1115 } 1116 for (i = 0; i < npages; i++) { 1117 tpg = pgs[i]; 1118 KASSERT(tpg->uobject == uobj); 1119 if (by_list && tpg == TAILQ_NEXT(pg, listq.queue)) 1120 pg = tpg; 1121 if (tpg->offset < startoff || tpg->offset >= endoff) 1122 continue; 1123 if (flags & PGO_DEACTIVATE && tpg->wire_count == 0) { 1124 uvm_pagedeactivate(tpg); 1125 } else if (flags & PGO_FREE) { 1126 pmap_page_protect(tpg, VM_PROT_NONE); 1127 if (tpg->flags & PG_BUSY) { 1128 tpg->flags |= freeflag; 1129 if (pagedaemon) { 1130 uvm_pageout_start(1); 1131 uvm_pagedequeue(tpg); 1132 } 1133 } else { 1134 1135 /* 1136 * ``page is not busy'' 1137 * implies that npages is 1 1138 * and needs_clean is false. 1139 */ 1140 1141 nextpg = TAILQ_NEXT(tpg, listq.queue); 1142 uvm_pagefree(tpg); 1143 if (pagedaemon) 1144 uvmexp.pdfreed++; 1145 } 1146 } 1147 } 1148 if (flags & (PGO_DEACTIVATE|PGO_FREE)) { 1149 mutex_exit(&uvm_pageqlock); 1150 } 1151 if (needs_clean) { 1152 modified = true; 1153 1154 /* 1155 * start the i/o. if we're traversing by list, 1156 * keep our place in the list with a marker page. 1157 */ 1158 1159 if (by_list) { 1160 TAILQ_INSERT_AFTER(&uobj->memq, pg, &curmp, 1161 listq.queue); 1162 } 1163 mutex_exit(slock); 1164 error = GOP_WRITE(vp, pgs, npages, flags); 1165 mutex_enter(slock); 1166 if (by_list) { 1167 pg = TAILQ_NEXT(&curmp, listq.queue); 1168 TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue); 1169 } 1170 if (error) { 1171 break; 1172 } 1173 if (by_list) { 1174 continue; 1175 } 1176 } 1177 1178 /* 1179 * find the next page and continue if there was no error. 1180 */ 1181 1182 if (by_list) { 1183 if (nextpg) { 1184 pg = nextpg; 1185 nextpg = NULL; 1186 } else { 1187 pg = TAILQ_NEXT(pg, listq.queue); 1188 } 1189 } else { 1190 off += (npages - nback) << PAGE_SHIFT; 1191 if (off < endoff) { 1192 pg = uvm_pagelookup(uobj, off); 1193 } 1194 } 1195 } 1196 if (by_list) { 1197 TAILQ_REMOVE(&uobj->memq, &endmp, listq.queue); 1198 } 1199 1200 if (modified && (vp->v_iflag & VI_WRMAPDIRTY) != 0 && 1201 (vp->v_type != VBLK || 1202 (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) { 1203 GOP_MARKUPDATE(vp, GOP_UPDATE_MODIFIED); 1204 } 1205 1206 /* 1207 * if we're cleaning and there was nothing to clean, 1208 * take us off the syncer list. if we started any i/o 1209 * and we're doing sync i/o, wait for all writes to finish. 1210 */ 1211 1212 if (cleanall && wasclean && gp->g_dirtygen == dirtygen && 1213 (vp->v_iflag & VI_ONWORKLST) != 0) { 1214 #if defined(DEBUG) 1215 TAILQ_FOREACH(pg, &uobj->memq, listq.queue) { 1216 if ((pg->flags & (PG_FAKE | PG_MARKER)) != 0) { 1217 continue; 1218 } 1219 if ((pg->flags & PG_CLEAN) == 0) { 1220 printf("%s: %p: !CLEAN\n", __func__, pg); 1221 } 1222 if (pmap_is_modified(pg)) { 1223 printf("%s: %p: modified\n", __func__, pg); 1224 } 1225 } 1226 #endif /* defined(DEBUG) */ 1227 vp->v_iflag &= ~VI_WRMAPDIRTY; 1228 if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL) 1229 vn_syncer_remove_from_worklist(vp); 1230 } 1231 1232 #if !defined(DEBUG) 1233 skip_scan: 1234 #endif /* !defined(DEBUG) */ 1235 1236 /* Wait for output to complete. */ 1237 if (!wasclean && !async && vp->v_numoutput != 0) { 1238 while (vp->v_numoutput != 0) 1239 cv_wait(&vp->v_cv, slock); 1240 } 1241 onworklst = (vp->v_iflag & VI_ONWORKLST) != 0; 1242 mutex_exit(slock); 1243 1244 if ((flags & PGO_RECLAIM) != 0 && onworklst) { 1245 /* 1246 * in the case of PGO_RECLAIM, ensure to make the vnode clean. 1247 * retrying is not a big deal because, in many cases, 1248 * uobj->uo_npages is already 0 here. 1249 */ 1250 mutex_enter(slock); 1251 goto retry; 1252 } 1253 1254 if (has_trans) { 1255 if (need_wapbl) 1256 WAPBL_END(vp->v_mount); 1257 fstrans_done(vp->v_mount); 1258 } 1259 1260 return (error); 1261 } 1262 1263 int 1264 genfs_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, int flags) 1265 { 1266 off_t off; 1267 vaddr_t kva; 1268 size_t len; 1269 int error; 1270 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1271 1272 UVMHIST_LOG(ubchist, "vp %p pgs %p npages %d flags 0x%x", 1273 vp, pgs, npages, flags); 1274 1275 off = pgs[0]->offset; 1276 kva = uvm_pagermapin(pgs, npages, 1277 UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK); 1278 len = npages << PAGE_SHIFT; 1279 1280 error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE, 1281 uvm_aio_biodone); 1282 1283 return error; 1284 } 1285 1286 int 1287 genfs_gop_write_rwmap(struct vnode *vp, struct vm_page **pgs, int npages, int flags) 1288 { 1289 off_t off; 1290 vaddr_t kva; 1291 size_t len; 1292 int error; 1293 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1294 1295 UVMHIST_LOG(ubchist, "vp %p pgs %p npages %d flags 0x%x", 1296 vp, pgs, npages, flags); 1297 1298 off = pgs[0]->offset; 1299 kva = uvm_pagermapin(pgs, npages, 1300 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 1301 len = npages << PAGE_SHIFT; 1302 1303 error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE, 1304 uvm_aio_biodone); 1305 1306 return error; 1307 } 1308 1309 /* 1310 * Backend routine for doing I/O to vnode pages. Pages are already locked 1311 * and mapped into kernel memory. Here we just look up the underlying 1312 * device block addresses and call the strategy routine. 1313 */ 1314 1315 static int 1316 genfs_do_io(struct vnode *vp, off_t off, vaddr_t kva, size_t len, int flags, 1317 enum uio_rw rw, void (*iodone)(struct buf *)) 1318 { 1319 int s, error; 1320 int fs_bshift, dev_bshift; 1321 off_t eof, offset, startoffset; 1322 size_t bytes, iobytes, skipbytes; 1323 struct buf *mbp, *bp; 1324 const bool async = (flags & PGO_SYNCIO) == 0; 1325 const bool lazy = (flags & PGO_LAZY) == 0; 1326 const bool iowrite = rw == UIO_WRITE; 1327 const int brw = iowrite ? B_WRITE : B_READ; 1328 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1329 1330 UVMHIST_LOG(ubchist, "vp %p kva %p len 0x%x flags 0x%x", 1331 vp, kva, len, flags); 1332 1333 KASSERT(vp->v_size <= vp->v_writesize); 1334 GOP_SIZE(vp, vp->v_writesize, &eof, 0); 1335 if (vp->v_type != VBLK) { 1336 fs_bshift = vp->v_mount->mnt_fs_bshift; 1337 dev_bshift = vp->v_mount->mnt_dev_bshift; 1338 } else { 1339 fs_bshift = DEV_BSHIFT; 1340 dev_bshift = DEV_BSHIFT; 1341 } 1342 error = 0; 1343 startoffset = off; 1344 bytes = MIN(len, eof - startoffset); 1345 skipbytes = 0; 1346 KASSERT(bytes != 0); 1347 1348 if (iowrite) { 1349 mutex_enter(vp->v_interlock); 1350 vp->v_numoutput += 2; 1351 mutex_exit(vp->v_interlock); 1352 } 1353 mbp = getiobuf(vp, true); 1354 UVMHIST_LOG(ubchist, "vp %p mbp %p num now %d bytes 0x%x", 1355 vp, mbp, vp->v_numoutput, bytes); 1356 mbp->b_bufsize = len; 1357 mbp->b_data = (void *)kva; 1358 mbp->b_resid = mbp->b_bcount = bytes; 1359 mbp->b_cflags = BC_BUSY | BC_AGE; 1360 if (async) { 1361 mbp->b_flags = brw | B_ASYNC; 1362 mbp->b_iodone = iodone; 1363 } else { 1364 mbp->b_flags = brw; 1365 mbp->b_iodone = NULL; 1366 } 1367 if (curlwp == uvm.pagedaemon_lwp) 1368 BIO_SETPRIO(mbp, BPRIO_TIMELIMITED); 1369 else if (async || lazy) 1370 BIO_SETPRIO(mbp, BPRIO_TIMENONCRITICAL); 1371 else 1372 BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL); 1373 1374 bp = NULL; 1375 for (offset = startoffset; 1376 bytes > 0; 1377 offset += iobytes, bytes -= iobytes) { 1378 int run; 1379 daddr_t lbn, blkno; 1380 struct vnode *devvp; 1381 1382 /* 1383 * bmap the file to find out the blkno to read from and 1384 * how much we can read in one i/o. if bmap returns an error, 1385 * skip the rest of the top-level i/o. 1386 */ 1387 1388 lbn = offset >> fs_bshift; 1389 error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run); 1390 if (error) { 1391 UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%x -> %d\n", 1392 lbn,error,0,0); 1393 skipbytes += bytes; 1394 bytes = 0; 1395 goto loopdone; 1396 } 1397 1398 /* 1399 * see how many pages can be read with this i/o. 1400 * reduce the i/o size if necessary to avoid 1401 * overwriting pages with valid data. 1402 */ 1403 1404 iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset, 1405 bytes); 1406 1407 /* 1408 * if this block isn't allocated, zero it instead of 1409 * reading it. unless we are going to allocate blocks, 1410 * mark the pages we zeroed PG_RDONLY. 1411 */ 1412 1413 if (blkno == (daddr_t)-1) { 1414 if (!iowrite) { 1415 memset((char *)kva + (offset - startoffset), 0, 1416 iobytes); 1417 } 1418 skipbytes += iobytes; 1419 continue; 1420 } 1421 1422 /* 1423 * allocate a sub-buf for this piece of the i/o 1424 * (or just use mbp if there's only 1 piece), 1425 * and start it going. 1426 */ 1427 1428 if (offset == startoffset && iobytes == bytes) { 1429 bp = mbp; 1430 } else { 1431 UVMHIST_LOG(ubchist, "vp %p bp %p num now %d", 1432 vp, bp, vp->v_numoutput, 0); 1433 bp = getiobuf(vp, true); 1434 nestiobuf_setup(mbp, bp, offset - startoffset, iobytes); 1435 } 1436 bp->b_lblkno = 0; 1437 1438 /* adjust physical blkno for partial blocks */ 1439 bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >> 1440 dev_bshift); 1441 1442 UVMHIST_LOG(ubchist, 1443 "bp %p offset 0x%x bcount 0x%x blkno 0x%x", 1444 bp, offset, bp->b_bcount, bp->b_blkno); 1445 1446 VOP_STRATEGY(devvp, bp); 1447 } 1448 1449 loopdone: 1450 if (skipbytes) { 1451 UVMHIST_LOG(ubchist, "skipbytes %d", skipbytes, 0,0,0); 1452 } 1453 nestiobuf_done(mbp, skipbytes, error); 1454 if (async) { 1455 UVMHIST_LOG(ubchist, "returning 0 (async)", 0,0,0,0); 1456 return (0); 1457 } 1458 UVMHIST_LOG(ubchist, "waiting for mbp %p", mbp,0,0,0); 1459 error = biowait(mbp); 1460 s = splbio(); 1461 (*iodone)(mbp); 1462 splx(s); 1463 UVMHIST_LOG(ubchist, "returning, error %d", error,0,0,0); 1464 return (error); 1465 } 1466 1467 int 1468 genfs_compat_getpages(void *v) 1469 { 1470 struct vop_getpages_args /* { 1471 struct vnode *a_vp; 1472 voff_t a_offset; 1473 struct vm_page **a_m; 1474 int *a_count; 1475 int a_centeridx; 1476 vm_prot_t a_access_type; 1477 int a_advice; 1478 int a_flags; 1479 } */ *ap = v; 1480 1481 off_t origoffset; 1482 struct vnode *vp = ap->a_vp; 1483 struct uvm_object *uobj = &vp->v_uobj; 1484 struct vm_page *pg, **pgs; 1485 vaddr_t kva; 1486 int i, error, orignpages, npages; 1487 struct iovec iov; 1488 struct uio uio; 1489 kauth_cred_t cred = curlwp->l_cred; 1490 const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0; 1491 1492 error = 0; 1493 origoffset = ap->a_offset; 1494 orignpages = *ap->a_count; 1495 pgs = ap->a_m; 1496 1497 if (ap->a_flags & PGO_LOCKED) { 1498 uvn_findpages(uobj, origoffset, ap->a_count, ap->a_m, 1499 UFP_NOWAIT|UFP_NOALLOC| (memwrite ? UFP_NORDONLY : 0)); 1500 1501 error = ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0; 1502 if (error == 0 && memwrite) { 1503 genfs_markdirty(vp); 1504 } 1505 return error; 1506 } 1507 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= vp->v_size) { 1508 mutex_exit(uobj->vmobjlock); 1509 return EINVAL; 1510 } 1511 if ((ap->a_flags & PGO_SYNCIO) == 0) { 1512 mutex_exit(uobj->vmobjlock); 1513 return 0; 1514 } 1515 npages = orignpages; 1516 uvn_findpages(uobj, origoffset, &npages, pgs, UFP_ALL); 1517 mutex_exit(uobj->vmobjlock); 1518 kva = uvm_pagermapin(pgs, npages, 1519 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 1520 for (i = 0; i < npages; i++) { 1521 pg = pgs[i]; 1522 if ((pg->flags & PG_FAKE) == 0) { 1523 continue; 1524 } 1525 iov.iov_base = (char *)kva + (i << PAGE_SHIFT); 1526 iov.iov_len = PAGE_SIZE; 1527 uio.uio_iov = &iov; 1528 uio.uio_iovcnt = 1; 1529 uio.uio_offset = origoffset + (i << PAGE_SHIFT); 1530 uio.uio_rw = UIO_READ; 1531 uio.uio_resid = PAGE_SIZE; 1532 UIO_SETUP_SYSSPACE(&uio); 1533 /* XXX vn_lock */ 1534 error = VOP_READ(vp, &uio, 0, cred); 1535 if (error) { 1536 break; 1537 } 1538 if (uio.uio_resid) { 1539 memset(iov.iov_base, 0, uio.uio_resid); 1540 } 1541 } 1542 uvm_pagermapout(kva, npages); 1543 mutex_enter(uobj->vmobjlock); 1544 mutex_enter(&uvm_pageqlock); 1545 for (i = 0; i < npages; i++) { 1546 pg = pgs[i]; 1547 if (error && (pg->flags & PG_FAKE) != 0) { 1548 pg->flags |= PG_RELEASED; 1549 } else { 1550 pmap_clear_modify(pg); 1551 uvm_pageactivate(pg); 1552 } 1553 } 1554 if (error) { 1555 uvm_page_unbusy(pgs, npages); 1556 } 1557 mutex_exit(&uvm_pageqlock); 1558 if (error == 0 && memwrite) { 1559 genfs_markdirty(vp); 1560 } 1561 mutex_exit(uobj->vmobjlock); 1562 return error; 1563 } 1564 1565 int 1566 genfs_compat_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, 1567 int flags) 1568 { 1569 off_t offset; 1570 struct iovec iov; 1571 struct uio uio; 1572 kauth_cred_t cred = curlwp->l_cred; 1573 struct buf *bp; 1574 vaddr_t kva; 1575 int error; 1576 1577 offset = pgs[0]->offset; 1578 kva = uvm_pagermapin(pgs, npages, 1579 UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK); 1580 1581 iov.iov_base = (void *)kva; 1582 iov.iov_len = npages << PAGE_SHIFT; 1583 uio.uio_iov = &iov; 1584 uio.uio_iovcnt = 1; 1585 uio.uio_offset = offset; 1586 uio.uio_rw = UIO_WRITE; 1587 uio.uio_resid = npages << PAGE_SHIFT; 1588 UIO_SETUP_SYSSPACE(&uio); 1589 /* XXX vn_lock */ 1590 error = VOP_WRITE(vp, &uio, 0, cred); 1591 1592 mutex_enter(vp->v_interlock); 1593 vp->v_numoutput++; 1594 mutex_exit(vp->v_interlock); 1595 1596 bp = getiobuf(vp, true); 1597 bp->b_cflags = BC_BUSY | BC_AGE; 1598 bp->b_lblkno = offset >> vp->v_mount->mnt_fs_bshift; 1599 bp->b_data = (char *)kva; 1600 bp->b_bcount = npages << PAGE_SHIFT; 1601 bp->b_bufsize = npages << PAGE_SHIFT; 1602 bp->b_resid = 0; 1603 bp->b_error = error; 1604 uvm_aio_aiodone(bp); 1605 return (error); 1606 } 1607 1608 /* 1609 * Process a uio using direct I/O. If we reach a part of the request 1610 * which cannot be processed in this fashion for some reason, just return. 1611 * The caller must handle some additional part of the request using 1612 * buffered I/O before trying direct I/O again. 1613 */ 1614 1615 void 1616 genfs_directio(struct vnode *vp, struct uio *uio, int ioflag) 1617 { 1618 struct vmspace *vs; 1619 struct iovec *iov; 1620 vaddr_t va; 1621 size_t len; 1622 const int mask = DEV_BSIZE - 1; 1623 int error; 1624 bool need_wapbl = (vp->v_mount && vp->v_mount->mnt_wapbl && 1625 (ioflag & IO_JOURNALLOCKED) == 0); 1626 1627 /* 1628 * We only support direct I/O to user space for now. 1629 */ 1630 1631 if (VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) { 1632 return; 1633 } 1634 1635 /* 1636 * If the vnode is mapped, we would need to get the getpages lock 1637 * to stabilize the bmap, but then we would get into trouble while 1638 * locking the pages if the pages belong to this same vnode (or a 1639 * multi-vnode cascade to the same effect). Just fall back to 1640 * buffered I/O if the vnode is mapped to avoid this mess. 1641 */ 1642 1643 if (vp->v_vflag & VV_MAPPED) { 1644 return; 1645 } 1646 1647 if (need_wapbl) { 1648 error = WAPBL_BEGIN(vp->v_mount); 1649 if (error) 1650 return; 1651 } 1652 1653 /* 1654 * Do as much of the uio as possible with direct I/O. 1655 */ 1656 1657 vs = uio->uio_vmspace; 1658 while (uio->uio_resid) { 1659 iov = uio->uio_iov; 1660 if (iov->iov_len == 0) { 1661 uio->uio_iov++; 1662 uio->uio_iovcnt--; 1663 continue; 1664 } 1665 va = (vaddr_t)iov->iov_base; 1666 len = MIN(iov->iov_len, genfs_maxdio); 1667 len &= ~mask; 1668 1669 /* 1670 * If the next chunk is smaller than DEV_BSIZE or extends past 1671 * the current EOF, then fall back to buffered I/O. 1672 */ 1673 1674 if (len == 0 || uio->uio_offset + len > vp->v_size) { 1675 break; 1676 } 1677 1678 /* 1679 * Check alignment. The file offset must be at least 1680 * sector-aligned. The exact constraint on memory alignment 1681 * is very hardware-dependent, but requiring sector-aligned 1682 * addresses there too is safe. 1683 */ 1684 1685 if (uio->uio_offset & mask || va & mask) { 1686 break; 1687 } 1688 error = genfs_do_directio(vs, va, len, vp, uio->uio_offset, 1689 uio->uio_rw); 1690 if (error) { 1691 break; 1692 } 1693 iov->iov_base = (char *)iov->iov_base + len; 1694 iov->iov_len -= len; 1695 uio->uio_offset += len; 1696 uio->uio_resid -= len; 1697 } 1698 1699 if (need_wapbl) 1700 WAPBL_END(vp->v_mount); 1701 } 1702 1703 /* 1704 * Iodone routine for direct I/O. We don't do much here since the request is 1705 * always synchronous, so the caller will do most of the work after biowait(). 1706 */ 1707 1708 static void 1709 genfs_dio_iodone(struct buf *bp) 1710 { 1711 1712 KASSERT((bp->b_flags & B_ASYNC) == 0); 1713 if ((bp->b_flags & B_READ) == 0 && (bp->b_cflags & BC_AGE) != 0) { 1714 mutex_enter(bp->b_objlock); 1715 vwakeup(bp); 1716 mutex_exit(bp->b_objlock); 1717 } 1718 putiobuf(bp); 1719 } 1720 1721 /* 1722 * Process one chunk of a direct I/O request. 1723 */ 1724 1725 static int 1726 genfs_do_directio(struct vmspace *vs, vaddr_t uva, size_t len, struct vnode *vp, 1727 off_t off, enum uio_rw rw) 1728 { 1729 struct vm_map *map; 1730 struct pmap *upm, *kpm; 1731 size_t klen = round_page(uva + len) - trunc_page(uva); 1732 off_t spoff, epoff; 1733 vaddr_t kva, puva; 1734 paddr_t pa; 1735 vm_prot_t prot; 1736 int error, rv, poff, koff; 1737 const int pgoflags = PGO_CLEANIT | PGO_SYNCIO | PGO_JOURNALLOCKED | 1738 (rw == UIO_WRITE ? PGO_FREE : 0); 1739 1740 /* 1741 * For writes, verify that this range of the file already has fully 1742 * allocated backing store. If there are any holes, just punt and 1743 * make the caller take the buffered write path. 1744 */ 1745 1746 if (rw == UIO_WRITE) { 1747 daddr_t lbn, elbn, blkno; 1748 int bsize, bshift, run; 1749 1750 bshift = vp->v_mount->mnt_fs_bshift; 1751 bsize = 1 << bshift; 1752 lbn = off >> bshift; 1753 elbn = (off + len + bsize - 1) >> bshift; 1754 while (lbn < elbn) { 1755 error = VOP_BMAP(vp, lbn, NULL, &blkno, &run); 1756 if (error) { 1757 return error; 1758 } 1759 if (blkno == (daddr_t)-1) { 1760 return ENOSPC; 1761 } 1762 lbn += 1 + run; 1763 } 1764 } 1765 1766 /* 1767 * Flush any cached pages for parts of the file that we're about to 1768 * access. If we're writing, invalidate pages as well. 1769 */ 1770 1771 spoff = trunc_page(off); 1772 epoff = round_page(off + len); 1773 mutex_enter(vp->v_interlock); 1774 error = VOP_PUTPAGES(vp, spoff, epoff, pgoflags); 1775 if (error) { 1776 return error; 1777 } 1778 1779 /* 1780 * Wire the user pages and remap them into kernel memory. 1781 */ 1782 1783 prot = rw == UIO_READ ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_READ; 1784 error = uvm_vslock(vs, (void *)uva, len, prot); 1785 if (error) { 1786 return error; 1787 } 1788 1789 map = &vs->vm_map; 1790 upm = vm_map_pmap(map); 1791 kpm = vm_map_pmap(kernel_map); 1792 puva = trunc_page(uva); 1793 kva = uvm_km_alloc(kernel_map, klen, atop(puva) & uvmexp.colormask, 1794 UVM_KMF_VAONLY | UVM_KMF_WAITVA | UVM_KMF_COLORMATCH); 1795 for (poff = 0; poff < klen; poff += PAGE_SIZE) { 1796 rv = pmap_extract(upm, puva + poff, &pa); 1797 KASSERT(rv); 1798 pmap_kenter_pa(kva + poff, pa, prot, PMAP_WIRED); 1799 } 1800 pmap_update(kpm); 1801 1802 /* 1803 * Do the I/O. 1804 */ 1805 1806 koff = uva - trunc_page(uva); 1807 error = genfs_do_io(vp, off, kva + koff, len, PGO_SYNCIO, rw, 1808 genfs_dio_iodone); 1809 1810 /* 1811 * Tear down the kernel mapping. 1812 */ 1813 1814 pmap_kremove(kva, klen); 1815 pmap_update(kpm); 1816 uvm_km_free(kernel_map, kva, klen, UVM_KMF_VAONLY); 1817 1818 /* 1819 * Unwire the user pages. 1820 */ 1821 1822 uvm_vsunlock(vs, (void *)uva, len); 1823 return error; 1824 } 1825