1 /* $NetBSD: genfs_io.c,v 1.9 2008/07/31 05:38:05 simonb Exp $ */ 2 3 /* 4 * Copyright (c) 1982, 1986, 1989, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the University nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 */ 32 33 #include <sys/cdefs.h> 34 __KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.9 2008/07/31 05:38:05 simonb Exp $"); 35 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/proc.h> 39 #include <sys/kernel.h> 40 #include <sys/mount.h> 41 #include <sys/namei.h> 42 #include <sys/vnode.h> 43 #include <sys/fcntl.h> 44 #include <sys/kmem.h> 45 #include <sys/poll.h> 46 #include <sys/mman.h> 47 #include <sys/file.h> 48 #include <sys/kauth.h> 49 #include <sys/fstrans.h> 50 51 #include <miscfs/genfs/genfs.h> 52 #include <miscfs/genfs/genfs_node.h> 53 #include <miscfs/specfs/specdev.h> 54 55 #include <uvm/uvm.h> 56 #include <uvm/uvm_pager.h> 57 58 static int genfs_do_directio(struct vmspace *, vaddr_t, size_t, struct vnode *, 59 off_t, enum uio_rw); 60 static void genfs_dio_iodone(struct buf *); 61 62 static int genfs_do_io(struct vnode *, off_t, vaddr_t, size_t, int, enum uio_rw, 63 void (*)(struct buf *)); 64 static inline void genfs_rel_pages(struct vm_page **, int); 65 66 #define MAX_READ_PAGES 16 /* XXXUBC 16 */ 67 68 int genfs_maxdio = MAXPHYS; 69 70 static inline void 71 genfs_rel_pages(struct vm_page **pgs, int npages) 72 { 73 int i; 74 75 for (i = 0; i < npages; i++) { 76 struct vm_page *pg = pgs[i]; 77 78 if (pg == NULL || pg == PGO_DONTCARE) 79 continue; 80 if (pg->flags & PG_FAKE) { 81 pg->flags |= PG_RELEASED; 82 } 83 } 84 mutex_enter(&uvm_pageqlock); 85 uvm_page_unbusy(pgs, npages); 86 mutex_exit(&uvm_pageqlock); 87 } 88 89 /* 90 * generic VM getpages routine. 91 * Return PG_BUSY pages for the given range, 92 * reading from backing store if necessary. 93 */ 94 95 int 96 genfs_getpages(void *v) 97 { 98 struct vop_getpages_args /* { 99 struct vnode *a_vp; 100 voff_t a_offset; 101 struct vm_page **a_m; 102 int *a_count; 103 int a_centeridx; 104 vm_prot_t a_access_type; 105 int a_advice; 106 int a_flags; 107 } */ *ap = v; 108 109 off_t newsize, diskeof, memeof; 110 off_t offset, origoffset, startoffset, endoffset; 111 daddr_t lbn, blkno; 112 int i, error, npages, orignpages, npgs, run, ridx, pidx, pcount; 113 int fs_bshift, fs_bsize, dev_bshift; 114 int flags = ap->a_flags; 115 size_t bytes, iobytes, tailstart, tailbytes, totalbytes, skipbytes; 116 vaddr_t kva; 117 struct buf *bp, *mbp; 118 struct vnode *vp = ap->a_vp; 119 struct vnode *devvp; 120 struct genfs_node *gp = VTOG(vp); 121 struct uvm_object *uobj = &vp->v_uobj; 122 struct vm_page *pg, **pgs, *pgs_onstack[MAX_READ_PAGES]; 123 int pgs_size; 124 kauth_cred_t cred = curlwp->l_cred; /* XXXUBC curlwp */ 125 bool async = (flags & PGO_SYNCIO) == 0; 126 bool write = (ap->a_access_type & VM_PROT_WRITE) != 0; 127 bool sawhole = false; 128 bool has_trans = false; 129 bool overwrite = (flags & PGO_OVERWRITE) != 0; 130 bool blockalloc = write && (flags & PGO_NOBLOCKALLOC) == 0; 131 voff_t origvsize; 132 UVMHIST_FUNC("genfs_getpages"); UVMHIST_CALLED(ubchist); 133 134 UVMHIST_LOG(ubchist, "vp %p off 0x%x/%x count %d", 135 vp, ap->a_offset >> 32, ap->a_offset, *ap->a_count); 136 137 KASSERT(vp->v_type == VREG || vp->v_type == VDIR || 138 vp->v_type == VLNK || vp->v_type == VBLK); 139 140 /* XXXUBC temp limit */ 141 if (*ap->a_count > MAX_READ_PAGES) { 142 panic("genfs_getpages: too many pages"); 143 } 144 145 pgs = pgs_onstack; 146 pgs_size = sizeof(pgs_onstack); 147 148 startover: 149 error = 0; 150 origvsize = vp->v_size; 151 origoffset = ap->a_offset; 152 orignpages = *ap->a_count; 153 GOP_SIZE(vp, origvsize, &diskeof, 0); 154 if (flags & PGO_PASTEOF) { 155 #if defined(DIAGNOSTIC) 156 off_t writeeof; 157 #endif /* defined(DIAGNOSTIC) */ 158 159 newsize = MAX(origvsize, 160 origoffset + (orignpages << PAGE_SHIFT)); 161 GOP_SIZE(vp, newsize, &memeof, GOP_SIZE_MEM); 162 #if defined(DIAGNOSTIC) 163 GOP_SIZE(vp, vp->v_writesize, &writeeof, GOP_SIZE_MEM); 164 if (newsize > round_page(writeeof)) { 165 panic("%s: past eof", __func__); 166 } 167 #endif /* defined(DIAGNOSTIC) */ 168 } else { 169 GOP_SIZE(vp, origvsize, &memeof, GOP_SIZE_MEM); 170 } 171 KASSERT(ap->a_centeridx >= 0 || ap->a_centeridx <= orignpages); 172 KASSERT((origoffset & (PAGE_SIZE - 1)) == 0 && origoffset >= 0); 173 KASSERT(orignpages > 0); 174 175 /* 176 * Bounds-check the request. 177 */ 178 179 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= memeof) { 180 if ((flags & PGO_LOCKED) == 0) { 181 mutex_exit(&uobj->vmobjlock); 182 } 183 UVMHIST_LOG(ubchist, "off 0x%x count %d goes past EOF 0x%x", 184 origoffset, *ap->a_count, memeof,0); 185 error = EINVAL; 186 goto out_err; 187 } 188 189 /* uobj is locked */ 190 191 if ((flags & PGO_NOTIMESTAMP) == 0 && 192 (vp->v_type != VBLK || 193 (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) { 194 int updflags = 0; 195 196 if ((vp->v_mount->mnt_flag & MNT_NOATIME) == 0) { 197 updflags = GOP_UPDATE_ACCESSED; 198 } 199 if (write) { 200 updflags |= GOP_UPDATE_MODIFIED; 201 } 202 if (updflags != 0) { 203 GOP_MARKUPDATE(vp, updflags); 204 } 205 } 206 207 if (write) { 208 gp->g_dirtygen++; 209 if ((vp->v_iflag & VI_ONWORKLST) == 0) { 210 vn_syncer_add_to_worklist(vp, filedelay); 211 } 212 if ((vp->v_iflag & (VI_WRMAP|VI_WRMAPDIRTY)) == VI_WRMAP) { 213 vp->v_iflag |= VI_WRMAPDIRTY; 214 } 215 } 216 217 /* 218 * For PGO_LOCKED requests, just return whatever's in memory. 219 */ 220 221 if (flags & PGO_LOCKED) { 222 int nfound; 223 224 npages = *ap->a_count; 225 #if defined(DEBUG) 226 for (i = 0; i < npages; i++) { 227 pg = ap->a_m[i]; 228 KASSERT(pg == NULL || pg == PGO_DONTCARE); 229 } 230 #endif /* defined(DEBUG) */ 231 nfound = uvn_findpages(uobj, origoffset, &npages, 232 ap->a_m, UFP_NOWAIT|UFP_NOALLOC|(write ? UFP_NORDONLY : 0)); 233 KASSERT(npages == *ap->a_count); 234 if (nfound == 0) { 235 error = EBUSY; 236 goto out_err; 237 } 238 if (!rw_tryenter(&gp->g_glock, RW_READER)) { 239 genfs_rel_pages(ap->a_m, npages); 240 241 /* 242 * restore the array. 243 */ 244 245 for (i = 0; i < npages; i++) { 246 pg = ap->a_m[i]; 247 248 if (pg != NULL || pg != PGO_DONTCARE) { 249 ap->a_m[i] = NULL; 250 } 251 } 252 } else { 253 rw_exit(&gp->g_glock); 254 } 255 error = (ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0); 256 goto out_err; 257 } 258 mutex_exit(&uobj->vmobjlock); 259 260 /* 261 * find the requested pages and make some simple checks. 262 * leave space in the page array for a whole block. 263 */ 264 265 if (vp->v_type != VBLK) { 266 fs_bshift = vp->v_mount->mnt_fs_bshift; 267 dev_bshift = vp->v_mount->mnt_dev_bshift; 268 } else { 269 fs_bshift = DEV_BSHIFT; 270 dev_bshift = DEV_BSHIFT; 271 } 272 fs_bsize = 1 << fs_bshift; 273 274 orignpages = MIN(orignpages, 275 round_page(memeof - origoffset) >> PAGE_SHIFT); 276 npages = orignpages; 277 startoffset = origoffset & ~(fs_bsize - 1); 278 endoffset = round_page((origoffset + (npages << PAGE_SHIFT) + 279 fs_bsize - 1) & ~(fs_bsize - 1)); 280 endoffset = MIN(endoffset, round_page(memeof)); 281 ridx = (origoffset - startoffset) >> PAGE_SHIFT; 282 283 pgs_size = sizeof(struct vm_page *) * 284 ((endoffset - startoffset) >> PAGE_SHIFT); 285 if (pgs_size > sizeof(pgs_onstack)) { 286 pgs = kmem_zalloc(pgs_size, async ? KM_NOSLEEP : KM_SLEEP); 287 if (pgs == NULL) { 288 pgs = pgs_onstack; 289 error = ENOMEM; 290 goto out_err; 291 } 292 } else { 293 /* pgs == pgs_onstack */ 294 memset(pgs, 0, pgs_size); 295 } 296 UVMHIST_LOG(ubchist, "ridx %d npages %d startoff %ld endoff %ld", 297 ridx, npages, startoffset, endoffset); 298 299 if (!has_trans) { 300 fstrans_start(vp->v_mount, FSTRANS_SHARED); 301 has_trans = true; 302 } 303 304 /* 305 * hold g_glock to prevent a race with truncate. 306 * 307 * check if our idea of v_size is still valid. 308 */ 309 310 if (blockalloc) { 311 rw_enter(&gp->g_glock, RW_WRITER); 312 } else { 313 rw_enter(&gp->g_glock, RW_READER); 314 } 315 mutex_enter(&uobj->vmobjlock); 316 if (vp->v_size < origvsize) { 317 rw_exit(&gp->g_glock); 318 if (pgs != pgs_onstack) 319 kmem_free(pgs, pgs_size); 320 goto startover; 321 } 322 323 if (uvn_findpages(uobj, origoffset, &npages, &pgs[ridx], 324 async ? UFP_NOWAIT : UFP_ALL) != orignpages) { 325 rw_exit(&gp->g_glock); 326 KASSERT(async != 0); 327 genfs_rel_pages(&pgs[ridx], orignpages); 328 mutex_exit(&uobj->vmobjlock); 329 error = EBUSY; 330 goto out_err; 331 } 332 333 /* 334 * if the pages are already resident, just return them. 335 */ 336 337 for (i = 0; i < npages; i++) { 338 struct vm_page *pg1 = pgs[ridx + i]; 339 340 if ((pg1->flags & PG_FAKE) || 341 (blockalloc && (pg1->flags & PG_RDONLY))) { 342 break; 343 } 344 } 345 if (i == npages) { 346 rw_exit(&gp->g_glock); 347 UVMHIST_LOG(ubchist, "returning cached pages", 0,0,0,0); 348 npages += ridx; 349 goto out; 350 } 351 352 /* 353 * if PGO_OVERWRITE is set, don't bother reading the pages. 354 */ 355 356 if (overwrite) { 357 rw_exit(&gp->g_glock); 358 UVMHIST_LOG(ubchist, "PGO_OVERWRITE",0,0,0,0); 359 360 for (i = 0; i < npages; i++) { 361 struct vm_page *pg1 = pgs[ridx + i]; 362 363 pg1->flags &= ~(PG_RDONLY|PG_CLEAN); 364 } 365 npages += ridx; 366 goto out; 367 } 368 369 /* 370 * the page wasn't resident and we're not overwriting, 371 * so we're going to have to do some i/o. 372 * find any additional pages needed to cover the expanded range. 373 */ 374 375 npages = (endoffset - startoffset) >> PAGE_SHIFT; 376 if (startoffset != origoffset || npages != orignpages) { 377 378 /* 379 * we need to avoid deadlocks caused by locking 380 * additional pages at lower offsets than pages we 381 * already have locked. unlock them all and start over. 382 */ 383 384 genfs_rel_pages(&pgs[ridx], orignpages); 385 memset(pgs, 0, pgs_size); 386 387 UVMHIST_LOG(ubchist, "reset npages start 0x%x end 0x%x", 388 startoffset, endoffset, 0,0); 389 npgs = npages; 390 if (uvn_findpages(uobj, startoffset, &npgs, pgs, 391 async ? UFP_NOWAIT : UFP_ALL) != npages) { 392 rw_exit(&gp->g_glock); 393 KASSERT(async != 0); 394 genfs_rel_pages(pgs, npages); 395 mutex_exit(&uobj->vmobjlock); 396 error = EBUSY; 397 goto out_err; 398 } 399 } 400 mutex_exit(&uobj->vmobjlock); 401 402 /* 403 * read the desired page(s). 404 */ 405 406 totalbytes = npages << PAGE_SHIFT; 407 bytes = MIN(totalbytes, MAX(diskeof - startoffset, 0)); 408 tailbytes = totalbytes - bytes; 409 skipbytes = 0; 410 411 kva = uvm_pagermapin(pgs, npages, 412 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 413 414 mbp = getiobuf(vp, true); 415 mbp->b_bufsize = totalbytes; 416 mbp->b_data = (void *)kva; 417 mbp->b_resid = mbp->b_bcount = bytes; 418 mbp->b_cflags = BC_BUSY; 419 if (async) { 420 mbp->b_flags = B_READ | B_ASYNC; 421 mbp->b_iodone = uvm_aio_biodone; 422 } else { 423 mbp->b_flags = B_READ; 424 mbp->b_iodone = NULL; 425 } 426 if (async) 427 BIO_SETPRIO(mbp, BPRIO_TIMELIMITED); 428 else 429 BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL); 430 431 /* 432 * if EOF is in the middle of the range, zero the part past EOF. 433 * skip over pages which are not PG_FAKE since in that case they have 434 * valid data that we need to preserve. 435 */ 436 437 tailstart = bytes; 438 while (tailbytes > 0) { 439 const int len = PAGE_SIZE - (tailstart & PAGE_MASK); 440 441 KASSERT(len <= tailbytes); 442 if ((pgs[tailstart >> PAGE_SHIFT]->flags & PG_FAKE) != 0) { 443 memset((void *)(kva + tailstart), 0, len); 444 UVMHIST_LOG(ubchist, "tailbytes %p 0x%x 0x%x", 445 kva, tailstart, len, 0); 446 } 447 tailstart += len; 448 tailbytes -= len; 449 } 450 451 /* 452 * now loop over the pages, reading as needed. 453 */ 454 455 bp = NULL; 456 for (offset = startoffset; 457 bytes > 0; 458 offset += iobytes, bytes -= iobytes) { 459 460 /* 461 * skip pages which don't need to be read. 462 */ 463 464 pidx = (offset - startoffset) >> PAGE_SHIFT; 465 while ((pgs[pidx]->flags & PG_FAKE) == 0) { 466 size_t b; 467 468 KASSERT((offset & (PAGE_SIZE - 1)) == 0); 469 if ((pgs[pidx]->flags & PG_RDONLY)) { 470 sawhole = true; 471 } 472 b = MIN(PAGE_SIZE, bytes); 473 offset += b; 474 bytes -= b; 475 skipbytes += b; 476 pidx++; 477 UVMHIST_LOG(ubchist, "skipping, new offset 0x%x", 478 offset, 0,0,0); 479 if (bytes == 0) { 480 goto loopdone; 481 } 482 } 483 484 /* 485 * bmap the file to find out the blkno to read from and 486 * how much we can read in one i/o. if bmap returns an error, 487 * skip the rest of the top-level i/o. 488 */ 489 490 lbn = offset >> fs_bshift; 491 error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run); 492 if (error) { 493 UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%x -> %d\n", 494 lbn, error,0,0); 495 skipbytes += bytes; 496 goto loopdone; 497 } 498 499 /* 500 * see how many pages can be read with this i/o. 501 * reduce the i/o size if necessary to avoid 502 * overwriting pages with valid data. 503 */ 504 505 iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset, 506 bytes); 507 if (offset + iobytes > round_page(offset)) { 508 pcount = 1; 509 while (pidx + pcount < npages && 510 pgs[pidx + pcount]->flags & PG_FAKE) { 511 pcount++; 512 } 513 iobytes = MIN(iobytes, (pcount << PAGE_SHIFT) - 514 (offset - trunc_page(offset))); 515 } 516 517 /* 518 * if this block isn't allocated, zero it instead of 519 * reading it. unless we are going to allocate blocks, 520 * mark the pages we zeroed PG_RDONLY. 521 */ 522 523 if (blkno < 0) { 524 int holepages = (round_page(offset + iobytes) - 525 trunc_page(offset)) >> PAGE_SHIFT; 526 UVMHIST_LOG(ubchist, "lbn 0x%x -> HOLE", lbn,0,0,0); 527 528 sawhole = true; 529 memset((char *)kva + (offset - startoffset), 0, 530 iobytes); 531 skipbytes += iobytes; 532 533 for (i = 0; i < holepages; i++) { 534 if (write) { 535 pgs[pidx + i]->flags &= ~PG_CLEAN; 536 } 537 if (!blockalloc) { 538 pgs[pidx + i]->flags |= PG_RDONLY; 539 } 540 } 541 continue; 542 } 543 544 /* 545 * allocate a sub-buf for this piece of the i/o 546 * (or just use mbp if there's only 1 piece), 547 * and start it going. 548 */ 549 550 if (offset == startoffset && iobytes == bytes) { 551 bp = mbp; 552 } else { 553 bp = getiobuf(vp, true); 554 nestiobuf_setup(mbp, bp, offset - startoffset, iobytes); 555 } 556 bp->b_lblkno = 0; 557 558 /* adjust physical blkno for partial blocks */ 559 bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >> 560 dev_bshift); 561 562 UVMHIST_LOG(ubchist, 563 "bp %p offset 0x%x bcount 0x%x blkno 0x%x", 564 bp, offset, iobytes, bp->b_blkno); 565 566 VOP_STRATEGY(devvp, bp); 567 } 568 569 loopdone: 570 nestiobuf_done(mbp, skipbytes, error); 571 if (async) { 572 UVMHIST_LOG(ubchist, "returning 0 (async)",0,0,0,0); 573 rw_exit(&gp->g_glock); 574 error = 0; 575 goto out_err; 576 } 577 if (bp != NULL) { 578 error = biowait(mbp); 579 } 580 putiobuf(mbp); 581 uvm_pagermapout(kva, npages); 582 583 /* 584 * if this we encountered a hole then we have to do a little more work. 585 * for read faults, we marked the page PG_RDONLY so that future 586 * write accesses to the page will fault again. 587 * for write faults, we must make sure that the backing store for 588 * the page is completely allocated while the pages are locked. 589 */ 590 591 if (!error && sawhole && blockalloc) { 592 /* 593 * XXX: This assumes that we come here only via 594 * the mmio path 595 */ 596 if (vp->v_mount->mnt_wapbl && write) { 597 error = WAPBL_BEGIN(vp->v_mount); 598 } 599 600 if (!error) { 601 error = GOP_ALLOC(vp, startoffset, 602 npages << PAGE_SHIFT, 0, cred); 603 if (vp->v_mount->mnt_wapbl && write) { 604 WAPBL_END(vp->v_mount); 605 } 606 } 607 608 UVMHIST_LOG(ubchist, "gop_alloc off 0x%x/0x%x -> %d", 609 startoffset, npages << PAGE_SHIFT, error,0); 610 if (!error) { 611 for (i = 0; i < npages; i++) { 612 if (pgs[i] == NULL) { 613 continue; 614 } 615 pgs[i]->flags &= ~(PG_CLEAN|PG_RDONLY); 616 UVMHIST_LOG(ubchist, "mark dirty pg %p", 617 pgs[i],0,0,0); 618 } 619 } 620 } 621 rw_exit(&gp->g_glock); 622 mutex_enter(&uobj->vmobjlock); 623 624 /* 625 * we're almost done! release the pages... 626 * for errors, we free the pages. 627 * otherwise we activate them and mark them as valid and clean. 628 * also, unbusy pages that were not actually requested. 629 */ 630 631 if (error) { 632 for (i = 0; i < npages; i++) { 633 if (pgs[i] == NULL) { 634 continue; 635 } 636 UVMHIST_LOG(ubchist, "examining pg %p flags 0x%x", 637 pgs[i], pgs[i]->flags, 0,0); 638 if (pgs[i]->flags & PG_FAKE) { 639 pgs[i]->flags |= PG_RELEASED; 640 } 641 } 642 mutex_enter(&uvm_pageqlock); 643 uvm_page_unbusy(pgs, npages); 644 mutex_exit(&uvm_pageqlock); 645 mutex_exit(&uobj->vmobjlock); 646 UVMHIST_LOG(ubchist, "returning error %d", error,0,0,0); 647 goto out_err; 648 } 649 650 out: 651 UVMHIST_LOG(ubchist, "succeeding, npages %d", npages,0,0,0); 652 error = 0; 653 mutex_enter(&uvm_pageqlock); 654 for (i = 0; i < npages; i++) { 655 pg = pgs[i]; 656 if (pg == NULL) { 657 continue; 658 } 659 UVMHIST_LOG(ubchist, "examining pg %p flags 0x%x", 660 pg, pg->flags, 0,0); 661 if (pg->flags & PG_FAKE && !overwrite) { 662 pg->flags &= ~(PG_FAKE); 663 pmap_clear_modify(pgs[i]); 664 } 665 KASSERT(!write || !blockalloc || (pg->flags & PG_RDONLY) == 0); 666 if (i < ridx || i >= ridx + orignpages || async) { 667 UVMHIST_LOG(ubchist, "unbusy pg %p offset 0x%x", 668 pg, pg->offset,0,0); 669 if (pg->flags & PG_WANTED) { 670 wakeup(pg); 671 } 672 if (pg->flags & PG_FAKE) { 673 KASSERT(overwrite); 674 uvm_pagezero(pg); 675 } 676 if (pg->flags & PG_RELEASED) { 677 uvm_pagefree(pg); 678 continue; 679 } 680 uvm_pageenqueue(pg); 681 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE); 682 UVM_PAGE_OWN(pg, NULL); 683 } 684 } 685 mutex_exit(&uvm_pageqlock); 686 mutex_exit(&uobj->vmobjlock); 687 if (ap->a_m != NULL) { 688 memcpy(ap->a_m, &pgs[ridx], 689 orignpages * sizeof(struct vm_page *)); 690 } 691 692 out_err: 693 if (pgs != pgs_onstack) 694 kmem_free(pgs, pgs_size); 695 if (has_trans) 696 fstrans_done(vp->v_mount); 697 return (error); 698 } 699 700 /* 701 * generic VM putpages routine. 702 * Write the given range of pages to backing store. 703 * 704 * => "offhi == 0" means flush all pages at or after "offlo". 705 * => object should be locked by caller. we return with the 706 * object unlocked. 707 * => if PGO_CLEANIT or PGO_SYNCIO is set, we may block (due to I/O). 708 * thus, a caller might want to unlock higher level resources 709 * (e.g. vm_map) before calling flush. 710 * => if neither PGO_CLEANIT nor PGO_SYNCIO is set, we will not block 711 * => if PGO_ALLPAGES is set, then all pages in the object will be processed. 712 * => NOTE: we rely on the fact that the object's memq is a TAILQ and 713 * that new pages are inserted on the tail end of the list. thus, 714 * we can make a complete pass through the object in one go by starting 715 * at the head and working towards the tail (new pages are put in 716 * front of us). 717 * => NOTE: we are allowed to lock the page queues, so the caller 718 * must not be holding the page queue lock. 719 * 720 * note on "cleaning" object and PG_BUSY pages: 721 * this routine is holding the lock on the object. the only time 722 * that it can run into a PG_BUSY page that it does not own is if 723 * some other process has started I/O on the page (e.g. either 724 * a pagein, or a pageout). if the PG_BUSY page is being paged 725 * in, then it can not be dirty (!PG_CLEAN) because no one has 726 * had a chance to modify it yet. if the PG_BUSY page is being 727 * paged out then it means that someone else has already started 728 * cleaning the page for us (how nice!). in this case, if we 729 * have syncio specified, then after we make our pass through the 730 * object we need to wait for the other PG_BUSY pages to clear 731 * off (i.e. we need to do an iosync). also note that once a 732 * page is PG_BUSY it must stay in its object until it is un-busyed. 733 * 734 * note on page traversal: 735 * we can traverse the pages in an object either by going down the 736 * linked list in "uobj->memq", or we can go over the address range 737 * by page doing hash table lookups for each address. depending 738 * on how many pages are in the object it may be cheaper to do one 739 * or the other. we set "by_list" to true if we are using memq. 740 * if the cost of a hash lookup was equal to the cost of the list 741 * traversal we could compare the number of pages in the start->stop 742 * range to the total number of pages in the object. however, it 743 * seems that a hash table lookup is more expensive than the linked 744 * list traversal, so we multiply the number of pages in the 745 * range by an estimate of the relatively higher cost of the hash lookup. 746 */ 747 748 int 749 genfs_putpages(void *v) 750 { 751 struct vop_putpages_args /* { 752 struct vnode *a_vp; 753 voff_t a_offlo; 754 voff_t a_offhi; 755 int a_flags; 756 } */ *ap = v; 757 758 return genfs_do_putpages(ap->a_vp, ap->a_offlo, ap->a_offhi, 759 ap->a_flags, NULL); 760 } 761 762 int 763 genfs_do_putpages(struct vnode *vp, off_t startoff, off_t endoff, 764 int origflags, struct vm_page **busypg) 765 { 766 struct uvm_object *uobj = &vp->v_uobj; 767 kmutex_t *slock = &uobj->vmobjlock; 768 off_t off; 769 /* Even for strange MAXPHYS, the shift rounds down to a page */ 770 #define maxpages (MAXPHYS >> PAGE_SHIFT) 771 int i, error, npages, nback; 772 int freeflag; 773 struct vm_page *pgs[maxpages], *pg, *nextpg, *tpg, curmp, endmp; 774 bool wasclean, by_list, needs_clean, yld; 775 bool async = (origflags & PGO_SYNCIO) == 0; 776 bool pagedaemon = curlwp == uvm.pagedaemon_lwp; 777 struct lwp *l = curlwp ? curlwp : &lwp0; 778 struct genfs_node *gp = VTOG(vp); 779 int flags; 780 int dirtygen; 781 bool modified; 782 bool has_trans; 783 bool cleanall; 784 bool onworklst; 785 786 UVMHIST_FUNC("genfs_putpages"); UVMHIST_CALLED(ubchist); 787 788 KASSERT(origflags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)); 789 KASSERT((startoff & PAGE_MASK) == 0 && (endoff & PAGE_MASK) == 0); 790 KASSERT(startoff < endoff || endoff == 0); 791 792 UVMHIST_LOG(ubchist, "vp %p pages %d off 0x%x len 0x%x", 793 vp, uobj->uo_npages, startoff, endoff - startoff); 794 795 has_trans = false; 796 797 retry: 798 modified = false; 799 flags = origflags; 800 KASSERT((vp->v_iflag & VI_ONWORKLST) != 0 || 801 (vp->v_iflag & VI_WRMAPDIRTY) == 0); 802 if (uobj->uo_npages == 0) { 803 if (vp->v_iflag & VI_ONWORKLST) { 804 vp->v_iflag &= ~VI_WRMAPDIRTY; 805 if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL) 806 vn_syncer_remove_from_worklist(vp); 807 } 808 if (has_trans) 809 fstrans_done(vp->v_mount); 810 mutex_exit(slock); 811 return (0); 812 } 813 814 /* 815 * the vnode has pages, set up to process the request. 816 */ 817 818 if (!has_trans && (flags & PGO_CLEANIT) != 0) { 819 mutex_exit(slock); 820 if (pagedaemon) { 821 error = fstrans_start_nowait(vp->v_mount, FSTRANS_LAZY); 822 if (error) 823 return error; 824 } else 825 fstrans_start(vp->v_mount, FSTRANS_LAZY); 826 has_trans = true; 827 mutex_enter(slock); 828 goto retry; 829 } 830 831 error = 0; 832 wasclean = (vp->v_numoutput == 0); 833 off = startoff; 834 if (endoff == 0 || flags & PGO_ALLPAGES) { 835 endoff = trunc_page(LLONG_MAX); 836 } 837 by_list = (uobj->uo_npages <= 838 ((endoff - startoff) >> PAGE_SHIFT) * UVM_PAGE_HASH_PENALTY); 839 840 #if !defined(DEBUG) 841 /* 842 * if this vnode is known not to have dirty pages, 843 * don't bother to clean it out. 844 */ 845 846 if ((vp->v_iflag & VI_ONWORKLST) == 0) { 847 if ((flags & (PGO_FREE|PGO_DEACTIVATE)) == 0) { 848 goto skip_scan; 849 } 850 flags &= ~PGO_CLEANIT; 851 } 852 #endif /* !defined(DEBUG) */ 853 854 /* 855 * start the loop. when scanning by list, hold the last page 856 * in the list before we start. pages allocated after we start 857 * will be added to the end of the list, so we can stop at the 858 * current last page. 859 */ 860 861 cleanall = (flags & PGO_CLEANIT) != 0 && wasclean && 862 startoff == 0 && endoff == trunc_page(LLONG_MAX) && 863 (vp->v_iflag & VI_ONWORKLST) != 0; 864 dirtygen = gp->g_dirtygen; 865 freeflag = pagedaemon ? PG_PAGEOUT : PG_RELEASED; 866 if (by_list) { 867 curmp.uobject = uobj; 868 curmp.offset = (voff_t)-1; 869 curmp.flags = PG_BUSY; 870 endmp.uobject = uobj; 871 endmp.offset = (voff_t)-1; 872 endmp.flags = PG_BUSY; 873 pg = TAILQ_FIRST(&uobj->memq); 874 TAILQ_INSERT_TAIL(&uobj->memq, &endmp, listq.queue); 875 uvm_lwp_hold(l); 876 } else { 877 pg = uvm_pagelookup(uobj, off); 878 } 879 nextpg = NULL; 880 while (by_list || off < endoff) { 881 882 /* 883 * if the current page is not interesting, move on to the next. 884 */ 885 886 KASSERT(pg == NULL || pg->uobject == uobj); 887 KASSERT(pg == NULL || 888 (pg->flags & (PG_RELEASED|PG_PAGEOUT)) == 0 || 889 (pg->flags & PG_BUSY) != 0); 890 if (by_list) { 891 if (pg == &endmp) { 892 break; 893 } 894 if (pg->offset < startoff || pg->offset >= endoff || 895 pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 896 if (pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 897 wasclean = false; 898 } 899 pg = TAILQ_NEXT(pg, listq.queue); 900 continue; 901 } 902 off = pg->offset; 903 } else if (pg == NULL || pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 904 if (pg != NULL) { 905 wasclean = false; 906 } 907 off += PAGE_SIZE; 908 if (off < endoff) { 909 pg = uvm_pagelookup(uobj, off); 910 } 911 continue; 912 } 913 914 /* 915 * if the current page needs to be cleaned and it's busy, 916 * wait for it to become unbusy. 917 */ 918 919 yld = (l->l_cpu->ci_schedstate.spc_flags & 920 SPCF_SHOULDYIELD) && !pagedaemon; 921 if (pg->flags & PG_BUSY || yld) { 922 UVMHIST_LOG(ubchist, "busy %p", pg,0,0,0); 923 if (flags & PGO_BUSYFAIL && pg->flags & PG_BUSY) { 924 UVMHIST_LOG(ubchist, "busyfail %p", pg, 0,0,0); 925 error = EDEADLK; 926 if (busypg != NULL) 927 *busypg = pg; 928 break; 929 } 930 if (pagedaemon) { 931 /* 932 * someone has taken the page while we 933 * dropped the lock for fstrans_start. 934 */ 935 break; 936 } 937 if (by_list) { 938 TAILQ_INSERT_BEFORE(pg, &curmp, listq.queue); 939 UVMHIST_LOG(ubchist, "curmp next %p", 940 TAILQ_NEXT(&curmp, listq.queue), 0,0,0); 941 } 942 if (yld) { 943 mutex_exit(slock); 944 preempt(); 945 mutex_enter(slock); 946 } else { 947 pg->flags |= PG_WANTED; 948 UVM_UNLOCK_AND_WAIT(pg, slock, 0, "genput", 0); 949 mutex_enter(slock); 950 } 951 if (by_list) { 952 UVMHIST_LOG(ubchist, "after next %p", 953 TAILQ_NEXT(&curmp, listq.queue), 0,0,0); 954 pg = TAILQ_NEXT(&curmp, listq.queue); 955 TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue); 956 } else { 957 pg = uvm_pagelookup(uobj, off); 958 } 959 continue; 960 } 961 962 /* 963 * if we're freeing, remove all mappings of the page now. 964 * if we're cleaning, check if the page is needs to be cleaned. 965 */ 966 967 if (flags & PGO_FREE) { 968 pmap_page_protect(pg, VM_PROT_NONE); 969 } else if (flags & PGO_CLEANIT) { 970 971 /* 972 * if we still have some hope to pull this vnode off 973 * from the syncer queue, write-protect the page. 974 */ 975 976 if (cleanall && wasclean && 977 gp->g_dirtygen == dirtygen) { 978 979 /* 980 * uobj pages get wired only by uvm_fault 981 * where uobj is locked. 982 */ 983 984 if (pg->wire_count == 0) { 985 pmap_page_protect(pg, 986 VM_PROT_READ|VM_PROT_EXECUTE); 987 } else { 988 cleanall = false; 989 } 990 } 991 } 992 993 if (flags & PGO_CLEANIT) { 994 needs_clean = pmap_clear_modify(pg) || 995 (pg->flags & PG_CLEAN) == 0; 996 pg->flags |= PG_CLEAN; 997 } else { 998 needs_clean = false; 999 } 1000 1001 /* 1002 * if we're cleaning, build a cluster. 1003 * the cluster will consist of pages which are currently dirty, 1004 * but they will be returned to us marked clean. 1005 * if not cleaning, just operate on the one page. 1006 */ 1007 1008 if (needs_clean) { 1009 KDASSERT((vp->v_iflag & VI_ONWORKLST)); 1010 wasclean = false; 1011 memset(pgs, 0, sizeof(pgs)); 1012 pg->flags |= PG_BUSY; 1013 UVM_PAGE_OWN(pg, "genfs_putpages"); 1014 1015 /* 1016 * first look backward. 1017 */ 1018 1019 npages = MIN(maxpages >> 1, off >> PAGE_SHIFT); 1020 nback = npages; 1021 uvn_findpages(uobj, off - PAGE_SIZE, &nback, &pgs[0], 1022 UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY|UFP_BACKWARD); 1023 if (nback) { 1024 memmove(&pgs[0], &pgs[npages - nback], 1025 nback * sizeof(pgs[0])); 1026 if (npages - nback < nback) 1027 memset(&pgs[nback], 0, 1028 (npages - nback) * sizeof(pgs[0])); 1029 else 1030 memset(&pgs[npages - nback], 0, 1031 nback * sizeof(pgs[0])); 1032 } 1033 1034 /* 1035 * then plug in our page of interest. 1036 */ 1037 1038 pgs[nback] = pg; 1039 1040 /* 1041 * then look forward to fill in the remaining space in 1042 * the array of pages. 1043 */ 1044 1045 npages = maxpages - nback - 1; 1046 uvn_findpages(uobj, off + PAGE_SIZE, &npages, 1047 &pgs[nback + 1], 1048 UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY); 1049 npages += nback + 1; 1050 } else { 1051 pgs[0] = pg; 1052 npages = 1; 1053 nback = 0; 1054 } 1055 1056 /* 1057 * apply FREE or DEACTIVATE options if requested. 1058 */ 1059 1060 if (flags & (PGO_DEACTIVATE|PGO_FREE)) { 1061 mutex_enter(&uvm_pageqlock); 1062 } 1063 for (i = 0; i < npages; i++) { 1064 tpg = pgs[i]; 1065 KASSERT(tpg->uobject == uobj); 1066 if (by_list && tpg == TAILQ_NEXT(pg, listq.queue)) 1067 pg = tpg; 1068 if (tpg->offset < startoff || tpg->offset >= endoff) 1069 continue; 1070 if (flags & PGO_DEACTIVATE && tpg->wire_count == 0) { 1071 uvm_pagedeactivate(tpg); 1072 } else if (flags & PGO_FREE) { 1073 pmap_page_protect(tpg, VM_PROT_NONE); 1074 if (tpg->flags & PG_BUSY) { 1075 tpg->flags |= freeflag; 1076 if (pagedaemon) { 1077 uvm_pageout_start(1); 1078 uvm_pagedequeue(tpg); 1079 } 1080 } else { 1081 1082 /* 1083 * ``page is not busy'' 1084 * implies that npages is 1 1085 * and needs_clean is false. 1086 */ 1087 1088 nextpg = TAILQ_NEXT(tpg, listq.queue); 1089 uvm_pagefree(tpg); 1090 if (pagedaemon) 1091 uvmexp.pdfreed++; 1092 } 1093 } 1094 } 1095 if (flags & (PGO_DEACTIVATE|PGO_FREE)) { 1096 mutex_exit(&uvm_pageqlock); 1097 } 1098 if (needs_clean) { 1099 modified = true; 1100 1101 /* 1102 * start the i/o. if we're traversing by list, 1103 * keep our place in the list with a marker page. 1104 */ 1105 1106 if (by_list) { 1107 TAILQ_INSERT_AFTER(&uobj->memq, pg, &curmp, 1108 listq.queue); 1109 } 1110 mutex_exit(slock); 1111 error = GOP_WRITE(vp, pgs, npages, flags); 1112 mutex_enter(slock); 1113 if (by_list) { 1114 pg = TAILQ_NEXT(&curmp, listq.queue); 1115 TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue); 1116 } 1117 if (error) { 1118 break; 1119 } 1120 if (by_list) { 1121 continue; 1122 } 1123 } 1124 1125 /* 1126 * find the next page and continue if there was no error. 1127 */ 1128 1129 if (by_list) { 1130 if (nextpg) { 1131 pg = nextpg; 1132 nextpg = NULL; 1133 } else { 1134 pg = TAILQ_NEXT(pg, listq.queue); 1135 } 1136 } else { 1137 off += (npages - nback) << PAGE_SHIFT; 1138 if (off < endoff) { 1139 pg = uvm_pagelookup(uobj, off); 1140 } 1141 } 1142 } 1143 if (by_list) { 1144 TAILQ_REMOVE(&uobj->memq, &endmp, listq.queue); 1145 uvm_lwp_rele(l); 1146 } 1147 1148 if (modified && (vp->v_iflag & VI_WRMAPDIRTY) != 0 && 1149 (vp->v_type != VBLK || 1150 (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) { 1151 GOP_MARKUPDATE(vp, GOP_UPDATE_MODIFIED); 1152 } 1153 1154 /* 1155 * if we're cleaning and there was nothing to clean, 1156 * take us off the syncer list. if we started any i/o 1157 * and we're doing sync i/o, wait for all writes to finish. 1158 */ 1159 1160 if (cleanall && wasclean && gp->g_dirtygen == dirtygen && 1161 (vp->v_iflag & VI_ONWORKLST) != 0) { 1162 #if defined(DEBUG) 1163 TAILQ_FOREACH(pg, &uobj->memq, listq.queue) { 1164 if ((pg->flags & PG_CLEAN) == 0) { 1165 printf("%s: %p: !CLEAN\n", __func__, pg); 1166 } 1167 if (pmap_is_modified(pg)) { 1168 printf("%s: %p: modified\n", __func__, pg); 1169 } 1170 } 1171 #endif /* defined(DEBUG) */ 1172 vp->v_iflag &= ~VI_WRMAPDIRTY; 1173 if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL) 1174 vn_syncer_remove_from_worklist(vp); 1175 } 1176 1177 #if !defined(DEBUG) 1178 skip_scan: 1179 #endif /* !defined(DEBUG) */ 1180 1181 /* Wait for output to complete. */ 1182 if (!wasclean && !async && vp->v_numoutput != 0) { 1183 while (vp->v_numoutput != 0) 1184 cv_wait(&vp->v_cv, slock); 1185 } 1186 onworklst = (vp->v_iflag & VI_ONWORKLST) != 0; 1187 mutex_exit(slock); 1188 1189 if ((flags & PGO_RECLAIM) != 0 && onworklst) { 1190 /* 1191 * in the case of PGO_RECLAIM, ensure to make the vnode clean. 1192 * retrying is not a big deal because, in many cases, 1193 * uobj->uo_npages is already 0 here. 1194 */ 1195 mutex_enter(slock); 1196 goto retry; 1197 } 1198 1199 if (has_trans) 1200 fstrans_done(vp->v_mount); 1201 1202 return (error); 1203 } 1204 1205 int 1206 genfs_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, int flags) 1207 { 1208 off_t off; 1209 vaddr_t kva; 1210 size_t len; 1211 int error; 1212 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1213 1214 UVMHIST_LOG(ubchist, "vp %p pgs %p npages %d flags 0x%x", 1215 vp, pgs, npages, flags); 1216 1217 off = pgs[0]->offset; 1218 kva = uvm_pagermapin(pgs, npages, 1219 UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK); 1220 len = npages << PAGE_SHIFT; 1221 1222 error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE, 1223 uvm_aio_biodone); 1224 1225 return error; 1226 } 1227 1228 int 1229 genfs_gop_write_rwmap(struct vnode *vp, struct vm_page **pgs, int npages, int flags) 1230 { 1231 off_t off; 1232 vaddr_t kva; 1233 size_t len; 1234 int error; 1235 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1236 1237 UVMHIST_LOG(ubchist, "vp %p pgs %p npages %d flags 0x%x", 1238 vp, pgs, npages, flags); 1239 1240 off = pgs[0]->offset; 1241 kva = uvm_pagermapin(pgs, npages, 1242 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 1243 len = npages << PAGE_SHIFT; 1244 1245 error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE, 1246 uvm_aio_biodone); 1247 1248 return error; 1249 } 1250 1251 /* 1252 * Backend routine for doing I/O to vnode pages. Pages are already locked 1253 * and mapped into kernel memory. Here we just look up the underlying 1254 * device block addresses and call the strategy routine. 1255 */ 1256 1257 static int 1258 genfs_do_io(struct vnode *vp, off_t off, vaddr_t kva, size_t len, int flags, 1259 enum uio_rw rw, void (*iodone)(struct buf *)) 1260 { 1261 int s, error, run; 1262 int fs_bshift, dev_bshift; 1263 off_t eof, offset, startoffset; 1264 size_t bytes, iobytes, skipbytes; 1265 daddr_t lbn, blkno; 1266 struct buf *mbp, *bp; 1267 struct vnode *devvp; 1268 bool async = (flags & PGO_SYNCIO) == 0; 1269 bool write = rw == UIO_WRITE; 1270 int brw = write ? B_WRITE : B_READ; 1271 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1272 1273 UVMHIST_LOG(ubchist, "vp %p kva %p len 0x%x flags 0x%x", 1274 vp, kva, len, flags); 1275 1276 KASSERT(vp->v_size <= vp->v_writesize); 1277 GOP_SIZE(vp, vp->v_writesize, &eof, 0); 1278 if (vp->v_type != VBLK) { 1279 fs_bshift = vp->v_mount->mnt_fs_bshift; 1280 dev_bshift = vp->v_mount->mnt_dev_bshift; 1281 } else { 1282 fs_bshift = DEV_BSHIFT; 1283 dev_bshift = DEV_BSHIFT; 1284 } 1285 error = 0; 1286 startoffset = off; 1287 bytes = MIN(len, eof - startoffset); 1288 skipbytes = 0; 1289 KASSERT(bytes != 0); 1290 1291 if (write) { 1292 mutex_enter(&vp->v_interlock); 1293 vp->v_numoutput += 2; 1294 mutex_exit(&vp->v_interlock); 1295 } 1296 mbp = getiobuf(vp, true); 1297 UVMHIST_LOG(ubchist, "vp %p mbp %p num now %d bytes 0x%x", 1298 vp, mbp, vp->v_numoutput, bytes); 1299 mbp->b_bufsize = len; 1300 mbp->b_data = (void *)kva; 1301 mbp->b_resid = mbp->b_bcount = bytes; 1302 mbp->b_cflags = BC_BUSY | BC_AGE; 1303 if (async) { 1304 mbp->b_flags = brw | B_ASYNC; 1305 mbp->b_iodone = iodone; 1306 } else { 1307 mbp->b_flags = brw; 1308 mbp->b_iodone = NULL; 1309 } 1310 if (curlwp == uvm.pagedaemon_lwp) 1311 BIO_SETPRIO(mbp, BPRIO_TIMELIMITED); 1312 else if (async) 1313 BIO_SETPRIO(mbp, BPRIO_TIMENONCRITICAL); 1314 else 1315 BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL); 1316 1317 bp = NULL; 1318 for (offset = startoffset; 1319 bytes > 0; 1320 offset += iobytes, bytes -= iobytes) { 1321 lbn = offset >> fs_bshift; 1322 error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run); 1323 if (error) { 1324 UVMHIST_LOG(ubchist, "VOP_BMAP() -> %d", error,0,0,0); 1325 skipbytes += bytes; 1326 bytes = 0; 1327 break; 1328 } 1329 1330 iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset, 1331 bytes); 1332 if (blkno == (daddr_t)-1) { 1333 if (!write) { 1334 memset((char *)kva + (offset - startoffset), 0, 1335 iobytes); 1336 } 1337 skipbytes += iobytes; 1338 continue; 1339 } 1340 1341 /* if it's really one i/o, don't make a second buf */ 1342 if (offset == startoffset && iobytes == bytes) { 1343 bp = mbp; 1344 } else { 1345 UVMHIST_LOG(ubchist, "vp %p bp %p num now %d", 1346 vp, bp, vp->v_numoutput, 0); 1347 bp = getiobuf(vp, true); 1348 nestiobuf_setup(mbp, bp, offset - startoffset, iobytes); 1349 } 1350 bp->b_lblkno = 0; 1351 1352 /* adjust physical blkno for partial blocks */ 1353 bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >> 1354 dev_bshift); 1355 UVMHIST_LOG(ubchist, 1356 "vp %p offset 0x%x bcount 0x%x blkno 0x%x", 1357 vp, offset, bp->b_bcount, bp->b_blkno); 1358 1359 VOP_STRATEGY(devvp, bp); 1360 } 1361 if (skipbytes) { 1362 UVMHIST_LOG(ubchist, "skipbytes %d", skipbytes, 0,0,0); 1363 } 1364 nestiobuf_done(mbp, skipbytes, error); 1365 if (async) { 1366 UVMHIST_LOG(ubchist, "returning 0 (async)", 0,0,0,0); 1367 return (0); 1368 } 1369 UVMHIST_LOG(ubchist, "waiting for mbp %p", mbp,0,0,0); 1370 error = biowait(mbp); 1371 s = splbio(); 1372 (*iodone)(mbp); 1373 splx(s); 1374 UVMHIST_LOG(ubchist, "returning, error %d", error,0,0,0); 1375 return (error); 1376 } 1377 1378 /* 1379 * VOP_PUTPAGES() for vnodes which never have pages. 1380 */ 1381 1382 int 1383 genfs_null_putpages(void *v) 1384 { 1385 struct vop_putpages_args /* { 1386 struct vnode *a_vp; 1387 voff_t a_offlo; 1388 voff_t a_offhi; 1389 int a_flags; 1390 } */ *ap = v; 1391 struct vnode *vp = ap->a_vp; 1392 1393 KASSERT(vp->v_uobj.uo_npages == 0); 1394 mutex_exit(&vp->v_interlock); 1395 return (0); 1396 } 1397 1398 int 1399 genfs_compat_getpages(void *v) 1400 { 1401 struct vop_getpages_args /* { 1402 struct vnode *a_vp; 1403 voff_t a_offset; 1404 struct vm_page **a_m; 1405 int *a_count; 1406 int a_centeridx; 1407 vm_prot_t a_access_type; 1408 int a_advice; 1409 int a_flags; 1410 } */ *ap = v; 1411 1412 off_t origoffset; 1413 struct vnode *vp = ap->a_vp; 1414 struct uvm_object *uobj = &vp->v_uobj; 1415 struct vm_page *pg, **pgs; 1416 vaddr_t kva; 1417 int i, error, orignpages, npages; 1418 struct iovec iov; 1419 struct uio uio; 1420 kauth_cred_t cred = curlwp->l_cred; 1421 bool write = (ap->a_access_type & VM_PROT_WRITE) != 0; 1422 1423 error = 0; 1424 origoffset = ap->a_offset; 1425 orignpages = *ap->a_count; 1426 pgs = ap->a_m; 1427 1428 if (write && (vp->v_iflag & VI_ONWORKLST) == 0) { 1429 vn_syncer_add_to_worklist(vp, filedelay); 1430 } 1431 if (ap->a_flags & PGO_LOCKED) { 1432 uvn_findpages(uobj, origoffset, ap->a_count, ap->a_m, 1433 UFP_NOWAIT|UFP_NOALLOC| (write ? UFP_NORDONLY : 0)); 1434 1435 return (ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0); 1436 } 1437 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= vp->v_size) { 1438 mutex_exit(&uobj->vmobjlock); 1439 return (EINVAL); 1440 } 1441 if ((ap->a_flags & PGO_SYNCIO) == 0) { 1442 mutex_exit(&uobj->vmobjlock); 1443 return 0; 1444 } 1445 npages = orignpages; 1446 uvn_findpages(uobj, origoffset, &npages, pgs, UFP_ALL); 1447 mutex_exit(&uobj->vmobjlock); 1448 kva = uvm_pagermapin(pgs, npages, 1449 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 1450 for (i = 0; i < npages; i++) { 1451 pg = pgs[i]; 1452 if ((pg->flags & PG_FAKE) == 0) { 1453 continue; 1454 } 1455 iov.iov_base = (char *)kva + (i << PAGE_SHIFT); 1456 iov.iov_len = PAGE_SIZE; 1457 uio.uio_iov = &iov; 1458 uio.uio_iovcnt = 1; 1459 uio.uio_offset = origoffset + (i << PAGE_SHIFT); 1460 uio.uio_rw = UIO_READ; 1461 uio.uio_resid = PAGE_SIZE; 1462 UIO_SETUP_SYSSPACE(&uio); 1463 /* XXX vn_lock */ 1464 error = VOP_READ(vp, &uio, 0, cred); 1465 if (error) { 1466 break; 1467 } 1468 if (uio.uio_resid) { 1469 memset(iov.iov_base, 0, uio.uio_resid); 1470 } 1471 } 1472 uvm_pagermapout(kva, npages); 1473 mutex_enter(&uobj->vmobjlock); 1474 mutex_enter(&uvm_pageqlock); 1475 for (i = 0; i < npages; i++) { 1476 pg = pgs[i]; 1477 if (error && (pg->flags & PG_FAKE) != 0) { 1478 pg->flags |= PG_RELEASED; 1479 } else { 1480 pmap_clear_modify(pg); 1481 uvm_pageactivate(pg); 1482 } 1483 } 1484 if (error) { 1485 uvm_page_unbusy(pgs, npages); 1486 } 1487 mutex_exit(&uvm_pageqlock); 1488 mutex_exit(&uobj->vmobjlock); 1489 return (error); 1490 } 1491 1492 int 1493 genfs_compat_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, 1494 int flags) 1495 { 1496 off_t offset; 1497 struct iovec iov; 1498 struct uio uio; 1499 kauth_cred_t cred = curlwp->l_cred; 1500 struct buf *bp; 1501 vaddr_t kva; 1502 int error; 1503 1504 offset = pgs[0]->offset; 1505 kva = uvm_pagermapin(pgs, npages, 1506 UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK); 1507 1508 iov.iov_base = (void *)kva; 1509 iov.iov_len = npages << PAGE_SHIFT; 1510 uio.uio_iov = &iov; 1511 uio.uio_iovcnt = 1; 1512 uio.uio_offset = offset; 1513 uio.uio_rw = UIO_WRITE; 1514 uio.uio_resid = npages << PAGE_SHIFT; 1515 UIO_SETUP_SYSSPACE(&uio); 1516 /* XXX vn_lock */ 1517 error = VOP_WRITE(vp, &uio, 0, cred); 1518 1519 mutex_enter(&vp->v_interlock); 1520 vp->v_numoutput++; 1521 mutex_exit(&vp->v_interlock); 1522 1523 bp = getiobuf(vp, true); 1524 bp->b_cflags = BC_BUSY | BC_AGE; 1525 bp->b_lblkno = offset >> vp->v_mount->mnt_fs_bshift; 1526 bp->b_data = (char *)kva; 1527 bp->b_bcount = npages << PAGE_SHIFT; 1528 bp->b_bufsize = npages << PAGE_SHIFT; 1529 bp->b_resid = 0; 1530 bp->b_error = error; 1531 uvm_aio_aiodone(bp); 1532 return (error); 1533 } 1534 1535 /* 1536 * Process a uio using direct I/O. If we reach a part of the request 1537 * which cannot be processed in this fashion for some reason, just return. 1538 * The caller must handle some additional part of the request using 1539 * buffered I/O before trying direct I/O again. 1540 */ 1541 1542 void 1543 genfs_directio(struct vnode *vp, struct uio *uio, int ioflag) 1544 { 1545 struct vmspace *vs; 1546 struct iovec *iov; 1547 vaddr_t va; 1548 size_t len; 1549 const int mask = DEV_BSIZE - 1; 1550 int error; 1551 1552 /* 1553 * We only support direct I/O to user space for now. 1554 */ 1555 1556 if (VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) { 1557 return; 1558 } 1559 1560 /* 1561 * If the vnode is mapped, we would need to get the getpages lock 1562 * to stabilize the bmap, but then we would get into trouble whil e 1563 * locking the pages if the pages belong to this same vnode (or a 1564 * multi-vnode cascade to the same effect). Just fall back to 1565 * buffered I/O if the vnode is mapped to avoid this mess. 1566 */ 1567 1568 if (vp->v_vflag & VV_MAPPED) { 1569 return; 1570 } 1571 1572 /* 1573 * Do as much of the uio as possible with direct I/O. 1574 */ 1575 1576 vs = uio->uio_vmspace; 1577 while (uio->uio_resid) { 1578 iov = uio->uio_iov; 1579 if (iov->iov_len == 0) { 1580 uio->uio_iov++; 1581 uio->uio_iovcnt--; 1582 continue; 1583 } 1584 va = (vaddr_t)iov->iov_base; 1585 len = MIN(iov->iov_len, genfs_maxdio); 1586 len &= ~mask; 1587 1588 /* 1589 * If the next chunk is smaller than DEV_BSIZE or extends past 1590 * the current EOF, then fall back to buffered I/O. 1591 */ 1592 1593 if (len == 0 || uio->uio_offset + len > vp->v_size) { 1594 return; 1595 } 1596 1597 /* 1598 * Check alignment. The file offset must be at least 1599 * sector-aligned. The exact constraint on memory alignment 1600 * is very hardware-dependent, but requiring sector-aligned 1601 * addresses there too is safe. 1602 */ 1603 1604 if (uio->uio_offset & mask || va & mask) { 1605 return; 1606 } 1607 error = genfs_do_directio(vs, va, len, vp, uio->uio_offset, 1608 uio->uio_rw); 1609 if (error) { 1610 break; 1611 } 1612 iov->iov_base = (char *)iov->iov_base + len; 1613 iov->iov_len -= len; 1614 uio->uio_offset += len; 1615 uio->uio_resid -= len; 1616 } 1617 } 1618 1619 /* 1620 * Iodone routine for direct I/O. We don't do much here since the request is 1621 * always synchronous, so the caller will do most of the work after biowait(). 1622 */ 1623 1624 static void 1625 genfs_dio_iodone(struct buf *bp) 1626 { 1627 1628 KASSERT((bp->b_flags & B_ASYNC) == 0); 1629 if ((bp->b_flags & B_READ) == 0 && (bp->b_cflags & BC_AGE) != 0) { 1630 mutex_enter(bp->b_objlock); 1631 vwakeup(bp); 1632 mutex_exit(bp->b_objlock); 1633 } 1634 putiobuf(bp); 1635 } 1636 1637 /* 1638 * Process one chunk of a direct I/O request. 1639 */ 1640 1641 static int 1642 genfs_do_directio(struct vmspace *vs, vaddr_t uva, size_t len, struct vnode *vp, 1643 off_t off, enum uio_rw rw) 1644 { 1645 struct vm_map *map; 1646 struct pmap *upm, *kpm; 1647 size_t klen = round_page(uva + len) - trunc_page(uva); 1648 off_t spoff, epoff; 1649 vaddr_t kva, puva; 1650 paddr_t pa; 1651 vm_prot_t prot; 1652 int error, rv, poff, koff; 1653 const int pgoflags = PGO_CLEANIT | PGO_SYNCIO | 1654 (rw == UIO_WRITE ? PGO_FREE : 0); 1655 1656 /* 1657 * For writes, verify that this range of the file already has fully 1658 * allocated backing store. If there are any holes, just punt and 1659 * make the caller take the buffered write path. 1660 */ 1661 1662 if (rw == UIO_WRITE) { 1663 daddr_t lbn, elbn, blkno; 1664 int bsize, bshift, run; 1665 1666 bshift = vp->v_mount->mnt_fs_bshift; 1667 bsize = 1 << bshift; 1668 lbn = off >> bshift; 1669 elbn = (off + len + bsize - 1) >> bshift; 1670 while (lbn < elbn) { 1671 error = VOP_BMAP(vp, lbn, NULL, &blkno, &run); 1672 if (error) { 1673 return error; 1674 } 1675 if (blkno == (daddr_t)-1) { 1676 return ENOSPC; 1677 } 1678 lbn += 1 + run; 1679 } 1680 } 1681 1682 /* 1683 * Flush any cached pages for parts of the file that we're about to 1684 * access. If we're writing, invalidate pages as well. 1685 */ 1686 1687 spoff = trunc_page(off); 1688 epoff = round_page(off + len); 1689 mutex_enter(&vp->v_interlock); 1690 error = VOP_PUTPAGES(vp, spoff, epoff, pgoflags); 1691 if (error) { 1692 return error; 1693 } 1694 1695 /* 1696 * Wire the user pages and remap them into kernel memory. 1697 */ 1698 1699 prot = rw == UIO_READ ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_READ; 1700 error = uvm_vslock(vs, (void *)uva, len, prot); 1701 if (error) { 1702 return error; 1703 } 1704 1705 map = &vs->vm_map; 1706 upm = vm_map_pmap(map); 1707 kpm = vm_map_pmap(kernel_map); 1708 kva = uvm_km_alloc(kernel_map, klen, 0, 1709 UVM_KMF_VAONLY | UVM_KMF_WAITVA); 1710 puva = trunc_page(uva); 1711 for (poff = 0; poff < klen; poff += PAGE_SIZE) { 1712 rv = pmap_extract(upm, puva + poff, &pa); 1713 KASSERT(rv); 1714 pmap_enter(kpm, kva + poff, pa, prot, prot | PMAP_WIRED); 1715 } 1716 pmap_update(kpm); 1717 1718 /* 1719 * Do the I/O. 1720 */ 1721 1722 koff = uva - trunc_page(uva); 1723 error = genfs_do_io(vp, off, kva + koff, len, PGO_SYNCIO, rw, 1724 genfs_dio_iodone); 1725 1726 /* 1727 * Tear down the kernel mapping. 1728 */ 1729 1730 pmap_remove(kpm, kva, kva + klen); 1731 pmap_update(kpm); 1732 uvm_km_free(kernel_map, kva, klen, UVM_KMF_VAONLY); 1733 1734 /* 1735 * Unwire the user pages. 1736 */ 1737 1738 uvm_vsunlock(vs, (void *)uva, len); 1739 return error; 1740 } 1741 1742