1 /* $NetBSD: genfs_io.c,v 1.76 2019/10/06 05:48:00 mlelstv Exp $ */ 2 3 /* 4 * Copyright (c) 1982, 1986, 1989, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the University nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 */ 32 33 #include <sys/cdefs.h> 34 __KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.76 2019/10/06 05:48:00 mlelstv Exp $"); 35 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/proc.h> 39 #include <sys/kernel.h> 40 #include <sys/mount.h> 41 #include <sys/vnode.h> 42 #include <sys/kmem.h> 43 #include <sys/kauth.h> 44 #include <sys/fstrans.h> 45 #include <sys/buf.h> 46 47 #include <miscfs/genfs/genfs.h> 48 #include <miscfs/genfs/genfs_node.h> 49 #include <miscfs/specfs/specdev.h> 50 51 #include <uvm/uvm.h> 52 #include <uvm/uvm_pager.h> 53 54 static int genfs_do_directio(struct vmspace *, vaddr_t, size_t, struct vnode *, 55 off_t, enum uio_rw); 56 static void genfs_dio_iodone(struct buf *); 57 58 static int genfs_getpages_read(struct vnode *, struct vm_page **, int, off_t, 59 off_t, bool, bool, bool, bool); 60 static int genfs_do_io(struct vnode *, off_t, vaddr_t, size_t, int, enum uio_rw, 61 void (*)(struct buf *)); 62 static void genfs_rel_pages(struct vm_page **, unsigned int); 63 static void genfs_markdirty(struct vnode *); 64 65 int genfs_maxdio = MAXPHYS; 66 67 static void 68 genfs_rel_pages(struct vm_page **pgs, unsigned int npages) 69 { 70 unsigned int i; 71 72 for (i = 0; i < npages; i++) { 73 struct vm_page *pg = pgs[i]; 74 75 if (pg == NULL || pg == PGO_DONTCARE) 76 continue; 77 KASSERT(uvm_page_locked_p(pg)); 78 if (pg->flags & PG_FAKE) { 79 pg->flags |= PG_RELEASED; 80 } 81 } 82 mutex_enter(&uvm_pageqlock); 83 uvm_page_unbusy(pgs, npages); 84 mutex_exit(&uvm_pageqlock); 85 } 86 87 static void 88 genfs_markdirty(struct vnode *vp) 89 { 90 struct genfs_node * const gp = VTOG(vp); 91 92 KASSERT(mutex_owned(vp->v_interlock)); 93 gp->g_dirtygen++; 94 if ((vp->v_iflag & VI_ONWORKLST) == 0) { 95 vn_syncer_add_to_worklist(vp, filedelay); 96 } 97 if ((vp->v_iflag & (VI_WRMAP|VI_WRMAPDIRTY)) == VI_WRMAP) { 98 vp->v_iflag |= VI_WRMAPDIRTY; 99 } 100 } 101 102 /* 103 * generic VM getpages routine. 104 * Return PG_BUSY pages for the given range, 105 * reading from backing store if necessary. 106 */ 107 108 int 109 genfs_getpages(void *v) 110 { 111 struct vop_getpages_args /* { 112 struct vnode *a_vp; 113 voff_t a_offset; 114 struct vm_page **a_m; 115 int *a_count; 116 int a_centeridx; 117 vm_prot_t a_access_type; 118 int a_advice; 119 int a_flags; 120 } */ * const ap = v; 121 122 off_t diskeof, memeof; 123 int i, error, npages; 124 const int flags = ap->a_flags; 125 struct vnode * const vp = ap->a_vp; 126 struct uvm_object * const uobj = &vp->v_uobj; 127 const bool async = (flags & PGO_SYNCIO) == 0; 128 const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0; 129 const bool overwrite = (flags & PGO_OVERWRITE) != 0; 130 const bool blockalloc = memwrite && (flags & PGO_NOBLOCKALLOC) == 0; 131 const bool need_wapbl = (vp->v_mount->mnt_wapbl && 132 (flags & PGO_JOURNALLOCKED) == 0); 133 const bool glocked = (flags & PGO_GLOCKHELD) != 0; 134 bool holds_wapbl = false; 135 struct mount *trans_mount = NULL; 136 UVMHIST_FUNC("genfs_getpages"); UVMHIST_CALLED(ubchist); 137 138 UVMHIST_LOG(ubchist, "vp %#jx off 0x%jx/%jx count %jd", 139 (uintptr_t)vp, ap->a_offset >> 32, ap->a_offset, *ap->a_count); 140 141 KASSERT(vp->v_type == VREG || vp->v_type == VDIR || 142 vp->v_type == VLNK || vp->v_type == VBLK); 143 144 #ifdef DIAGNOSTIC 145 if ((flags & PGO_JOURNALLOCKED) && vp->v_mount->mnt_wapbl) 146 WAPBL_JLOCK_ASSERT(vp->v_mount); 147 #endif 148 149 error = vdead_check(vp, VDEAD_NOWAIT); 150 if (error) { 151 if ((flags & PGO_LOCKED) == 0) 152 mutex_exit(uobj->vmobjlock); 153 return error; 154 } 155 156 startover: 157 error = 0; 158 const voff_t origvsize = vp->v_size; 159 const off_t origoffset = ap->a_offset; 160 const int orignpages = *ap->a_count; 161 162 GOP_SIZE(vp, origvsize, &diskeof, 0); 163 if (flags & PGO_PASTEOF) { 164 off_t newsize; 165 #if defined(DIAGNOSTIC) 166 off_t writeeof; 167 #endif /* defined(DIAGNOSTIC) */ 168 169 newsize = MAX(origvsize, 170 origoffset + (orignpages << PAGE_SHIFT)); 171 GOP_SIZE(vp, newsize, &memeof, GOP_SIZE_MEM); 172 #if defined(DIAGNOSTIC) 173 GOP_SIZE(vp, vp->v_writesize, &writeeof, GOP_SIZE_MEM); 174 if (newsize > round_page(writeeof)) { 175 panic("%s: past eof: %" PRId64 " vs. %" PRId64, 176 __func__, newsize, round_page(writeeof)); 177 } 178 #endif /* defined(DIAGNOSTIC) */ 179 } else { 180 GOP_SIZE(vp, origvsize, &memeof, GOP_SIZE_MEM); 181 } 182 KASSERT(ap->a_centeridx >= 0 || ap->a_centeridx <= orignpages); 183 KASSERT((origoffset & (PAGE_SIZE - 1)) == 0 && origoffset >= 0); 184 KASSERT(orignpages > 0); 185 186 /* 187 * Bounds-check the request. 188 */ 189 190 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= memeof) { 191 if ((flags & PGO_LOCKED) == 0) { 192 mutex_exit(uobj->vmobjlock); 193 } 194 UVMHIST_LOG(ubchist, "off 0x%jx count %jd goes past EOF 0x%jx", 195 origoffset, *ap->a_count, memeof,0); 196 error = EINVAL; 197 goto out_err; 198 } 199 200 /* uobj is locked */ 201 202 if ((flags & PGO_NOTIMESTAMP) == 0 && 203 (vp->v_type != VBLK || 204 (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) { 205 int updflags = 0; 206 207 if ((vp->v_mount->mnt_flag & MNT_NOATIME) == 0) { 208 updflags = GOP_UPDATE_ACCESSED; 209 } 210 if (memwrite) { 211 updflags |= GOP_UPDATE_MODIFIED; 212 } 213 if (updflags != 0) { 214 GOP_MARKUPDATE(vp, updflags); 215 } 216 } 217 218 /* 219 * For PGO_LOCKED requests, just return whatever's in memory. 220 */ 221 222 if (flags & PGO_LOCKED) { 223 int nfound; 224 struct vm_page *pg; 225 226 KASSERT(!glocked); 227 npages = *ap->a_count; 228 #if defined(DEBUG) 229 for (i = 0; i < npages; i++) { 230 pg = ap->a_m[i]; 231 KASSERT(pg == NULL || pg == PGO_DONTCARE); 232 } 233 #endif /* defined(DEBUG) */ 234 nfound = uvn_findpages(uobj, origoffset, &npages, 235 ap->a_m, UFP_NOWAIT|UFP_NOALLOC|(memwrite ? UFP_NORDONLY : 0)); 236 KASSERT(npages == *ap->a_count); 237 if (nfound == 0) { 238 error = EBUSY; 239 goto out_err; 240 } 241 if (!genfs_node_rdtrylock(vp)) { 242 genfs_rel_pages(ap->a_m, npages); 243 244 /* 245 * restore the array. 246 */ 247 248 for (i = 0; i < npages; i++) { 249 pg = ap->a_m[i]; 250 251 if (pg != NULL && pg != PGO_DONTCARE) { 252 ap->a_m[i] = NULL; 253 } 254 KASSERT(ap->a_m[i] == NULL || 255 ap->a_m[i] == PGO_DONTCARE); 256 } 257 } else { 258 genfs_node_unlock(vp); 259 } 260 error = (ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0); 261 if (error == 0 && memwrite) { 262 genfs_markdirty(vp); 263 } 264 goto out_err; 265 } 266 mutex_exit(uobj->vmobjlock); 267 268 /* 269 * find the requested pages and make some simple checks. 270 * leave space in the page array for a whole block. 271 */ 272 273 const int fs_bshift = (vp->v_type != VBLK) ? 274 vp->v_mount->mnt_fs_bshift : DEV_BSHIFT; 275 const int fs_bsize = 1 << fs_bshift; 276 #define blk_mask (fs_bsize - 1) 277 #define trunc_blk(x) ((x) & ~blk_mask) 278 #define round_blk(x) (((x) + blk_mask) & ~blk_mask) 279 280 const int orignmempages = MIN(orignpages, 281 round_page(memeof - origoffset) >> PAGE_SHIFT); 282 npages = orignmempages; 283 const off_t startoffset = trunc_blk(origoffset); 284 const off_t endoffset = MIN( 285 round_page(round_blk(origoffset + (npages << PAGE_SHIFT))), 286 round_page(memeof)); 287 const int ridx = (origoffset - startoffset) >> PAGE_SHIFT; 288 289 const int pgs_size = sizeof(struct vm_page *) * 290 ((endoffset - startoffset) >> PAGE_SHIFT); 291 struct vm_page **pgs, *pgs_onstack[UBC_MAX_PAGES]; 292 293 if (pgs_size > sizeof(pgs_onstack)) { 294 pgs = kmem_zalloc(pgs_size, async ? KM_NOSLEEP : KM_SLEEP); 295 if (pgs == NULL) { 296 pgs = pgs_onstack; 297 error = ENOMEM; 298 goto out_err; 299 } 300 } else { 301 pgs = pgs_onstack; 302 (void)memset(pgs, 0, pgs_size); 303 } 304 305 UVMHIST_LOG(ubchist, "ridx %jd npages %jd startoff %jd endoff %jd", 306 ridx, npages, startoffset, endoffset); 307 308 if (trans_mount == NULL) { 309 trans_mount = vp->v_mount; 310 fstrans_start(trans_mount); 311 /* 312 * check if this vnode is still valid. 313 */ 314 mutex_enter(vp->v_interlock); 315 error = vdead_check(vp, 0); 316 mutex_exit(vp->v_interlock); 317 if (error) 318 goto out_err_free; 319 /* 320 * XXX: This assumes that we come here only via 321 * the mmio path 322 */ 323 if (blockalloc && need_wapbl) { 324 error = WAPBL_BEGIN(trans_mount); 325 if (error) 326 goto out_err_free; 327 holds_wapbl = true; 328 } 329 } 330 331 /* 332 * hold g_glock to prevent a race with truncate. 333 * 334 * check if our idea of v_size is still valid. 335 */ 336 337 KASSERT(!glocked || genfs_node_wrlocked(vp)); 338 if (!glocked) { 339 if (blockalloc) { 340 genfs_node_wrlock(vp); 341 } else { 342 genfs_node_rdlock(vp); 343 } 344 } 345 mutex_enter(uobj->vmobjlock); 346 if (vp->v_size < origvsize) { 347 if (!glocked) { 348 genfs_node_unlock(vp); 349 } 350 if (pgs != pgs_onstack) 351 kmem_free(pgs, pgs_size); 352 goto startover; 353 } 354 355 if (uvn_findpages(uobj, origoffset, &npages, &pgs[ridx], 356 async ? UFP_NOWAIT : UFP_ALL) != orignmempages) { 357 if (!glocked) { 358 genfs_node_unlock(vp); 359 } 360 KASSERT(async != 0); 361 genfs_rel_pages(&pgs[ridx], orignmempages); 362 mutex_exit(uobj->vmobjlock); 363 error = EBUSY; 364 goto out_err_free; 365 } 366 367 /* 368 * if the pages are already resident, just return them. 369 */ 370 371 for (i = 0; i < npages; i++) { 372 struct vm_page *pg = pgs[ridx + i]; 373 374 if ((pg->flags & PG_FAKE) || 375 (blockalloc && (pg->flags & PG_RDONLY))) { 376 break; 377 } 378 } 379 if (i == npages) { 380 if (!glocked) { 381 genfs_node_unlock(vp); 382 } 383 UVMHIST_LOG(ubchist, "returning cached pages", 0,0,0,0); 384 npages += ridx; 385 goto out; 386 } 387 388 /* 389 * if PGO_OVERWRITE is set, don't bother reading the pages. 390 */ 391 392 if (overwrite) { 393 if (!glocked) { 394 genfs_node_unlock(vp); 395 } 396 UVMHIST_LOG(ubchist, "PGO_OVERWRITE",0,0,0,0); 397 398 for (i = 0; i < npages; i++) { 399 struct vm_page *pg = pgs[ridx + i]; 400 401 pg->flags &= ~(PG_RDONLY|PG_CLEAN); 402 } 403 npages += ridx; 404 goto out; 405 } 406 407 /* 408 * the page wasn't resident and we're not overwriting, 409 * so we're going to have to do some i/o. 410 * find any additional pages needed to cover the expanded range. 411 */ 412 413 npages = (endoffset - startoffset) >> PAGE_SHIFT; 414 if (startoffset != origoffset || npages != orignmempages) { 415 int npgs; 416 417 /* 418 * we need to avoid deadlocks caused by locking 419 * additional pages at lower offsets than pages we 420 * already have locked. unlock them all and start over. 421 */ 422 423 genfs_rel_pages(&pgs[ridx], orignmempages); 424 memset(pgs, 0, pgs_size); 425 426 UVMHIST_LOG(ubchist, "reset npages start 0x%jx end 0x%jx", 427 startoffset, endoffset, 0,0); 428 npgs = npages; 429 if (uvn_findpages(uobj, startoffset, &npgs, pgs, 430 async ? UFP_NOWAIT : UFP_ALL) != npages) { 431 if (!glocked) { 432 genfs_node_unlock(vp); 433 } 434 KASSERT(async != 0); 435 genfs_rel_pages(pgs, npages); 436 mutex_exit(uobj->vmobjlock); 437 error = EBUSY; 438 goto out_err_free; 439 } 440 } 441 442 mutex_exit(uobj->vmobjlock); 443 error = genfs_getpages_read(vp, pgs, npages, startoffset, diskeof, 444 async, memwrite, blockalloc, glocked); 445 if (!glocked) { 446 genfs_node_unlock(vp); 447 } 448 if (error == 0 && async) 449 goto out_err_free; 450 mutex_enter(uobj->vmobjlock); 451 452 /* 453 * we're almost done! release the pages... 454 * for errors, we free the pages. 455 * otherwise we activate them and mark them as valid and clean. 456 * also, unbusy pages that were not actually requested. 457 */ 458 459 if (error) { 460 genfs_rel_pages(pgs, npages); 461 mutex_exit(uobj->vmobjlock); 462 UVMHIST_LOG(ubchist, "returning error %jd", error,0,0,0); 463 goto out_err_free; 464 } 465 466 out: 467 UVMHIST_LOG(ubchist, "succeeding, npages %jd", npages,0,0,0); 468 error = 0; 469 mutex_enter(&uvm_pageqlock); 470 for (i = 0; i < npages; i++) { 471 struct vm_page *pg = pgs[i]; 472 if (pg == NULL) { 473 continue; 474 } 475 UVMHIST_LOG(ubchist, "examining pg %#jx flags 0x%jx", 476 (uintptr_t)pg, pg->flags, 0,0); 477 if (pg->flags & PG_FAKE && !overwrite) { 478 pg->flags &= ~(PG_FAKE); 479 pmap_clear_modify(pgs[i]); 480 } 481 KASSERT(!memwrite || !blockalloc || (pg->flags & PG_RDONLY) == 0); 482 if (i < ridx || i >= ridx + orignmempages || async) { 483 UVMHIST_LOG(ubchist, "unbusy pg %#jx offset 0x%jx", 484 (uintptr_t)pg, pg->offset,0,0); 485 if (pg->flags & PG_WANTED) { 486 wakeup(pg); 487 } 488 if (pg->flags & PG_FAKE) { 489 KASSERT(overwrite); 490 uvm_pagezero(pg); 491 } 492 if (pg->flags & PG_RELEASED) { 493 uvm_pagefree(pg); 494 continue; 495 } 496 uvm_pageenqueue(pg); 497 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE); 498 UVM_PAGE_OWN(pg, NULL); 499 } 500 } 501 mutex_exit(&uvm_pageqlock); 502 if (memwrite) { 503 genfs_markdirty(vp); 504 } 505 mutex_exit(uobj->vmobjlock); 506 if (ap->a_m != NULL) { 507 memcpy(ap->a_m, &pgs[ridx], 508 orignmempages * sizeof(struct vm_page *)); 509 } 510 511 out_err_free: 512 if (pgs != NULL && pgs != pgs_onstack) 513 kmem_free(pgs, pgs_size); 514 out_err: 515 if (trans_mount != NULL) { 516 if (holds_wapbl) 517 WAPBL_END(trans_mount); 518 fstrans_done(trans_mount); 519 } 520 return error; 521 } 522 523 /* 524 * genfs_getpages_read: Read the pages in with VOP_BMAP/VOP_STRATEGY. 525 * 526 * "glocked" (which is currently not actually used) tells us not whether 527 * the genfs_node is locked on entry (it always is) but whether it was 528 * locked on entry to genfs_getpages. 529 */ 530 static int 531 genfs_getpages_read(struct vnode *vp, struct vm_page **pgs, int npages, 532 off_t startoffset, off_t diskeof, 533 bool async, bool memwrite, bool blockalloc, bool glocked) 534 { 535 struct uvm_object * const uobj = &vp->v_uobj; 536 const int fs_bshift = (vp->v_type != VBLK) ? 537 vp->v_mount->mnt_fs_bshift : DEV_BSHIFT; 538 const int dev_bshift = (vp->v_type != VBLK) ? 539 vp->v_mount->mnt_dev_bshift : DEV_BSHIFT; 540 kauth_cred_t const cred = curlwp->l_cred; /* XXXUBC curlwp */ 541 size_t bytes, iobytes, tailstart, tailbytes, totalbytes, skipbytes; 542 vaddr_t kva; 543 struct buf *bp, *mbp; 544 bool sawhole = false; 545 int i; 546 int error = 0; 547 548 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 549 550 /* 551 * read the desired page(s). 552 */ 553 554 totalbytes = npages << PAGE_SHIFT; 555 bytes = MIN(totalbytes, MAX(diskeof - startoffset, 0)); 556 tailbytes = totalbytes - bytes; 557 skipbytes = 0; 558 559 kva = uvm_pagermapin(pgs, npages, 560 UVMPAGER_MAPIN_READ | (async ? 0 : UVMPAGER_MAPIN_WAITOK)); 561 if (kva == 0) 562 return EBUSY; 563 564 if (uvm.aiodone_queue == NULL) 565 async = 0; 566 567 mbp = getiobuf(vp, true); 568 mbp->b_bufsize = totalbytes; 569 mbp->b_data = (void *)kva; 570 mbp->b_resid = mbp->b_bcount = bytes; 571 mbp->b_cflags = BC_BUSY; 572 if (async) { 573 mbp->b_flags = B_READ | B_ASYNC; 574 mbp->b_iodone = uvm_aio_biodone; 575 } else { 576 mbp->b_flags = B_READ; 577 mbp->b_iodone = NULL; 578 } 579 if (async) 580 BIO_SETPRIO(mbp, BPRIO_TIMELIMITED); 581 else 582 BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL); 583 584 /* 585 * if EOF is in the middle of the range, zero the part past EOF. 586 * skip over pages which are not PG_FAKE since in that case they have 587 * valid data that we need to preserve. 588 */ 589 590 tailstart = bytes; 591 while (tailbytes > 0) { 592 const int len = PAGE_SIZE - (tailstart & PAGE_MASK); 593 594 KASSERT(len <= tailbytes); 595 if ((pgs[tailstart >> PAGE_SHIFT]->flags & PG_FAKE) != 0) { 596 memset((void *)(kva + tailstart), 0, len); 597 UVMHIST_LOG(ubchist, "tailbytes %#jx 0x%jx 0x%jx", 598 (uintptr_t)kva, tailstart, len, 0); 599 } 600 tailstart += len; 601 tailbytes -= len; 602 } 603 604 /* 605 * now loop over the pages, reading as needed. 606 */ 607 608 bp = NULL; 609 off_t offset; 610 for (offset = startoffset; 611 bytes > 0; 612 offset += iobytes, bytes -= iobytes) { 613 int run; 614 daddr_t lbn, blkno; 615 int pidx; 616 struct vnode *devvp; 617 618 /* 619 * skip pages which don't need to be read. 620 */ 621 622 pidx = (offset - startoffset) >> PAGE_SHIFT; 623 while ((pgs[pidx]->flags & PG_FAKE) == 0) { 624 size_t b; 625 626 KASSERT((offset & (PAGE_SIZE - 1)) == 0); 627 if ((pgs[pidx]->flags & PG_RDONLY)) { 628 sawhole = true; 629 } 630 b = MIN(PAGE_SIZE, bytes); 631 offset += b; 632 bytes -= b; 633 skipbytes += b; 634 pidx++; 635 UVMHIST_LOG(ubchist, "skipping, new offset 0x%jx", 636 offset, 0,0,0); 637 if (bytes == 0) { 638 goto loopdone; 639 } 640 } 641 642 /* 643 * bmap the file to find out the blkno to read from and 644 * how much we can read in one i/o. if bmap returns an error, 645 * skip the rest of the top-level i/o. 646 */ 647 648 lbn = offset >> fs_bshift; 649 error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run); 650 if (error) { 651 UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%jx -> %jd\n", 652 lbn,error,0,0); 653 skipbytes += bytes; 654 bytes = 0; 655 goto loopdone; 656 } 657 658 /* 659 * see how many pages can be read with this i/o. 660 * reduce the i/o size if necessary to avoid 661 * overwriting pages with valid data. 662 */ 663 664 iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset, 665 bytes); 666 if (offset + iobytes > round_page(offset)) { 667 int pcount; 668 669 pcount = 1; 670 while (pidx + pcount < npages && 671 pgs[pidx + pcount]->flags & PG_FAKE) { 672 pcount++; 673 } 674 iobytes = MIN(iobytes, (pcount << PAGE_SHIFT) - 675 (offset - trunc_page(offset))); 676 } 677 678 /* 679 * if this block isn't allocated, zero it instead of 680 * reading it. unless we are going to allocate blocks, 681 * mark the pages we zeroed PG_RDONLY. 682 */ 683 684 if (blkno == (daddr_t)-1) { 685 int holepages = (round_page(offset + iobytes) - 686 trunc_page(offset)) >> PAGE_SHIFT; 687 UVMHIST_LOG(ubchist, "lbn 0x%jx -> HOLE", lbn,0,0,0); 688 689 sawhole = true; 690 memset((char *)kva + (offset - startoffset), 0, 691 iobytes); 692 skipbytes += iobytes; 693 694 mutex_enter(uobj->vmobjlock); 695 for (i = 0; i < holepages; i++) { 696 if (memwrite) { 697 pgs[pidx + i]->flags &= ~PG_CLEAN; 698 } 699 if (!blockalloc) { 700 pgs[pidx + i]->flags |= PG_RDONLY; 701 } 702 } 703 mutex_exit(uobj->vmobjlock); 704 continue; 705 } 706 707 /* 708 * allocate a sub-buf for this piece of the i/o 709 * (or just use mbp if there's only 1 piece), 710 * and start it going. 711 */ 712 713 if (offset == startoffset && iobytes == bytes) { 714 bp = mbp; 715 } else { 716 UVMHIST_LOG(ubchist, "vp %#jx bp %#jx num now %jd", 717 (uintptr_t)vp, (uintptr_t)bp, vp->v_numoutput, 0); 718 bp = getiobuf(vp, true); 719 nestiobuf_setup(mbp, bp, offset - startoffset, iobytes); 720 } 721 bp->b_lblkno = 0; 722 723 /* adjust physical blkno for partial blocks */ 724 bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >> 725 dev_bshift); 726 727 UVMHIST_LOG(ubchist, 728 "bp %#jx offset 0x%x bcount 0x%x blkno 0x%x", 729 (uintptr_t)bp, offset, bp->b_bcount, bp->b_blkno); 730 731 VOP_STRATEGY(devvp, bp); 732 } 733 734 loopdone: 735 nestiobuf_done(mbp, skipbytes, error); 736 if (async) { 737 UVMHIST_LOG(ubchist, "returning 0 (async)",0,0,0,0); 738 return 0; 739 } 740 if (bp != NULL) { 741 error = biowait(mbp); 742 } 743 744 /* Remove the mapping (make KVA available as soon as possible) */ 745 uvm_pagermapout(kva, npages); 746 747 /* 748 * if this we encountered a hole then we have to do a little more work. 749 * for read faults, we marked the page PG_RDONLY so that future 750 * write accesses to the page will fault again. 751 * for write faults, we must make sure that the backing store for 752 * the page is completely allocated while the pages are locked. 753 */ 754 755 if (!error && sawhole && blockalloc) { 756 error = GOP_ALLOC(vp, startoffset, 757 npages << PAGE_SHIFT, 0, cred); 758 UVMHIST_LOG(ubchist, "gop_alloc off 0x%jx/0x%jx -> %jd", 759 startoffset, npages << PAGE_SHIFT, error,0); 760 if (!error) { 761 mutex_enter(uobj->vmobjlock); 762 for (i = 0; i < npages; i++) { 763 struct vm_page *pg = pgs[i]; 764 765 if (pg == NULL) { 766 continue; 767 } 768 pg->flags &= ~(PG_CLEAN|PG_RDONLY); 769 UVMHIST_LOG(ubchist, "mark dirty pg %#jx", 770 (uintptr_t)pg, 0, 0, 0); 771 } 772 mutex_exit(uobj->vmobjlock); 773 } 774 } 775 776 putiobuf(mbp); 777 return error; 778 } 779 780 /* 781 * generic VM putpages routine. 782 * Write the given range of pages to backing store. 783 * 784 * => "offhi == 0" means flush all pages at or after "offlo". 785 * => object should be locked by caller. we return with the 786 * object unlocked. 787 * => if PGO_CLEANIT or PGO_SYNCIO is set, we may block (due to I/O). 788 * thus, a caller might want to unlock higher level resources 789 * (e.g. vm_map) before calling flush. 790 * => if neither PGO_CLEANIT nor PGO_SYNCIO is set, we will not block 791 * => if PGO_ALLPAGES is set, then all pages in the object will be processed. 792 * => NOTE: we rely on the fact that the object's memq is a TAILQ and 793 * that new pages are inserted on the tail end of the list. thus, 794 * we can make a complete pass through the object in one go by starting 795 * at the head and working towards the tail (new pages are put in 796 * front of us). 797 * => NOTE: we are allowed to lock the page queues, so the caller 798 * must not be holding the page queue lock. 799 * 800 * note on "cleaning" object and PG_BUSY pages: 801 * this routine is holding the lock on the object. the only time 802 * that it can run into a PG_BUSY page that it does not own is if 803 * some other process has started I/O on the page (e.g. either 804 * a pagein, or a pageout). if the PG_BUSY page is being paged 805 * in, then it can not be dirty (!PG_CLEAN) because no one has 806 * had a chance to modify it yet. if the PG_BUSY page is being 807 * paged out then it means that someone else has already started 808 * cleaning the page for us (how nice!). in this case, if we 809 * have syncio specified, then after we make our pass through the 810 * object we need to wait for the other PG_BUSY pages to clear 811 * off (i.e. we need to do an iosync). also note that once a 812 * page is PG_BUSY it must stay in its object until it is un-busyed. 813 * 814 * note on page traversal: 815 * we can traverse the pages in an object either by going down the 816 * linked list in "uobj->memq", or we can go over the address range 817 * by page doing hash table lookups for each address. depending 818 * on how many pages are in the object it may be cheaper to do one 819 * or the other. we set "by_list" to true if we are using memq. 820 * if the cost of a hash lookup was equal to the cost of the list 821 * traversal we could compare the number of pages in the start->stop 822 * range to the total number of pages in the object. however, it 823 * seems that a hash table lookup is more expensive than the linked 824 * list traversal, so we multiply the number of pages in the 825 * range by an estimate of the relatively higher cost of the hash lookup. 826 */ 827 828 int 829 genfs_putpages(void *v) 830 { 831 struct vop_putpages_args /* { 832 struct vnode *a_vp; 833 voff_t a_offlo; 834 voff_t a_offhi; 835 int a_flags; 836 } */ * const ap = v; 837 838 return genfs_do_putpages(ap->a_vp, ap->a_offlo, ap->a_offhi, 839 ap->a_flags, NULL); 840 } 841 842 int 843 genfs_do_putpages(struct vnode *vp, off_t startoff, off_t endoff, 844 int origflags, struct vm_page **busypg) 845 { 846 struct uvm_object * const uobj = &vp->v_uobj; 847 kmutex_t * const slock = uobj->vmobjlock; 848 off_t off; 849 int i, error, npages, nback; 850 int freeflag; 851 /* 852 * This array is larger than it should so that it's size is constant. 853 * The right size is MAXPAGES. 854 */ 855 struct vm_page *pgs[MAXPHYS / MIN_PAGE_SIZE]; 856 #define MAXPAGES (MAXPHYS / PAGE_SIZE) 857 struct vm_page *pg, *nextpg, *tpg, curmp, endmp; 858 bool wasclean, by_list, needs_clean, yld; 859 bool async = (origflags & PGO_SYNCIO) == 0; 860 bool pagedaemon = curlwp == uvm.pagedaemon_lwp; 861 struct lwp * const l = curlwp ? curlwp : &lwp0; 862 struct genfs_node * const gp = VTOG(vp); 863 struct mount *trans_mp; 864 int flags; 865 int dirtygen; 866 bool modified; 867 bool holds_wapbl; 868 bool cleanall; 869 bool onworklst; 870 871 UVMHIST_FUNC("genfs_putpages"); UVMHIST_CALLED(ubchist); 872 873 KASSERT(origflags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)); 874 KASSERT((startoff & PAGE_MASK) == 0 && (endoff & PAGE_MASK) == 0); 875 KASSERT(startoff < endoff || endoff == 0); 876 877 UVMHIST_LOG(ubchist, "vp %#jx pages %jd off 0x%jx len 0x%jx", 878 (uintptr_t)vp, uobj->uo_npages, startoff, endoff - startoff); 879 880 #ifdef DIAGNOSTIC 881 if ((origflags & PGO_JOURNALLOCKED) && vp->v_mount->mnt_wapbl) 882 WAPBL_JLOCK_ASSERT(vp->v_mount); 883 #endif 884 885 trans_mp = NULL; 886 holds_wapbl = false; 887 888 retry: 889 modified = false; 890 flags = origflags; 891 KASSERT((vp->v_iflag & VI_ONWORKLST) != 0 || 892 (vp->v_iflag & VI_WRMAPDIRTY) == 0); 893 if (uobj->uo_npages == 0) { 894 if (vp->v_iflag & VI_ONWORKLST) { 895 vp->v_iflag &= ~VI_WRMAPDIRTY; 896 if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL) 897 vn_syncer_remove_from_worklist(vp); 898 } 899 if (trans_mp) { 900 if (holds_wapbl) 901 WAPBL_END(trans_mp); 902 fstrans_done(trans_mp); 903 } 904 mutex_exit(slock); 905 return (0); 906 } 907 908 /* 909 * the vnode has pages, set up to process the request. 910 */ 911 912 if (trans_mp == NULL && (flags & PGO_CLEANIT) != 0) { 913 if (pagedaemon) { 914 /* Pagedaemon must not sleep here. */ 915 trans_mp = vp->v_mount; 916 error = fstrans_start_nowait(trans_mp); 917 if (error) { 918 mutex_exit(slock); 919 return error; 920 } 921 } else { 922 /* 923 * Cannot use vdeadcheck() here as this operation 924 * usually gets used from VOP_RECLAIM(). Test for 925 * change of v_mount instead and retry on change. 926 */ 927 mutex_exit(slock); 928 trans_mp = vp->v_mount; 929 fstrans_start(trans_mp); 930 if (vp->v_mount != trans_mp) { 931 fstrans_done(trans_mp); 932 trans_mp = NULL; 933 } else { 934 holds_wapbl = (trans_mp->mnt_wapbl && 935 (origflags & PGO_JOURNALLOCKED) == 0); 936 if (holds_wapbl) { 937 error = WAPBL_BEGIN(trans_mp); 938 if (error) { 939 fstrans_done(trans_mp); 940 return error; 941 } 942 } 943 } 944 mutex_enter(slock); 945 goto retry; 946 } 947 } 948 949 error = 0; 950 wasclean = (vp->v_numoutput == 0); 951 off = startoff; 952 if (endoff == 0 || flags & PGO_ALLPAGES) { 953 endoff = trunc_page(LLONG_MAX); 954 } 955 by_list = (uobj->uo_npages <= 956 ((endoff - startoff) >> PAGE_SHIFT) * UVM_PAGE_TREE_PENALTY); 957 958 /* 959 * if this vnode is known not to have dirty pages, 960 * don't bother to clean it out. 961 */ 962 963 if ((vp->v_iflag & VI_ONWORKLST) == 0) { 964 #if !defined(DEBUG) 965 if ((flags & (PGO_FREE|PGO_DEACTIVATE)) == 0) { 966 goto skip_scan; 967 } 968 #endif /* !defined(DEBUG) */ 969 flags &= ~PGO_CLEANIT; 970 } 971 972 /* 973 * start the loop. when scanning by list, hold the last page 974 * in the list before we start. pages allocated after we start 975 * will be added to the end of the list, so we can stop at the 976 * current last page. 977 */ 978 979 cleanall = (flags & PGO_CLEANIT) != 0 && wasclean && 980 startoff == 0 && endoff == trunc_page(LLONG_MAX) && 981 (vp->v_iflag & VI_ONWORKLST) != 0; 982 dirtygen = gp->g_dirtygen; 983 freeflag = pagedaemon ? PG_PAGEOUT : PG_RELEASED; 984 if (by_list) { 985 curmp.flags = PG_MARKER; 986 endmp.flags = PG_MARKER; 987 pg = TAILQ_FIRST(&uobj->memq); 988 TAILQ_INSERT_TAIL(&uobj->memq, &endmp, listq.queue); 989 } else { 990 pg = uvm_pagelookup(uobj, off); 991 } 992 nextpg = NULL; 993 while (by_list || off < endoff) { 994 995 /* 996 * if the current page is not interesting, move on to the next. 997 */ 998 999 KASSERT(pg == NULL || (pg->flags & PG_MARKER) != 0 || 1000 pg->uobject == uobj); 1001 KASSERT(pg == NULL || 1002 (pg->flags & (PG_RELEASED|PG_PAGEOUT)) == 0 || 1003 (pg->flags & (PG_BUSY|PG_MARKER)) != 0); 1004 if (by_list) { 1005 if (pg == &endmp) { 1006 break; 1007 } 1008 if (pg->flags & PG_MARKER) { 1009 pg = TAILQ_NEXT(pg, listq.queue); 1010 continue; 1011 } 1012 if (pg->offset < startoff || pg->offset >= endoff || 1013 pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 1014 if (pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 1015 wasclean = false; 1016 } 1017 pg = TAILQ_NEXT(pg, listq.queue); 1018 continue; 1019 } 1020 off = pg->offset; 1021 } else if (pg == NULL || pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 1022 if (pg != NULL) { 1023 wasclean = false; 1024 } 1025 off += PAGE_SIZE; 1026 if (off < endoff) { 1027 pg = uvm_pagelookup(uobj, off); 1028 } 1029 continue; 1030 } 1031 1032 /* 1033 * if the current page needs to be cleaned and it's busy, 1034 * wait for it to become unbusy. 1035 */ 1036 1037 yld = (l->l_cpu->ci_schedstate.spc_flags & 1038 SPCF_SHOULDYIELD) && !pagedaemon; 1039 if (pg->flags & PG_BUSY || yld) { 1040 UVMHIST_LOG(ubchist, "busy %#jx", (uintptr_t)pg, 1041 0, 0, 0); 1042 if (flags & PGO_BUSYFAIL && pg->flags & PG_BUSY) { 1043 UVMHIST_LOG(ubchist, "busyfail %#jx", 1044 (uintptr_t)pg, 0, 0, 0); 1045 error = EDEADLK; 1046 if (busypg != NULL) 1047 *busypg = pg; 1048 break; 1049 } 1050 if (pagedaemon) { 1051 /* 1052 * someone has taken the page while we 1053 * dropped the lock for fstrans_start. 1054 */ 1055 break; 1056 } 1057 if (by_list) { 1058 TAILQ_INSERT_BEFORE(pg, &curmp, listq.queue); 1059 UVMHIST_LOG(ubchist, "curmp next %#jx", 1060 (uintptr_t)TAILQ_NEXT(&curmp, listq.queue), 1061 0, 0, 0); 1062 } 1063 if (yld) { 1064 mutex_exit(slock); 1065 preempt(); 1066 mutex_enter(slock); 1067 } else { 1068 pg->flags |= PG_WANTED; 1069 UVM_UNLOCK_AND_WAIT(pg, slock, 0, "genput", 0); 1070 mutex_enter(slock); 1071 } 1072 if (by_list) { 1073 UVMHIST_LOG(ubchist, "after next %#jx", 1074 (uintptr_t)TAILQ_NEXT(&curmp, listq.queue), 1075 0, 0, 0); 1076 pg = TAILQ_NEXT(&curmp, listq.queue); 1077 TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue); 1078 } else { 1079 pg = uvm_pagelookup(uobj, off); 1080 } 1081 continue; 1082 } 1083 1084 /* 1085 * if we're freeing, remove all mappings of the page now. 1086 * if we're cleaning, check if the page is needs to be cleaned. 1087 */ 1088 1089 if (flags & PGO_FREE) { 1090 pmap_page_protect(pg, VM_PROT_NONE); 1091 } else if (flags & PGO_CLEANIT) { 1092 1093 /* 1094 * if we still have some hope to pull this vnode off 1095 * from the syncer queue, write-protect the page. 1096 */ 1097 1098 if (cleanall && wasclean && 1099 gp->g_dirtygen == dirtygen) { 1100 1101 /* 1102 * uobj pages get wired only by uvm_fault 1103 * where uobj is locked. 1104 */ 1105 1106 if (pg->wire_count == 0) { 1107 pmap_page_protect(pg, 1108 VM_PROT_READ|VM_PROT_EXECUTE); 1109 } else { 1110 cleanall = false; 1111 } 1112 } 1113 } 1114 1115 if (flags & PGO_CLEANIT) { 1116 needs_clean = pmap_clear_modify(pg) || 1117 (pg->flags & PG_CLEAN) == 0; 1118 pg->flags |= PG_CLEAN; 1119 } else { 1120 needs_clean = false; 1121 } 1122 1123 /* 1124 * if we're cleaning, build a cluster. 1125 * the cluster will consist of pages which are currently dirty, 1126 * but they will be returned to us marked clean. 1127 * if not cleaning, just operate on the one page. 1128 */ 1129 1130 if (needs_clean) { 1131 KDASSERT((vp->v_iflag & VI_ONWORKLST)); 1132 wasclean = false; 1133 memset(pgs, 0, sizeof(pgs)); 1134 pg->flags |= PG_BUSY; 1135 UVM_PAGE_OWN(pg, "genfs_putpages"); 1136 1137 /* 1138 * let the fs constrain the offset range of the cluster. 1139 * we additionally constrain the range here such that 1140 * it fits in the "pgs" pages array. 1141 */ 1142 1143 off_t fslo, fshi, genlo, lo; 1144 GOP_PUTRANGE(vp, off, &fslo, &fshi); 1145 KASSERT(fslo == trunc_page(fslo)); 1146 KASSERT(fslo <= off); 1147 KASSERT(fshi == trunc_page(fshi)); 1148 KASSERT(fshi == 0 || off < fshi); 1149 1150 if (off > MAXPHYS / 2) 1151 genlo = trunc_page(off - (MAXPHYS / 2)); 1152 else 1153 genlo = 0; 1154 lo = MAX(fslo, genlo); 1155 1156 /* 1157 * first look backward. 1158 */ 1159 1160 npages = (off - lo) >> PAGE_SHIFT; 1161 nback = npages; 1162 uvn_findpages(uobj, off - PAGE_SIZE, &nback, &pgs[0], 1163 UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY|UFP_BACKWARD); 1164 if (nback) { 1165 memmove(&pgs[0], &pgs[npages - nback], 1166 nback * sizeof(pgs[0])); 1167 if (npages - nback < nback) 1168 memset(&pgs[nback], 0, 1169 (npages - nback) * sizeof(pgs[0])); 1170 else 1171 memset(&pgs[npages - nback], 0, 1172 nback * sizeof(pgs[0])); 1173 } 1174 1175 /* 1176 * then plug in our page of interest. 1177 */ 1178 1179 pgs[nback] = pg; 1180 1181 /* 1182 * then look forward to fill in the remaining space in 1183 * the array of pages. 1184 */ 1185 1186 npages = MAXPAGES - nback - 1; 1187 if (fshi) 1188 npages = MIN(npages, 1189 (fshi - off - 1) >> PAGE_SHIFT); 1190 uvn_findpages(uobj, off + PAGE_SIZE, &npages, 1191 &pgs[nback + 1], 1192 UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY); 1193 npages += nback + 1; 1194 } else { 1195 pgs[0] = pg; 1196 npages = 1; 1197 nback = 0; 1198 } 1199 1200 /* 1201 * apply FREE or DEACTIVATE options if requested. 1202 */ 1203 1204 if (flags & (PGO_DEACTIVATE|PGO_FREE)) { 1205 mutex_enter(&uvm_pageqlock); 1206 } 1207 for (i = 0; i < npages; i++) { 1208 tpg = pgs[i]; 1209 KASSERT(tpg->uobject == uobj); 1210 if (by_list && tpg == TAILQ_NEXT(pg, listq.queue)) 1211 pg = tpg; 1212 if (tpg->offset < startoff || tpg->offset >= endoff) 1213 continue; 1214 if (flags & PGO_DEACTIVATE && tpg->wire_count == 0) { 1215 uvm_pagedeactivate(tpg); 1216 } else if (flags & PGO_FREE) { 1217 pmap_page_protect(tpg, VM_PROT_NONE); 1218 if (tpg->flags & PG_BUSY) { 1219 tpg->flags |= freeflag; 1220 if (pagedaemon) { 1221 uvm_pageout_start(1); 1222 uvm_pagedequeue(tpg); 1223 } 1224 } else { 1225 1226 /* 1227 * ``page is not busy'' 1228 * implies that npages is 1 1229 * and needs_clean is false. 1230 */ 1231 1232 nextpg = TAILQ_NEXT(tpg, listq.queue); 1233 uvm_pagefree(tpg); 1234 if (pagedaemon) 1235 uvmexp.pdfreed++; 1236 } 1237 } 1238 } 1239 if (flags & (PGO_DEACTIVATE|PGO_FREE)) { 1240 mutex_exit(&uvm_pageqlock); 1241 } 1242 if (needs_clean) { 1243 modified = true; 1244 1245 /* 1246 * start the i/o. if we're traversing by list, 1247 * keep our place in the list with a marker page. 1248 */ 1249 1250 if (by_list) { 1251 TAILQ_INSERT_AFTER(&uobj->memq, pg, &curmp, 1252 listq.queue); 1253 } 1254 mutex_exit(slock); 1255 error = GOP_WRITE(vp, pgs, npages, flags); 1256 mutex_enter(slock); 1257 if (by_list) { 1258 pg = TAILQ_NEXT(&curmp, listq.queue); 1259 TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue); 1260 } 1261 if (error) { 1262 break; 1263 } 1264 if (by_list) { 1265 continue; 1266 } 1267 } 1268 1269 /* 1270 * find the next page and continue if there was no error. 1271 */ 1272 1273 if (by_list) { 1274 if (nextpg) { 1275 pg = nextpg; 1276 nextpg = NULL; 1277 } else { 1278 pg = TAILQ_NEXT(pg, listq.queue); 1279 } 1280 } else { 1281 off += (npages - nback) << PAGE_SHIFT; 1282 if (off < endoff) { 1283 pg = uvm_pagelookup(uobj, off); 1284 } 1285 } 1286 } 1287 if (by_list) { 1288 TAILQ_REMOVE(&uobj->memq, &endmp, listq.queue); 1289 } 1290 1291 if (modified && (vp->v_iflag & VI_WRMAPDIRTY) != 0 && 1292 (vp->v_type != VBLK || 1293 (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) { 1294 GOP_MARKUPDATE(vp, GOP_UPDATE_MODIFIED); 1295 } 1296 1297 /* 1298 * if we're cleaning and there was nothing to clean, 1299 * take us off the syncer list. if we started any i/o 1300 * and we're doing sync i/o, wait for all writes to finish. 1301 */ 1302 1303 if (cleanall && wasclean && gp->g_dirtygen == dirtygen && 1304 (vp->v_iflag & VI_ONWORKLST) != 0) { 1305 #if defined(DEBUG) 1306 TAILQ_FOREACH(pg, &uobj->memq, listq.queue) { 1307 if ((pg->flags & (PG_FAKE | PG_MARKER)) != 0) { 1308 continue; 1309 } 1310 if ((pg->flags & PG_CLEAN) == 0) { 1311 printf("%s: %p: !CLEAN\n", __func__, pg); 1312 } 1313 if (pmap_is_modified(pg)) { 1314 printf("%s: %p: modified\n", __func__, pg); 1315 } 1316 } 1317 #endif /* defined(DEBUG) */ 1318 vp->v_iflag &= ~VI_WRMAPDIRTY; 1319 if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL) 1320 vn_syncer_remove_from_worklist(vp); 1321 } 1322 1323 #if !defined(DEBUG) 1324 skip_scan: 1325 #endif /* !defined(DEBUG) */ 1326 1327 /* Wait for output to complete. */ 1328 if (!wasclean && !async && vp->v_numoutput != 0) { 1329 while (vp->v_numoutput != 0) 1330 cv_wait(&vp->v_cv, slock); 1331 } 1332 onworklst = (vp->v_iflag & VI_ONWORKLST) != 0; 1333 mutex_exit(slock); 1334 1335 if ((flags & PGO_RECLAIM) != 0 && onworklst) { 1336 /* 1337 * in the case of PGO_RECLAIM, ensure to make the vnode clean. 1338 * retrying is not a big deal because, in many cases, 1339 * uobj->uo_npages is already 0 here. 1340 */ 1341 mutex_enter(slock); 1342 goto retry; 1343 } 1344 1345 if (trans_mp) { 1346 if (holds_wapbl) 1347 WAPBL_END(trans_mp); 1348 fstrans_done(trans_mp); 1349 } 1350 1351 return (error); 1352 } 1353 1354 /* 1355 * Default putrange method for file systems that do not care 1356 * how many pages are given to one GOP_WRITE() call. 1357 */ 1358 void 1359 genfs_gop_putrange(struct vnode *vp, off_t off, off_t *lop, off_t *hip) 1360 { 1361 1362 *lop = 0; 1363 *hip = 0; 1364 } 1365 1366 int 1367 genfs_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, int flags) 1368 { 1369 off_t off; 1370 vaddr_t kva; 1371 size_t len; 1372 int error; 1373 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1374 1375 UVMHIST_LOG(ubchist, "vp %#jx pgs %#jx npages %jd flags 0x%jx", 1376 (uintptr_t)vp, (uintptr_t)pgs, npages, flags); 1377 1378 off = pgs[0]->offset; 1379 kva = uvm_pagermapin(pgs, npages, 1380 UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK); 1381 len = npages << PAGE_SHIFT; 1382 1383 KASSERT(uvm.aiodone_queue != NULL); 1384 error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE, 1385 uvm_aio_biodone); 1386 1387 return error; 1388 } 1389 1390 int 1391 genfs_gop_write_rwmap(struct vnode *vp, struct vm_page **pgs, int npages, int flags) 1392 { 1393 off_t off; 1394 vaddr_t kva; 1395 size_t len; 1396 int error; 1397 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1398 1399 UVMHIST_LOG(ubchist, "vp %#jx pgs %#jx npages %jd flags 0x%jx", 1400 (uintptr_t)vp, (uintptr_t)pgs, npages, flags); 1401 1402 off = pgs[0]->offset; 1403 kva = uvm_pagermapin(pgs, npages, 1404 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 1405 len = npages << PAGE_SHIFT; 1406 1407 KASSERT(uvm.aiodone_queue != NULL); 1408 error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE, 1409 uvm_aio_biodone); 1410 1411 return error; 1412 } 1413 1414 /* 1415 * Backend routine for doing I/O to vnode pages. Pages are already locked 1416 * and mapped into kernel memory. Here we just look up the underlying 1417 * device block addresses and call the strategy routine. 1418 */ 1419 1420 static int 1421 genfs_do_io(struct vnode *vp, off_t off, vaddr_t kva, size_t len, int flags, 1422 enum uio_rw rw, void (*iodone)(struct buf *)) 1423 { 1424 int s, error; 1425 int fs_bshift, dev_bshift; 1426 off_t eof, offset, startoffset; 1427 size_t bytes, iobytes, skipbytes; 1428 struct buf *mbp, *bp; 1429 const bool async = (flags & PGO_SYNCIO) == 0; 1430 const bool lazy = (flags & PGO_LAZY) == 0; 1431 const bool iowrite = rw == UIO_WRITE; 1432 const int brw = iowrite ? B_WRITE : B_READ; 1433 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1434 1435 UVMHIST_LOG(ubchist, "vp %#jx kva %#jx len 0x%jx flags 0x%jx", 1436 (uintptr_t)vp, (uintptr_t)kva, len, flags); 1437 1438 KASSERT(vp->v_size <= vp->v_writesize); 1439 GOP_SIZE(vp, vp->v_writesize, &eof, 0); 1440 if (vp->v_type != VBLK) { 1441 fs_bshift = vp->v_mount->mnt_fs_bshift; 1442 dev_bshift = vp->v_mount->mnt_dev_bshift; 1443 } else { 1444 fs_bshift = DEV_BSHIFT; 1445 dev_bshift = DEV_BSHIFT; 1446 } 1447 error = 0; 1448 startoffset = off; 1449 bytes = MIN(len, eof - startoffset); 1450 skipbytes = 0; 1451 KASSERT(bytes != 0); 1452 1453 if (iowrite) { 1454 mutex_enter(vp->v_interlock); 1455 vp->v_numoutput += 2; 1456 mutex_exit(vp->v_interlock); 1457 } 1458 mbp = getiobuf(vp, true); 1459 UVMHIST_LOG(ubchist, "vp %#jx mbp %#jx num now %jd bytes 0x%jx", 1460 (uintptr_t)vp, (uintptr_t)mbp, vp->v_numoutput, bytes); 1461 mbp->b_bufsize = len; 1462 mbp->b_data = (void *)kva; 1463 mbp->b_resid = mbp->b_bcount = bytes; 1464 mbp->b_cflags = BC_BUSY | BC_AGE; 1465 if (async) { 1466 mbp->b_flags = brw | B_ASYNC; 1467 mbp->b_iodone = iodone; 1468 } else { 1469 mbp->b_flags = brw; 1470 mbp->b_iodone = NULL; 1471 } 1472 if (curlwp == uvm.pagedaemon_lwp) 1473 BIO_SETPRIO(mbp, BPRIO_TIMELIMITED); 1474 else if (async || lazy) 1475 BIO_SETPRIO(mbp, BPRIO_TIMENONCRITICAL); 1476 else 1477 BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL); 1478 1479 bp = NULL; 1480 for (offset = startoffset; 1481 bytes > 0; 1482 offset += iobytes, bytes -= iobytes) { 1483 int run; 1484 daddr_t lbn, blkno; 1485 struct vnode *devvp; 1486 1487 /* 1488 * bmap the file to find out the blkno to read from and 1489 * how much we can read in one i/o. if bmap returns an error, 1490 * skip the rest of the top-level i/o. 1491 */ 1492 1493 lbn = offset >> fs_bshift; 1494 error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run); 1495 if (error) { 1496 UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%jx -> %jd\n", 1497 lbn, error, 0, 0); 1498 skipbytes += bytes; 1499 bytes = 0; 1500 goto loopdone; 1501 } 1502 1503 /* 1504 * see how many pages can be read with this i/o. 1505 * reduce the i/o size if necessary to avoid 1506 * overwriting pages with valid data. 1507 */ 1508 1509 iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset, 1510 bytes); 1511 1512 /* 1513 * if this block isn't allocated, zero it instead of 1514 * reading it. unless we are going to allocate blocks, 1515 * mark the pages we zeroed PG_RDONLY. 1516 */ 1517 1518 if (blkno == (daddr_t)-1) { 1519 if (!iowrite) { 1520 memset((char *)kva + (offset - startoffset), 0, 1521 iobytes); 1522 } 1523 skipbytes += iobytes; 1524 continue; 1525 } 1526 1527 /* 1528 * allocate a sub-buf for this piece of the i/o 1529 * (or just use mbp if there's only 1 piece), 1530 * and start it going. 1531 */ 1532 1533 if (offset == startoffset && iobytes == bytes) { 1534 bp = mbp; 1535 } else { 1536 UVMHIST_LOG(ubchist, "vp %#jx bp %#jx num now %jd", 1537 (uintptr_t)vp, (uintptr_t)bp, vp->v_numoutput, 0); 1538 bp = getiobuf(vp, true); 1539 nestiobuf_setup(mbp, bp, offset - startoffset, iobytes); 1540 } 1541 bp->b_lblkno = 0; 1542 1543 /* adjust physical blkno for partial blocks */ 1544 bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >> 1545 dev_bshift); 1546 1547 UVMHIST_LOG(ubchist, 1548 "bp %#jx offset 0x%jx bcount 0x%jx blkno 0x%jx", 1549 (uintptr_t)bp, offset, bp->b_bcount, bp->b_blkno); 1550 1551 VOP_STRATEGY(devvp, bp); 1552 } 1553 1554 loopdone: 1555 if (skipbytes) { 1556 UVMHIST_LOG(ubchist, "skipbytes %jd", skipbytes, 0,0,0); 1557 } 1558 nestiobuf_done(mbp, skipbytes, error); 1559 if (async) { 1560 UVMHIST_LOG(ubchist, "returning 0 (async)", 0,0,0,0); 1561 return (0); 1562 } 1563 UVMHIST_LOG(ubchist, "waiting for mbp %#jx", (uintptr_t)mbp, 0, 0, 0); 1564 error = biowait(mbp); 1565 s = splbio(); 1566 (*iodone)(mbp); 1567 splx(s); 1568 UVMHIST_LOG(ubchist, "returning, error %jd", error, 0, 0, 0); 1569 return (error); 1570 } 1571 1572 int 1573 genfs_compat_getpages(void *v) 1574 { 1575 struct vop_getpages_args /* { 1576 struct vnode *a_vp; 1577 voff_t a_offset; 1578 struct vm_page **a_m; 1579 int *a_count; 1580 int a_centeridx; 1581 vm_prot_t a_access_type; 1582 int a_advice; 1583 int a_flags; 1584 } */ *ap = v; 1585 1586 off_t origoffset; 1587 struct vnode *vp = ap->a_vp; 1588 struct uvm_object *uobj = &vp->v_uobj; 1589 struct vm_page *pg, **pgs; 1590 vaddr_t kva; 1591 int i, error, orignpages, npages; 1592 struct iovec iov; 1593 struct uio uio; 1594 kauth_cred_t cred = curlwp->l_cred; 1595 const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0; 1596 1597 error = 0; 1598 origoffset = ap->a_offset; 1599 orignpages = *ap->a_count; 1600 pgs = ap->a_m; 1601 1602 if (ap->a_flags & PGO_LOCKED) { 1603 uvn_findpages(uobj, origoffset, ap->a_count, ap->a_m, 1604 UFP_NOWAIT|UFP_NOALLOC| (memwrite ? UFP_NORDONLY : 0)); 1605 1606 error = ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0; 1607 if (error == 0 && memwrite) { 1608 genfs_markdirty(vp); 1609 } 1610 return error; 1611 } 1612 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= vp->v_size) { 1613 mutex_exit(uobj->vmobjlock); 1614 return EINVAL; 1615 } 1616 if ((ap->a_flags & PGO_SYNCIO) == 0) { 1617 mutex_exit(uobj->vmobjlock); 1618 return 0; 1619 } 1620 npages = orignpages; 1621 uvn_findpages(uobj, origoffset, &npages, pgs, UFP_ALL); 1622 mutex_exit(uobj->vmobjlock); 1623 kva = uvm_pagermapin(pgs, npages, 1624 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 1625 for (i = 0; i < npages; i++) { 1626 pg = pgs[i]; 1627 if ((pg->flags & PG_FAKE) == 0) { 1628 continue; 1629 } 1630 iov.iov_base = (char *)kva + (i << PAGE_SHIFT); 1631 iov.iov_len = PAGE_SIZE; 1632 uio.uio_iov = &iov; 1633 uio.uio_iovcnt = 1; 1634 uio.uio_offset = origoffset + (i << PAGE_SHIFT); 1635 uio.uio_rw = UIO_READ; 1636 uio.uio_resid = PAGE_SIZE; 1637 UIO_SETUP_SYSSPACE(&uio); 1638 /* XXX vn_lock */ 1639 error = VOP_READ(vp, &uio, 0, cred); 1640 if (error) { 1641 break; 1642 } 1643 if (uio.uio_resid) { 1644 memset(iov.iov_base, 0, uio.uio_resid); 1645 } 1646 } 1647 uvm_pagermapout(kva, npages); 1648 mutex_enter(uobj->vmobjlock); 1649 mutex_enter(&uvm_pageqlock); 1650 for (i = 0; i < npages; i++) { 1651 pg = pgs[i]; 1652 if (error && (pg->flags & PG_FAKE) != 0) { 1653 pg->flags |= PG_RELEASED; 1654 } else { 1655 pmap_clear_modify(pg); 1656 uvm_pageactivate(pg); 1657 } 1658 } 1659 if (error) { 1660 uvm_page_unbusy(pgs, npages); 1661 } 1662 mutex_exit(&uvm_pageqlock); 1663 if (error == 0 && memwrite) { 1664 genfs_markdirty(vp); 1665 } 1666 mutex_exit(uobj->vmobjlock); 1667 return error; 1668 } 1669 1670 int 1671 genfs_compat_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, 1672 int flags) 1673 { 1674 off_t offset; 1675 struct iovec iov; 1676 struct uio uio; 1677 kauth_cred_t cred = curlwp->l_cred; 1678 struct buf *bp; 1679 vaddr_t kva; 1680 int error; 1681 1682 offset = pgs[0]->offset; 1683 kva = uvm_pagermapin(pgs, npages, 1684 UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK); 1685 1686 iov.iov_base = (void *)kva; 1687 iov.iov_len = npages << PAGE_SHIFT; 1688 uio.uio_iov = &iov; 1689 uio.uio_iovcnt = 1; 1690 uio.uio_offset = offset; 1691 uio.uio_rw = UIO_WRITE; 1692 uio.uio_resid = npages << PAGE_SHIFT; 1693 UIO_SETUP_SYSSPACE(&uio); 1694 /* XXX vn_lock */ 1695 error = VOP_WRITE(vp, &uio, 0, cred); 1696 1697 mutex_enter(vp->v_interlock); 1698 vp->v_numoutput++; 1699 mutex_exit(vp->v_interlock); 1700 1701 bp = getiobuf(vp, true); 1702 bp->b_cflags = BC_BUSY | BC_AGE; 1703 bp->b_lblkno = offset >> vp->v_mount->mnt_fs_bshift; 1704 bp->b_data = (char *)kva; 1705 bp->b_bcount = npages << PAGE_SHIFT; 1706 bp->b_bufsize = npages << PAGE_SHIFT; 1707 bp->b_resid = 0; 1708 bp->b_error = error; 1709 uvm_aio_aiodone(bp); 1710 return (error); 1711 } 1712 1713 /* 1714 * Process a uio using direct I/O. If we reach a part of the request 1715 * which cannot be processed in this fashion for some reason, just return. 1716 * The caller must handle some additional part of the request using 1717 * buffered I/O before trying direct I/O again. 1718 */ 1719 1720 void 1721 genfs_directio(struct vnode *vp, struct uio *uio, int ioflag) 1722 { 1723 struct vmspace *vs; 1724 struct iovec *iov; 1725 vaddr_t va; 1726 size_t len; 1727 const int mask = DEV_BSIZE - 1; 1728 int error; 1729 bool need_wapbl = (vp->v_mount && vp->v_mount->mnt_wapbl && 1730 (ioflag & IO_JOURNALLOCKED) == 0); 1731 1732 #ifdef DIAGNOSTIC 1733 if ((ioflag & IO_JOURNALLOCKED) && vp->v_mount->mnt_wapbl) 1734 WAPBL_JLOCK_ASSERT(vp->v_mount); 1735 #endif 1736 1737 /* 1738 * We only support direct I/O to user space for now. 1739 */ 1740 1741 if (VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) { 1742 return; 1743 } 1744 1745 /* 1746 * If the vnode is mapped, we would need to get the getpages lock 1747 * to stabilize the bmap, but then we would get into trouble while 1748 * locking the pages if the pages belong to this same vnode (or a 1749 * multi-vnode cascade to the same effect). Just fall back to 1750 * buffered I/O if the vnode is mapped to avoid this mess. 1751 */ 1752 1753 if (vp->v_vflag & VV_MAPPED) { 1754 return; 1755 } 1756 1757 if (need_wapbl) { 1758 error = WAPBL_BEGIN(vp->v_mount); 1759 if (error) 1760 return; 1761 } 1762 1763 /* 1764 * Do as much of the uio as possible with direct I/O. 1765 */ 1766 1767 vs = uio->uio_vmspace; 1768 while (uio->uio_resid) { 1769 iov = uio->uio_iov; 1770 if (iov->iov_len == 0) { 1771 uio->uio_iov++; 1772 uio->uio_iovcnt--; 1773 continue; 1774 } 1775 va = (vaddr_t)iov->iov_base; 1776 len = MIN(iov->iov_len, genfs_maxdio); 1777 len &= ~mask; 1778 1779 /* 1780 * If the next chunk is smaller than DEV_BSIZE or extends past 1781 * the current EOF, then fall back to buffered I/O. 1782 */ 1783 1784 if (len == 0 || uio->uio_offset + len > vp->v_size) { 1785 break; 1786 } 1787 1788 /* 1789 * Check alignment. The file offset must be at least 1790 * sector-aligned. The exact constraint on memory alignment 1791 * is very hardware-dependent, but requiring sector-aligned 1792 * addresses there too is safe. 1793 */ 1794 1795 if (uio->uio_offset & mask || va & mask) { 1796 break; 1797 } 1798 error = genfs_do_directio(vs, va, len, vp, uio->uio_offset, 1799 uio->uio_rw); 1800 if (error) { 1801 break; 1802 } 1803 iov->iov_base = (char *)iov->iov_base + len; 1804 iov->iov_len -= len; 1805 uio->uio_offset += len; 1806 uio->uio_resid -= len; 1807 } 1808 1809 if (need_wapbl) 1810 WAPBL_END(vp->v_mount); 1811 } 1812 1813 /* 1814 * Iodone routine for direct I/O. We don't do much here since the request is 1815 * always synchronous, so the caller will do most of the work after biowait(). 1816 */ 1817 1818 static void 1819 genfs_dio_iodone(struct buf *bp) 1820 { 1821 1822 KASSERT((bp->b_flags & B_ASYNC) == 0); 1823 if ((bp->b_flags & B_READ) == 0 && (bp->b_cflags & BC_AGE) != 0) { 1824 mutex_enter(bp->b_objlock); 1825 vwakeup(bp); 1826 mutex_exit(bp->b_objlock); 1827 } 1828 putiobuf(bp); 1829 } 1830 1831 /* 1832 * Process one chunk of a direct I/O request. 1833 */ 1834 1835 static int 1836 genfs_do_directio(struct vmspace *vs, vaddr_t uva, size_t len, struct vnode *vp, 1837 off_t off, enum uio_rw rw) 1838 { 1839 struct vm_map *map; 1840 struct pmap *upm, *kpm __unused; 1841 size_t klen = round_page(uva + len) - trunc_page(uva); 1842 off_t spoff, epoff; 1843 vaddr_t kva, puva; 1844 paddr_t pa; 1845 vm_prot_t prot; 1846 int error, rv __diagused, poff, koff; 1847 const int pgoflags = PGO_CLEANIT | PGO_SYNCIO | PGO_JOURNALLOCKED | 1848 (rw == UIO_WRITE ? PGO_FREE : 0); 1849 1850 /* 1851 * For writes, verify that this range of the file already has fully 1852 * allocated backing store. If there are any holes, just punt and 1853 * make the caller take the buffered write path. 1854 */ 1855 1856 if (rw == UIO_WRITE) { 1857 daddr_t lbn, elbn, blkno; 1858 int bsize, bshift, run; 1859 1860 bshift = vp->v_mount->mnt_fs_bshift; 1861 bsize = 1 << bshift; 1862 lbn = off >> bshift; 1863 elbn = (off + len + bsize - 1) >> bshift; 1864 while (lbn < elbn) { 1865 error = VOP_BMAP(vp, lbn, NULL, &blkno, &run); 1866 if (error) { 1867 return error; 1868 } 1869 if (blkno == (daddr_t)-1) { 1870 return ENOSPC; 1871 } 1872 lbn += 1 + run; 1873 } 1874 } 1875 1876 /* 1877 * Flush any cached pages for parts of the file that we're about to 1878 * access. If we're writing, invalidate pages as well. 1879 */ 1880 1881 spoff = trunc_page(off); 1882 epoff = round_page(off + len); 1883 mutex_enter(vp->v_interlock); 1884 error = VOP_PUTPAGES(vp, spoff, epoff, pgoflags); 1885 if (error) { 1886 return error; 1887 } 1888 1889 /* 1890 * Wire the user pages and remap them into kernel memory. 1891 */ 1892 1893 prot = rw == UIO_READ ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_READ; 1894 error = uvm_vslock(vs, (void *)uva, len, prot); 1895 if (error) { 1896 return error; 1897 } 1898 1899 map = &vs->vm_map; 1900 upm = vm_map_pmap(map); 1901 kpm = vm_map_pmap(kernel_map); 1902 puva = trunc_page(uva); 1903 kva = uvm_km_alloc(kernel_map, klen, atop(puva) & uvmexp.colormask, 1904 UVM_KMF_VAONLY | UVM_KMF_WAITVA | UVM_KMF_COLORMATCH); 1905 for (poff = 0; poff < klen; poff += PAGE_SIZE) { 1906 rv = pmap_extract(upm, puva + poff, &pa); 1907 KASSERT(rv); 1908 pmap_kenter_pa(kva + poff, pa, prot, PMAP_WIRED); 1909 } 1910 pmap_update(kpm); 1911 1912 /* 1913 * Do the I/O. 1914 */ 1915 1916 koff = uva - trunc_page(uva); 1917 error = genfs_do_io(vp, off, kva + koff, len, PGO_SYNCIO, rw, 1918 genfs_dio_iodone); 1919 1920 /* 1921 * Tear down the kernel mapping. 1922 */ 1923 1924 pmap_kremove(kva, klen); 1925 pmap_update(kpm); 1926 uvm_km_free(kernel_map, kva, klen, UVM_KMF_VAONLY); 1927 1928 /* 1929 * Unwire the user pages. 1930 */ 1931 1932 uvm_vsunlock(vs, (void *)uva, len); 1933 return error; 1934 } 1935