1 /* $NetBSD: genfs_io.c,v 1.98 2020/06/14 00:25:22 ad Exp $ */ 2 3 /* 4 * Copyright (c) 1982, 1986, 1989, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the University nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 */ 32 33 #include <sys/cdefs.h> 34 __KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.98 2020/06/14 00:25:22 ad Exp $"); 35 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/proc.h> 39 #include <sys/kernel.h> 40 #include <sys/mount.h> 41 #include <sys/vnode.h> 42 #include <sys/kmem.h> 43 #include <sys/kauth.h> 44 #include <sys/fstrans.h> 45 #include <sys/buf.h> 46 #include <sys/atomic.h> 47 48 #include <miscfs/genfs/genfs.h> 49 #include <miscfs/genfs/genfs_node.h> 50 #include <miscfs/specfs/specdev.h> 51 52 #include <uvm/uvm.h> 53 #include <uvm/uvm_pager.h> 54 #include <uvm/uvm_page_array.h> 55 56 static int genfs_do_directio(struct vmspace *, vaddr_t, size_t, struct vnode *, 57 off_t, enum uio_rw); 58 static void genfs_dio_iodone(struct buf *); 59 60 static int genfs_getpages_read(struct vnode *, struct vm_page **, int, off_t, 61 off_t, bool, bool, bool, bool); 62 static int genfs_do_io(struct vnode *, off_t, vaddr_t, size_t, int, enum uio_rw, 63 void (*)(struct buf *)); 64 static void genfs_rel_pages(struct vm_page **, unsigned int); 65 66 int genfs_maxdio = MAXPHYS; 67 68 static void 69 genfs_rel_pages(struct vm_page **pgs, unsigned int npages) 70 { 71 unsigned int i; 72 73 for (i = 0; i < npages; i++) { 74 struct vm_page *pg = pgs[i]; 75 76 if (pg == NULL || pg == PGO_DONTCARE) 77 continue; 78 KASSERT(uvm_page_owner_locked_p(pg, true)); 79 if (pg->flags & PG_FAKE) { 80 pg->flags |= PG_RELEASED; 81 } 82 } 83 uvm_page_unbusy(pgs, npages); 84 } 85 86 /* 87 * generic VM getpages routine. 88 * Return PG_BUSY pages for the given range, 89 * reading from backing store if necessary. 90 */ 91 92 int 93 genfs_getpages(void *v) 94 { 95 struct vop_getpages_args /* { 96 struct vnode *a_vp; 97 voff_t a_offset; 98 struct vm_page **a_m; 99 int *a_count; 100 int a_centeridx; 101 vm_prot_t a_access_type; 102 int a_advice; 103 int a_flags; 104 } */ * const ap = v; 105 106 off_t diskeof, memeof; 107 int i, error, npages, iflag; 108 const int flags = ap->a_flags; 109 struct vnode * const vp = ap->a_vp; 110 struct uvm_object * const uobj = &vp->v_uobj; 111 const bool async = (flags & PGO_SYNCIO) == 0; 112 const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0; 113 const bool overwrite = (flags & PGO_OVERWRITE) != 0; 114 const bool blockalloc = memwrite && (flags & PGO_NOBLOCKALLOC) == 0; 115 const bool need_wapbl = (vp->v_mount->mnt_wapbl && 116 (flags & PGO_JOURNALLOCKED) == 0); 117 const bool glocked = (flags & PGO_GLOCKHELD) != 0; 118 bool holds_wapbl = false; 119 struct mount *trans_mount = NULL; 120 UVMHIST_FUNC("genfs_getpages"); UVMHIST_CALLED(ubchist); 121 122 UVMHIST_LOG(ubchist, "vp %#jx off 0x%jx/%jx count %jd", 123 (uintptr_t)vp, ap->a_offset >> 32, ap->a_offset, *ap->a_count); 124 125 KASSERT(memwrite >= overwrite); 126 KASSERT(vp->v_type == VREG || vp->v_type == VDIR || 127 vp->v_type == VLNK || vp->v_type == VBLK); 128 129 /* 130 * the object must be locked. it can only be a read lock when 131 * processing a read fault with PGO_LOCKED. 132 */ 133 134 KASSERT(rw_lock_held(uobj->vmobjlock)); 135 KASSERT(rw_write_held(uobj->vmobjlock) || 136 ((flags & PGO_LOCKED) != 0 && !memwrite)); 137 138 #ifdef DIAGNOSTIC 139 if ((flags & PGO_JOURNALLOCKED) && vp->v_mount->mnt_wapbl) 140 WAPBL_JLOCK_ASSERT(vp->v_mount); 141 #endif 142 143 /* 144 * check for reclaimed vnode. v_interlock is not held here, but 145 * VI_DEADCHECK is set with vmobjlock held. 146 */ 147 148 iflag = atomic_load_relaxed(&vp->v_iflag); 149 if (__predict_false((iflag & VI_DEADCHECK) != 0)) { 150 mutex_enter(vp->v_interlock); 151 error = vdead_check(vp, VDEAD_NOWAIT); 152 mutex_exit(vp->v_interlock); 153 if (error) { 154 if ((flags & PGO_LOCKED) == 0) 155 rw_exit(uobj->vmobjlock); 156 return error; 157 } 158 } 159 160 startover: 161 error = 0; 162 const voff_t origvsize = vp->v_size; 163 const off_t origoffset = ap->a_offset; 164 const int orignpages = *ap->a_count; 165 166 GOP_SIZE(vp, origvsize, &diskeof, 0); 167 if (flags & PGO_PASTEOF) { 168 off_t newsize; 169 #if defined(DIAGNOSTIC) 170 off_t writeeof; 171 #endif /* defined(DIAGNOSTIC) */ 172 173 newsize = MAX(origvsize, 174 origoffset + (orignpages << PAGE_SHIFT)); 175 GOP_SIZE(vp, newsize, &memeof, GOP_SIZE_MEM); 176 #if defined(DIAGNOSTIC) 177 GOP_SIZE(vp, vp->v_writesize, &writeeof, GOP_SIZE_MEM); 178 if (newsize > round_page(writeeof)) { 179 panic("%s: past eof: %" PRId64 " vs. %" PRId64, 180 __func__, newsize, round_page(writeeof)); 181 } 182 #endif /* defined(DIAGNOSTIC) */ 183 } else { 184 GOP_SIZE(vp, origvsize, &memeof, GOP_SIZE_MEM); 185 } 186 KASSERT(ap->a_centeridx >= 0 || ap->a_centeridx <= orignpages); 187 KASSERT((origoffset & (PAGE_SIZE - 1)) == 0 && origoffset >= 0); 188 KASSERT(orignpages > 0); 189 190 /* 191 * Bounds-check the request. 192 */ 193 194 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= memeof) { 195 if ((flags & PGO_LOCKED) == 0) { 196 rw_exit(uobj->vmobjlock); 197 } 198 UVMHIST_LOG(ubchist, "off 0x%jx count %jd goes past EOF 0x%jx", 199 origoffset, *ap->a_count, memeof,0); 200 error = EINVAL; 201 goto out_err; 202 } 203 204 /* uobj is locked */ 205 206 if ((flags & PGO_NOTIMESTAMP) == 0 && 207 (vp->v_type != VBLK || 208 (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) { 209 int updflags = 0; 210 211 if ((vp->v_mount->mnt_flag & MNT_NOATIME) == 0) { 212 updflags = GOP_UPDATE_ACCESSED; 213 } 214 if (memwrite) { 215 updflags |= GOP_UPDATE_MODIFIED; 216 } 217 if (updflags != 0) { 218 GOP_MARKUPDATE(vp, updflags); 219 } 220 } 221 222 /* 223 * For PGO_LOCKED requests, just return whatever's in memory. 224 */ 225 226 if (flags & PGO_LOCKED) { 227 int nfound; 228 struct vm_page *pg; 229 230 KASSERT(!glocked); 231 npages = *ap->a_count; 232 #if defined(DEBUG) 233 for (i = 0; i < npages; i++) { 234 pg = ap->a_m[i]; 235 KASSERT(pg == NULL || pg == PGO_DONTCARE); 236 } 237 #endif /* defined(DEBUG) */ 238 nfound = uvn_findpages(uobj, origoffset, &npages, 239 ap->a_m, NULL, 240 UFP_NOWAIT | UFP_NOALLOC | UFP_NOBUSY | 241 (memwrite ? UFP_NORDONLY : 0)); 242 KASSERT(npages == *ap->a_count); 243 if (nfound == 0) { 244 error = EBUSY; 245 goto out_err; 246 } 247 /* 248 * lock and unlock g_glock to ensure that no one is truncating 249 * the file behind us. 250 */ 251 if (!genfs_node_rdtrylock(vp)) { 252 /* 253 * restore the array. 254 */ 255 256 for (i = 0; i < npages; i++) { 257 pg = ap->a_m[i]; 258 259 if (pg != NULL && pg != PGO_DONTCARE) { 260 ap->a_m[i] = NULL; 261 } 262 KASSERT(ap->a_m[i] == NULL || 263 ap->a_m[i] == PGO_DONTCARE); 264 } 265 } else { 266 genfs_node_unlock(vp); 267 } 268 error = (ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0); 269 if (error == 0 && memwrite) { 270 for (i = 0; i < npages; i++) { 271 pg = ap->a_m[i]; 272 if (pg == NULL || pg == PGO_DONTCARE) { 273 continue; 274 } 275 if (uvm_pagegetdirty(pg) == 276 UVM_PAGE_STATUS_CLEAN) { 277 uvm_pagemarkdirty(pg, 278 UVM_PAGE_STATUS_UNKNOWN); 279 } 280 } 281 } 282 goto out_err; 283 } 284 rw_exit(uobj->vmobjlock); 285 286 /* 287 * find the requested pages and make some simple checks. 288 * leave space in the page array for a whole block. 289 */ 290 291 const int fs_bshift = (vp->v_type != VBLK) ? 292 vp->v_mount->mnt_fs_bshift : DEV_BSHIFT; 293 const int fs_bsize = 1 << fs_bshift; 294 #define blk_mask (fs_bsize - 1) 295 #define trunc_blk(x) ((x) & ~blk_mask) 296 #define round_blk(x) (((x) + blk_mask) & ~blk_mask) 297 298 const int orignmempages = MIN(orignpages, 299 round_page(memeof - origoffset) >> PAGE_SHIFT); 300 npages = orignmempages; 301 const off_t startoffset = trunc_blk(origoffset); 302 const off_t endoffset = MIN( 303 round_page(round_blk(origoffset + (npages << PAGE_SHIFT))), 304 round_page(memeof)); 305 const int ridx = (origoffset - startoffset) >> PAGE_SHIFT; 306 307 const int pgs_size = sizeof(struct vm_page *) * 308 ((endoffset - startoffset) >> PAGE_SHIFT); 309 struct vm_page **pgs, *pgs_onstack[UBC_MAX_PAGES]; 310 311 if (pgs_size > sizeof(pgs_onstack)) { 312 pgs = kmem_zalloc(pgs_size, async ? KM_NOSLEEP : KM_SLEEP); 313 if (pgs == NULL) { 314 pgs = pgs_onstack; 315 error = ENOMEM; 316 goto out_err; 317 } 318 } else { 319 pgs = pgs_onstack; 320 (void)memset(pgs, 0, pgs_size); 321 } 322 323 UVMHIST_LOG(ubchist, "ridx %jd npages %jd startoff %jd endoff %jd", 324 ridx, npages, startoffset, endoffset); 325 326 if (trans_mount == NULL) { 327 trans_mount = vp->v_mount; 328 fstrans_start(trans_mount); 329 /* 330 * check if this vnode is still valid. 331 */ 332 mutex_enter(vp->v_interlock); 333 error = vdead_check(vp, 0); 334 mutex_exit(vp->v_interlock); 335 if (error) 336 goto out_err_free; 337 /* 338 * XXX: This assumes that we come here only via 339 * the mmio path 340 */ 341 if (blockalloc && need_wapbl) { 342 error = WAPBL_BEGIN(trans_mount); 343 if (error) 344 goto out_err_free; 345 holds_wapbl = true; 346 } 347 } 348 349 /* 350 * hold g_glock to prevent a race with truncate. 351 * 352 * check if our idea of v_size is still valid. 353 */ 354 355 KASSERT(!glocked || genfs_node_wrlocked(vp)); 356 if (!glocked) { 357 if (blockalloc) { 358 genfs_node_wrlock(vp); 359 } else { 360 genfs_node_rdlock(vp); 361 } 362 } 363 rw_enter(uobj->vmobjlock, RW_WRITER); 364 if (vp->v_size < origvsize) { 365 if (!glocked) { 366 genfs_node_unlock(vp); 367 } 368 if (pgs != pgs_onstack) 369 kmem_free(pgs, pgs_size); 370 goto startover; 371 } 372 373 if (uvn_findpages(uobj, origoffset, &npages, &pgs[ridx], NULL, 374 async ? UFP_NOWAIT : UFP_ALL) != orignmempages) { 375 if (!glocked) { 376 genfs_node_unlock(vp); 377 } 378 KASSERT(async != 0); 379 genfs_rel_pages(&pgs[ridx], orignmempages); 380 rw_exit(uobj->vmobjlock); 381 error = EBUSY; 382 goto out_err_free; 383 } 384 385 /* 386 * if PGO_OVERWRITE is set, don't bother reading the pages. 387 */ 388 389 if (overwrite) { 390 if (!glocked) { 391 genfs_node_unlock(vp); 392 } 393 UVMHIST_LOG(ubchist, "PGO_OVERWRITE",0,0,0,0); 394 395 for (i = 0; i < npages; i++) { 396 struct vm_page *pg = pgs[ridx + i]; 397 398 /* 399 * it's caller's responsibility to allocate blocks 400 * beforehand for the overwrite case. 401 */ 402 403 KASSERT((pg->flags & PG_RDONLY) == 0 || !blockalloc); 404 pg->flags &= ~PG_RDONLY; 405 406 /* 407 * mark the page DIRTY. 408 * otherwise another thread can do putpages and pull 409 * our vnode from syncer's queue before our caller does 410 * ubc_release. note that putpages won't see CLEAN 411 * pages even if they are BUSY. 412 */ 413 414 uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY); 415 } 416 npages += ridx; 417 goto out; 418 } 419 420 /* 421 * if the pages are already resident, just return them. 422 */ 423 424 for (i = 0; i < npages; i++) { 425 struct vm_page *pg = pgs[ridx + i]; 426 427 if ((pg->flags & PG_FAKE) || 428 (blockalloc && (pg->flags & PG_RDONLY) != 0)) { 429 break; 430 } 431 } 432 if (i == npages) { 433 if (!glocked) { 434 genfs_node_unlock(vp); 435 } 436 UVMHIST_LOG(ubchist, "returning cached pages", 0,0,0,0); 437 npages += ridx; 438 goto out; 439 } 440 441 /* 442 * the page wasn't resident and we're not overwriting, 443 * so we're going to have to do some i/o. 444 * find any additional pages needed to cover the expanded range. 445 */ 446 447 npages = (endoffset - startoffset) >> PAGE_SHIFT; 448 if (startoffset != origoffset || npages != orignmempages) { 449 int npgs; 450 451 /* 452 * we need to avoid deadlocks caused by locking 453 * additional pages at lower offsets than pages we 454 * already have locked. unlock them all and start over. 455 */ 456 457 genfs_rel_pages(&pgs[ridx], orignmempages); 458 memset(pgs, 0, pgs_size); 459 460 UVMHIST_LOG(ubchist, "reset npages start 0x%jx end 0x%jx", 461 startoffset, endoffset, 0,0); 462 npgs = npages; 463 if (uvn_findpages(uobj, startoffset, &npgs, pgs, NULL, 464 async ? UFP_NOWAIT : UFP_ALL) != npages) { 465 if (!glocked) { 466 genfs_node_unlock(vp); 467 } 468 KASSERT(async != 0); 469 genfs_rel_pages(pgs, npages); 470 rw_exit(uobj->vmobjlock); 471 error = EBUSY; 472 goto out_err_free; 473 } 474 } 475 476 rw_exit(uobj->vmobjlock); 477 error = genfs_getpages_read(vp, pgs, npages, startoffset, diskeof, 478 async, memwrite, blockalloc, glocked); 479 if (!glocked) { 480 genfs_node_unlock(vp); 481 } 482 if (error == 0 && async) 483 goto out_err_free; 484 rw_enter(uobj->vmobjlock, RW_WRITER); 485 486 /* 487 * we're almost done! release the pages... 488 * for errors, we free the pages. 489 * otherwise we activate them and mark them as valid and clean. 490 * also, unbusy pages that were not actually requested. 491 */ 492 493 if (error) { 494 genfs_rel_pages(pgs, npages); 495 rw_exit(uobj->vmobjlock); 496 UVMHIST_LOG(ubchist, "returning error %jd", error,0,0,0); 497 goto out_err_free; 498 } 499 500 out: 501 UVMHIST_LOG(ubchist, "succeeding, npages %jd", npages,0,0,0); 502 error = 0; 503 for (i = 0; i < npages; i++) { 504 struct vm_page *pg = pgs[i]; 505 if (pg == NULL) { 506 continue; 507 } 508 UVMHIST_LOG(ubchist, "examining pg %#jx flags 0x%jx", 509 (uintptr_t)pg, pg->flags, 0,0); 510 if (pg->flags & PG_FAKE && !overwrite) { 511 /* 512 * we've read page's contents from the backing storage. 513 * 514 * for a read fault, we keep them CLEAN; if we 515 * encountered a hole while reading, the pages can 516 * already been dirtied with zeros. 517 */ 518 KASSERTMSG(blockalloc || uvm_pagegetdirty(pg) == 519 UVM_PAGE_STATUS_CLEAN, "page %p not clean", pg); 520 pg->flags &= ~PG_FAKE; 521 } 522 KASSERT(!memwrite || !blockalloc || (pg->flags & PG_RDONLY) == 0); 523 if (i < ridx || i >= ridx + orignmempages || async) { 524 UVMHIST_LOG(ubchist, "unbusy pg %#jx offset 0x%jx", 525 (uintptr_t)pg, pg->offset,0,0); 526 if (pg->flags & PG_FAKE) { 527 KASSERT(overwrite); 528 uvm_pagezero(pg); 529 } 530 if (pg->flags & PG_RELEASED) { 531 uvm_pagefree(pg); 532 continue; 533 } 534 uvm_pagelock(pg); 535 uvm_pageenqueue(pg); 536 uvm_pagewakeup(pg); 537 uvm_pageunlock(pg); 538 pg->flags &= ~(PG_BUSY|PG_FAKE); 539 UVM_PAGE_OWN(pg, NULL); 540 } else if (memwrite && !overwrite && 541 uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_CLEAN) { 542 /* 543 * for a write fault, start dirtiness tracking of 544 * requested pages. 545 */ 546 uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_UNKNOWN); 547 } 548 } 549 rw_exit(uobj->vmobjlock); 550 if (ap->a_m != NULL) { 551 memcpy(ap->a_m, &pgs[ridx], 552 orignmempages * sizeof(struct vm_page *)); 553 } 554 555 out_err_free: 556 if (pgs != NULL && pgs != pgs_onstack) 557 kmem_free(pgs, pgs_size); 558 out_err: 559 if (trans_mount != NULL) { 560 if (holds_wapbl) 561 WAPBL_END(trans_mount); 562 fstrans_done(trans_mount); 563 } 564 return error; 565 } 566 567 /* 568 * genfs_getpages_read: Read the pages in with VOP_BMAP/VOP_STRATEGY. 569 * 570 * "glocked" (which is currently not actually used) tells us not whether 571 * the genfs_node is locked on entry (it always is) but whether it was 572 * locked on entry to genfs_getpages. 573 */ 574 static int 575 genfs_getpages_read(struct vnode *vp, struct vm_page **pgs, int npages, 576 off_t startoffset, off_t diskeof, 577 bool async, bool memwrite, bool blockalloc, bool glocked) 578 { 579 struct uvm_object * const uobj = &vp->v_uobj; 580 const int fs_bshift = (vp->v_type != VBLK) ? 581 vp->v_mount->mnt_fs_bshift : DEV_BSHIFT; 582 const int dev_bshift = (vp->v_type != VBLK) ? 583 vp->v_mount->mnt_dev_bshift : DEV_BSHIFT; 584 kauth_cred_t const cred = curlwp->l_cred; /* XXXUBC curlwp */ 585 size_t bytes, iobytes, tailstart, tailbytes, totalbytes, skipbytes; 586 vaddr_t kva; 587 struct buf *bp, *mbp; 588 bool sawhole = false; 589 int i; 590 int error = 0; 591 592 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 593 594 /* 595 * read the desired page(s). 596 */ 597 598 totalbytes = npages << PAGE_SHIFT; 599 bytes = MIN(totalbytes, MAX(diskeof - startoffset, 0)); 600 tailbytes = totalbytes - bytes; 601 skipbytes = 0; 602 603 kva = uvm_pagermapin(pgs, npages, 604 UVMPAGER_MAPIN_READ | (async ? 0 : UVMPAGER_MAPIN_WAITOK)); 605 if (kva == 0) 606 return EBUSY; 607 608 mbp = getiobuf(vp, true); 609 mbp->b_bufsize = totalbytes; 610 mbp->b_data = (void *)kva; 611 mbp->b_resid = mbp->b_bcount = bytes; 612 mbp->b_cflags |= BC_BUSY; 613 if (async) { 614 mbp->b_flags = B_READ | B_ASYNC; 615 mbp->b_iodone = uvm_aio_aiodone; 616 } else { 617 mbp->b_flags = B_READ; 618 mbp->b_iodone = NULL; 619 } 620 if (async) 621 BIO_SETPRIO(mbp, BPRIO_TIMELIMITED); 622 else 623 BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL); 624 625 /* 626 * if EOF is in the middle of the range, zero the part past EOF. 627 * skip over pages which are not PG_FAKE since in that case they have 628 * valid data that we need to preserve. 629 */ 630 631 tailstart = bytes; 632 while (tailbytes > 0) { 633 const int len = PAGE_SIZE - (tailstart & PAGE_MASK); 634 635 KASSERT(len <= tailbytes); 636 if ((pgs[tailstart >> PAGE_SHIFT]->flags & PG_FAKE) != 0) { 637 memset((void *)(kva + tailstart), 0, len); 638 UVMHIST_LOG(ubchist, "tailbytes %#jx 0x%jx 0x%jx", 639 (uintptr_t)kva, tailstart, len, 0); 640 } 641 tailstart += len; 642 tailbytes -= len; 643 } 644 645 /* 646 * now loop over the pages, reading as needed. 647 */ 648 649 bp = NULL; 650 off_t offset; 651 for (offset = startoffset; 652 bytes > 0; 653 offset += iobytes, bytes -= iobytes) { 654 int run; 655 daddr_t lbn, blkno; 656 int pidx; 657 struct vnode *devvp; 658 659 /* 660 * skip pages which don't need to be read. 661 */ 662 663 pidx = (offset - startoffset) >> PAGE_SHIFT; 664 while ((pgs[pidx]->flags & PG_FAKE) == 0) { 665 size_t b; 666 667 KASSERT((offset & (PAGE_SIZE - 1)) == 0); 668 if ((pgs[pidx]->flags & PG_RDONLY)) { 669 sawhole = true; 670 } 671 b = MIN(PAGE_SIZE, bytes); 672 offset += b; 673 bytes -= b; 674 skipbytes += b; 675 pidx++; 676 UVMHIST_LOG(ubchist, "skipping, new offset 0x%jx", 677 offset, 0,0,0); 678 if (bytes == 0) { 679 goto loopdone; 680 } 681 } 682 683 /* 684 * bmap the file to find out the blkno to read from and 685 * how much we can read in one i/o. if bmap returns an error, 686 * skip the rest of the top-level i/o. 687 */ 688 689 lbn = offset >> fs_bshift; 690 error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run); 691 if (error) { 692 UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%jx -> %jd\n", 693 lbn,error,0,0); 694 skipbytes += bytes; 695 bytes = 0; 696 goto loopdone; 697 } 698 699 /* 700 * see how many pages can be read with this i/o. 701 * reduce the i/o size if necessary to avoid 702 * overwriting pages with valid data. 703 */ 704 705 iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset, 706 bytes); 707 if (offset + iobytes > round_page(offset)) { 708 int pcount; 709 710 pcount = 1; 711 while (pidx + pcount < npages && 712 pgs[pidx + pcount]->flags & PG_FAKE) { 713 pcount++; 714 } 715 iobytes = MIN(iobytes, (pcount << PAGE_SHIFT) - 716 (offset - trunc_page(offset))); 717 } 718 719 /* 720 * if this block isn't allocated, zero it instead of 721 * reading it. unless we are going to allocate blocks, 722 * mark the pages we zeroed PG_RDONLY. 723 */ 724 725 if (blkno == (daddr_t)-1) { 726 int holepages = (round_page(offset + iobytes) - 727 trunc_page(offset)) >> PAGE_SHIFT; 728 UVMHIST_LOG(ubchist, "lbn 0x%jx -> HOLE", lbn,0,0,0); 729 730 sawhole = true; 731 memset((char *)kva + (offset - startoffset), 0, 732 iobytes); 733 skipbytes += iobytes; 734 735 if (!blockalloc) { 736 rw_enter(uobj->vmobjlock, RW_WRITER); 737 for (i = 0; i < holepages; i++) { 738 pgs[pidx + i]->flags |= PG_RDONLY; 739 } 740 rw_exit(uobj->vmobjlock); 741 } 742 continue; 743 } 744 745 /* 746 * allocate a sub-buf for this piece of the i/o 747 * (or just use mbp if there's only 1 piece), 748 * and start it going. 749 */ 750 751 if (offset == startoffset && iobytes == bytes) { 752 bp = mbp; 753 } else { 754 UVMHIST_LOG(ubchist, "vp %#jx bp %#jx num now %jd", 755 (uintptr_t)vp, (uintptr_t)bp, vp->v_numoutput, 0); 756 bp = getiobuf(vp, true); 757 nestiobuf_setup(mbp, bp, offset - startoffset, iobytes); 758 } 759 bp->b_lblkno = 0; 760 761 /* adjust physical blkno for partial blocks */ 762 bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >> 763 dev_bshift); 764 765 UVMHIST_LOG(ubchist, 766 "bp %#jx offset 0x%x bcount 0x%x blkno 0x%x", 767 (uintptr_t)bp, offset, bp->b_bcount, bp->b_blkno); 768 769 VOP_STRATEGY(devvp, bp); 770 } 771 772 loopdone: 773 nestiobuf_done(mbp, skipbytes, error); 774 if (async) { 775 UVMHIST_LOG(ubchist, "returning 0 (async)",0,0,0,0); 776 return 0; 777 } 778 if (bp != NULL) { 779 error = biowait(mbp); 780 } 781 782 /* Remove the mapping (make KVA available as soon as possible) */ 783 uvm_pagermapout(kva, npages); 784 785 /* 786 * if this we encountered a hole then we have to do a little more work. 787 * for read faults, we marked the page PG_RDONLY so that future 788 * write accesses to the page will fault again. 789 * for write faults, we must make sure that the backing store for 790 * the page is completely allocated while the pages are locked. 791 */ 792 793 if (!error && sawhole && blockalloc) { 794 error = GOP_ALLOC(vp, startoffset, 795 npages << PAGE_SHIFT, 0, cred); 796 UVMHIST_LOG(ubchist, "gop_alloc off 0x%jx/0x%jx -> %jd", 797 startoffset, npages << PAGE_SHIFT, error,0); 798 if (!error) { 799 rw_enter(uobj->vmobjlock, RW_WRITER); 800 for (i = 0; i < npages; i++) { 801 struct vm_page *pg = pgs[i]; 802 803 if (pg == NULL) { 804 continue; 805 } 806 pg->flags &= ~PG_RDONLY; 807 uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY); 808 UVMHIST_LOG(ubchist, "mark dirty pg %#jx", 809 (uintptr_t)pg, 0, 0, 0); 810 } 811 rw_exit(uobj->vmobjlock); 812 } 813 } 814 815 putiobuf(mbp); 816 return error; 817 } 818 819 /* 820 * generic VM putpages routine. 821 * Write the given range of pages to backing store. 822 * 823 * => "offhi == 0" means flush all pages at or after "offlo". 824 * => object should be locked by caller. we return with the 825 * object unlocked. 826 * => if PGO_CLEANIT or PGO_SYNCIO is set, we may block (due to I/O). 827 * thus, a caller might want to unlock higher level resources 828 * (e.g. vm_map) before calling flush. 829 * => if neither PGO_CLEANIT nor PGO_SYNCIO is set, we will not block 830 * => if PGO_ALLPAGES is set, then all pages in the object will be processed. 831 * 832 * note on "cleaning" object and PG_BUSY pages: 833 * this routine is holding the lock on the object. the only time 834 * that it can run into a PG_BUSY page that it does not own is if 835 * some other process has started I/O on the page (e.g. either 836 * a pagein, or a pageout). if the PG_BUSY page is being paged 837 * in, then it can not be dirty (!UVM_PAGE_STATUS_CLEAN) because no 838 * one has had a chance to modify it yet. if the PG_BUSY page is 839 * being paged out then it means that someone else has already started 840 * cleaning the page for us (how nice!). in this case, if we 841 * have syncio specified, then after we make our pass through the 842 * object we need to wait for the other PG_BUSY pages to clear 843 * off (i.e. we need to do an iosync). also note that once a 844 * page is PG_BUSY it must stay in its object until it is un-busyed. 845 */ 846 847 int 848 genfs_putpages(void *v) 849 { 850 struct vop_putpages_args /* { 851 struct vnode *a_vp; 852 voff_t a_offlo; 853 voff_t a_offhi; 854 int a_flags; 855 } */ * const ap = v; 856 857 return genfs_do_putpages(ap->a_vp, ap->a_offlo, ap->a_offhi, 858 ap->a_flags, NULL); 859 } 860 861 int 862 genfs_do_putpages(struct vnode *vp, off_t startoff, off_t endoff, 863 int origflags, struct vm_page **busypg) 864 { 865 struct uvm_object * const uobj = &vp->v_uobj; 866 krwlock_t * const slock = uobj->vmobjlock; 867 off_t nextoff; 868 int i, error, npages, nback; 869 int freeflag; 870 /* 871 * This array is larger than it should so that it's size is constant. 872 * The right size is MAXPAGES. 873 */ 874 struct vm_page *pgs[MAXPHYS / MIN_PAGE_SIZE]; 875 #define MAXPAGES (MAXPHYS / PAGE_SIZE) 876 struct vm_page *pg, *tpg; 877 struct uvm_page_array a; 878 bool wasclean, needs_clean; 879 bool async = (origflags & PGO_SYNCIO) == 0; 880 bool pagedaemon = curlwp == uvm.pagedaemon_lwp; 881 struct mount *trans_mp; 882 int flags; 883 bool modified; /* if we write out any pages */ 884 bool holds_wapbl; 885 bool cleanall; /* try to pull off from the syncer's list */ 886 bool onworklst; 887 bool nodirty; 888 const bool dirtyonly = (origflags & (PGO_DEACTIVATE|PGO_FREE)) == 0; 889 890 UVMHIST_FUNC("genfs_putpages"); UVMHIST_CALLED(ubchist); 891 892 KASSERT(origflags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)); 893 KASSERT((startoff & PAGE_MASK) == 0 && (endoff & PAGE_MASK) == 0); 894 KASSERT(startoff < endoff || endoff == 0); 895 KASSERT(rw_write_held(slock)); 896 897 UVMHIST_LOG(ubchist, "vp %#jx pages %jd off 0x%jx len 0x%jx", 898 (uintptr_t)vp, uobj->uo_npages, startoff, endoff - startoff); 899 900 #ifdef DIAGNOSTIC 901 if ((origflags & PGO_JOURNALLOCKED) && vp->v_mount->mnt_wapbl) 902 WAPBL_JLOCK_ASSERT(vp->v_mount); 903 #endif 904 905 trans_mp = NULL; 906 holds_wapbl = false; 907 908 retry: 909 modified = false; 910 flags = origflags; 911 912 /* 913 * shortcut if we have no pages to process. 914 */ 915 916 nodirty = radix_tree_empty_tagged_tree_p(&uobj->uo_pages, 917 UVM_PAGE_DIRTY_TAG); 918 #ifdef DIAGNOSTIC 919 mutex_enter(vp->v_interlock); 920 KASSERT((vp->v_iflag & VI_ONWORKLST) != 0 || nodirty); 921 mutex_exit(vp->v_interlock); 922 #endif 923 if (uobj->uo_npages == 0 || (dirtyonly && nodirty)) { 924 mutex_enter(vp->v_interlock); 925 if (vp->v_iflag & VI_ONWORKLST) { 926 if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL) 927 vn_syncer_remove_from_worklist(vp); 928 } 929 mutex_exit(vp->v_interlock); 930 if (trans_mp) { 931 if (holds_wapbl) 932 WAPBL_END(trans_mp); 933 fstrans_done(trans_mp); 934 } 935 rw_exit(slock); 936 return (0); 937 } 938 939 /* 940 * the vnode has pages, set up to process the request. 941 */ 942 943 if (trans_mp == NULL && (flags & PGO_CLEANIT) != 0) { 944 if (pagedaemon) { 945 /* Pagedaemon must not sleep here. */ 946 trans_mp = vp->v_mount; 947 error = fstrans_start_nowait(trans_mp); 948 if (error) { 949 rw_exit(slock); 950 return error; 951 } 952 } else { 953 /* 954 * Cannot use vdeadcheck() here as this operation 955 * usually gets used from VOP_RECLAIM(). Test for 956 * change of v_mount instead and retry on change. 957 */ 958 rw_exit(slock); 959 trans_mp = vp->v_mount; 960 fstrans_start(trans_mp); 961 if (vp->v_mount != trans_mp) { 962 fstrans_done(trans_mp); 963 trans_mp = NULL; 964 } else { 965 holds_wapbl = (trans_mp->mnt_wapbl && 966 (origflags & PGO_JOURNALLOCKED) == 0); 967 if (holds_wapbl) { 968 error = WAPBL_BEGIN(trans_mp); 969 if (error) { 970 fstrans_done(trans_mp); 971 return error; 972 } 973 } 974 } 975 rw_enter(slock, RW_WRITER); 976 goto retry; 977 } 978 } 979 980 error = 0; 981 wasclean = radix_tree_empty_tagged_tree_p(&uobj->uo_pages, 982 UVM_PAGE_WRITEBACK_TAG); 983 nextoff = startoff; 984 if (endoff == 0 || flags & PGO_ALLPAGES) { 985 endoff = trunc_page(LLONG_MAX); 986 } 987 988 /* 989 * if this vnode is known not to have dirty pages, 990 * don't bother to clean it out. 991 */ 992 993 if (nodirty) { 994 #if !defined(DEBUG) 995 if (dirtyonly) { 996 goto skip_scan; 997 } 998 #endif /* !defined(DEBUG) */ 999 flags &= ~PGO_CLEANIT; 1000 } 1001 1002 /* 1003 * start the loop to scan pages. 1004 */ 1005 1006 cleanall = true; 1007 freeflag = pagedaemon ? PG_PAGEOUT : PG_RELEASED; 1008 uvm_page_array_init(&a, uobj, dirtyonly ? (UVM_PAGE_ARRAY_FILL_DIRTY | 1009 (!async ? UVM_PAGE_ARRAY_FILL_WRITEBACK : 0)) : 0); 1010 for (;;) { 1011 bool pgprotected; 1012 1013 /* 1014 * if !dirtyonly, iterate over all resident pages in the range. 1015 * 1016 * if dirtyonly, only possibly dirty pages are interesting. 1017 * however, if we are asked to sync for integrity, we should 1018 * wait on pages being written back by other threads as well. 1019 */ 1020 1021 pg = uvm_page_array_fill_and_peek(&a, nextoff, 0); 1022 if (pg == NULL) { 1023 break; 1024 } 1025 1026 KASSERT(pg->uobject == uobj); 1027 KASSERT((pg->flags & (PG_RELEASED|PG_PAGEOUT)) == 0 || 1028 (pg->flags & (PG_BUSY)) != 0); 1029 KASSERT(pg->offset >= startoff); 1030 KASSERT(pg->offset >= nextoff); 1031 KASSERT(!dirtyonly || 1032 uvm_pagegetdirty(pg) != UVM_PAGE_STATUS_CLEAN || 1033 radix_tree_get_tag(&uobj->uo_pages, 1034 pg->offset >> PAGE_SHIFT, UVM_PAGE_WRITEBACK_TAG)); 1035 1036 if (pg->offset >= endoff) { 1037 break; 1038 } 1039 1040 /* 1041 * a preempt point. 1042 */ 1043 1044 if (preempt_needed()) { 1045 nextoff = pg->offset; /* visit this page again */ 1046 rw_exit(slock); 1047 preempt(); 1048 /* 1049 * as we dropped the object lock, our cached pages can 1050 * be stale. 1051 */ 1052 uvm_page_array_clear(&a); 1053 rw_enter(slock, RW_WRITER); 1054 continue; 1055 } 1056 1057 /* 1058 * if the current page is busy, wait for it to become unbusy. 1059 */ 1060 1061 if ((pg->flags & PG_BUSY) != 0) { 1062 UVMHIST_LOG(ubchist, "busy %#jx", (uintptr_t)pg, 1063 0, 0, 0); 1064 if ((pg->flags & (PG_RELEASED|PG_PAGEOUT)) != 0 1065 && (flags & PGO_BUSYFAIL) != 0) { 1066 UVMHIST_LOG(ubchist, "busyfail %#jx", 1067 (uintptr_t)pg, 0, 0, 0); 1068 error = EDEADLK; 1069 if (busypg != NULL) 1070 *busypg = pg; 1071 break; 1072 } 1073 if (pagedaemon) { 1074 /* 1075 * someone has taken the page while we 1076 * dropped the lock for fstrans_start. 1077 */ 1078 break; 1079 } 1080 /* 1081 * don't bother to wait on other's activities 1082 * unless we are asked to sync for integrity. 1083 */ 1084 if (!async && (flags & PGO_RECLAIM) == 0) { 1085 wasclean = false; 1086 nextoff = pg->offset + PAGE_SIZE; 1087 uvm_page_array_advance(&a); 1088 continue; 1089 } 1090 nextoff = pg->offset; /* visit this page again */ 1091 uvm_pagewait(pg, slock, "genput"); 1092 /* 1093 * as we dropped the object lock, our cached pages can 1094 * be stale. 1095 */ 1096 uvm_page_array_clear(&a); 1097 rw_enter(slock, RW_WRITER); 1098 continue; 1099 } 1100 1101 nextoff = pg->offset + PAGE_SIZE; 1102 uvm_page_array_advance(&a); 1103 1104 /* 1105 * if we're freeing, remove all mappings of the page now. 1106 * if we're cleaning, check if the page is needs to be cleaned. 1107 */ 1108 1109 pgprotected = false; 1110 if (flags & PGO_FREE) { 1111 pmap_page_protect(pg, VM_PROT_NONE); 1112 pgprotected = true; 1113 } else if (flags & PGO_CLEANIT) { 1114 1115 /* 1116 * if we still have some hope to pull this vnode off 1117 * from the syncer queue, write-protect the page. 1118 */ 1119 1120 if (cleanall && wasclean) { 1121 1122 /* 1123 * uobj pages get wired only by uvm_fault 1124 * where uobj is locked. 1125 */ 1126 1127 if (pg->wire_count == 0) { 1128 pmap_page_protect(pg, 1129 VM_PROT_READ|VM_PROT_EXECUTE); 1130 pgprotected = true; 1131 } else { 1132 cleanall = false; 1133 } 1134 } 1135 } 1136 1137 if (flags & PGO_CLEANIT) { 1138 needs_clean = uvm_pagecheckdirty(pg, pgprotected); 1139 } else { 1140 needs_clean = false; 1141 } 1142 1143 /* 1144 * if we're cleaning, build a cluster. 1145 * the cluster will consist of pages which are currently dirty. 1146 * if not cleaning, just operate on the one page. 1147 */ 1148 1149 if (needs_clean) { 1150 wasclean = false; 1151 memset(pgs, 0, sizeof(pgs)); 1152 pg->flags |= PG_BUSY; 1153 UVM_PAGE_OWN(pg, "genfs_putpages"); 1154 1155 /* 1156 * let the fs constrain the offset range of the cluster. 1157 * we additionally constrain the range here such that 1158 * it fits in the "pgs" pages array. 1159 */ 1160 1161 off_t fslo, fshi, genlo, lo, off = pg->offset; 1162 GOP_PUTRANGE(vp, off, &fslo, &fshi); 1163 KASSERT(fslo == trunc_page(fslo)); 1164 KASSERT(fslo <= off); 1165 KASSERT(fshi == trunc_page(fshi)); 1166 KASSERT(fshi == 0 || off < fshi); 1167 1168 if (off > MAXPHYS / 2) 1169 genlo = trunc_page(off - (MAXPHYS / 2)); 1170 else 1171 genlo = 0; 1172 lo = MAX(fslo, genlo); 1173 1174 /* 1175 * first look backward. 1176 */ 1177 1178 npages = (off - lo) >> PAGE_SHIFT; 1179 nback = npages; 1180 uvn_findpages(uobj, off - PAGE_SIZE, &nback, 1181 &pgs[0], NULL, 1182 UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY|UFP_BACKWARD); 1183 if (nback) { 1184 memmove(&pgs[0], &pgs[npages - nback], 1185 nback * sizeof(pgs[0])); 1186 if (npages - nback < nback) 1187 memset(&pgs[nback], 0, 1188 (npages - nback) * sizeof(pgs[0])); 1189 else 1190 memset(&pgs[npages - nback], 0, 1191 nback * sizeof(pgs[0])); 1192 } 1193 1194 /* 1195 * then plug in our page of interest. 1196 */ 1197 1198 pgs[nback] = pg; 1199 1200 /* 1201 * then look forward to fill in the remaining space in 1202 * the array of pages. 1203 * 1204 * pass our cached array of pages so that hopefully 1205 * uvn_findpages can find some good pages in it. 1206 * the array a was filled above with the one of 1207 * following sets of flags: 1208 * 0 1209 * UVM_PAGE_ARRAY_FILL_DIRTY 1210 * UVM_PAGE_ARRAY_FILL_DIRTY|WRITEBACK 1211 * 1212 * XXX this is fragile but it'll work: the array 1213 * was earlier filled sparsely, but UFP_DIRTYONLY 1214 * implies dense. see corresponding comment in 1215 * uvn_findpages(). 1216 */ 1217 1218 npages = MAXPAGES - nback - 1; 1219 if (fshi) 1220 npages = MIN(npages, 1221 (fshi - off - 1) >> PAGE_SHIFT); 1222 uvn_findpages(uobj, off + PAGE_SIZE, &npages, 1223 &pgs[nback + 1], &a, 1224 UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY); 1225 npages += nback + 1; 1226 } else { 1227 pgs[0] = pg; 1228 npages = 1; 1229 nback = 0; 1230 } 1231 1232 /* 1233 * apply FREE or DEACTIVATE options if requested. 1234 */ 1235 1236 for (i = 0; i < npages; i++) { 1237 tpg = pgs[i]; 1238 KASSERT(tpg->uobject == uobj); 1239 KASSERT(i == 0 || 1240 pgs[i-1]->offset + PAGE_SIZE == tpg->offset); 1241 KASSERT(!needs_clean || uvm_pagegetdirty(pgs[i]) != 1242 UVM_PAGE_STATUS_DIRTY); 1243 if (needs_clean) { 1244 /* 1245 * mark pages as WRITEBACK so that concurrent 1246 * fsync can find and wait for our activities. 1247 */ 1248 radix_tree_set_tag(&uobj->uo_pages, 1249 pgs[i]->offset >> PAGE_SHIFT, 1250 UVM_PAGE_WRITEBACK_TAG); 1251 } 1252 if (tpg->offset < startoff || tpg->offset >= endoff) 1253 continue; 1254 if (flags & PGO_DEACTIVATE && tpg->wire_count == 0) { 1255 uvm_pagelock(tpg); 1256 uvm_pagedeactivate(tpg); 1257 uvm_pageunlock(tpg); 1258 } else if (flags & PGO_FREE) { 1259 pmap_page_protect(tpg, VM_PROT_NONE); 1260 if (tpg->flags & PG_BUSY) { 1261 tpg->flags |= freeflag; 1262 if (pagedaemon) { 1263 uvm_pageout_start(1); 1264 uvm_pagelock(tpg); 1265 uvm_pagedequeue(tpg); 1266 uvm_pageunlock(tpg); 1267 } 1268 } else { 1269 1270 /* 1271 * ``page is not busy'' 1272 * implies that npages is 1 1273 * and needs_clean is false. 1274 */ 1275 1276 KASSERT(npages == 1); 1277 KASSERT(!needs_clean); 1278 KASSERT(pg == tpg); 1279 KASSERT(nextoff == 1280 tpg->offset + PAGE_SIZE); 1281 uvm_pagefree(tpg); 1282 if (pagedaemon) 1283 uvmexp.pdfreed++; 1284 } 1285 } 1286 } 1287 if (needs_clean) { 1288 modified = true; 1289 KASSERT(nextoff == pg->offset + PAGE_SIZE); 1290 KASSERT(nback < npages); 1291 nextoff = pg->offset + ((npages - nback) << PAGE_SHIFT); 1292 KASSERT(pgs[nback] == pg); 1293 KASSERT(nextoff == pgs[npages - 1]->offset + PAGE_SIZE); 1294 1295 /* 1296 * start the i/o. 1297 */ 1298 rw_exit(slock); 1299 error = GOP_WRITE(vp, pgs, npages, flags); 1300 /* 1301 * as we dropped the object lock, our cached pages can 1302 * be stale. 1303 */ 1304 uvm_page_array_clear(&a); 1305 rw_enter(slock, RW_WRITER); 1306 if (error) { 1307 break; 1308 } 1309 } 1310 } 1311 uvm_page_array_fini(&a); 1312 1313 /* 1314 * update ctime/mtime if the modification we started writing out might 1315 * be from mmap'ed write. 1316 * 1317 * this is necessary when an application keeps a file mmaped and 1318 * repeatedly modifies it via the window. note that, because we 1319 * don't always write-protect pages when cleaning, such modifications 1320 * might not involve any page faults. 1321 */ 1322 1323 mutex_enter(vp->v_interlock); 1324 if (modified && (vp->v_iflag & VI_WRMAP) != 0 && 1325 (vp->v_type != VBLK || 1326 (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) { 1327 GOP_MARKUPDATE(vp, GOP_UPDATE_MODIFIED); 1328 } 1329 1330 /* 1331 * if we no longer have any possibly dirty pages, take us off the 1332 * syncer list. 1333 */ 1334 1335 if ((vp->v_iflag & VI_ONWORKLST) != 0 && 1336 radix_tree_empty_tagged_tree_p(&uobj->uo_pages, 1337 UVM_PAGE_DIRTY_TAG)) { 1338 if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL) 1339 vn_syncer_remove_from_worklist(vp); 1340 } 1341 1342 #if !defined(DEBUG) 1343 skip_scan: 1344 #endif /* !defined(DEBUG) */ 1345 1346 /* Wait for output to complete. */ 1347 rw_exit(slock); 1348 if (!wasclean && !async && vp->v_numoutput != 0) { 1349 while (vp->v_numoutput != 0) 1350 cv_wait(&vp->v_cv, vp->v_interlock); 1351 } 1352 onworklst = (vp->v_iflag & VI_ONWORKLST) != 0; 1353 mutex_exit(vp->v_interlock); 1354 1355 if ((flags & PGO_RECLAIM) != 0 && onworklst) { 1356 /* 1357 * in the case of PGO_RECLAIM, ensure to make the vnode clean. 1358 * retrying is not a big deal because, in many cases, 1359 * uobj->uo_npages is already 0 here. 1360 */ 1361 rw_enter(slock, RW_WRITER); 1362 goto retry; 1363 } 1364 1365 if (trans_mp) { 1366 if (holds_wapbl) 1367 WAPBL_END(trans_mp); 1368 fstrans_done(trans_mp); 1369 } 1370 1371 return (error); 1372 } 1373 1374 /* 1375 * Default putrange method for file systems that do not care 1376 * how many pages are given to one GOP_WRITE() call. 1377 */ 1378 void 1379 genfs_gop_putrange(struct vnode *vp, off_t off, off_t *lop, off_t *hip) 1380 { 1381 1382 *lop = 0; 1383 *hip = 0; 1384 } 1385 1386 int 1387 genfs_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, int flags) 1388 { 1389 off_t off; 1390 vaddr_t kva; 1391 size_t len; 1392 int error; 1393 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1394 1395 UVMHIST_LOG(ubchist, "vp %#jx pgs %#jx npages %jd flags 0x%jx", 1396 (uintptr_t)vp, (uintptr_t)pgs, npages, flags); 1397 1398 off = pgs[0]->offset; 1399 kva = uvm_pagermapin(pgs, npages, 1400 UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK); 1401 len = npages << PAGE_SHIFT; 1402 1403 error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE, 1404 uvm_aio_aiodone); 1405 1406 return error; 1407 } 1408 1409 /* 1410 * genfs_gop_write_rwmap: 1411 * 1412 * a variant of genfs_gop_write. it's used by UDF for its directory buffers. 1413 * this maps pages with PROT_WRITE so that VOP_STRATEGY can modifies 1414 * the contents before writing it out to the underlying storage. 1415 */ 1416 1417 int 1418 genfs_gop_write_rwmap(struct vnode *vp, struct vm_page **pgs, int npages, 1419 int flags) 1420 { 1421 off_t off; 1422 vaddr_t kva; 1423 size_t len; 1424 int error; 1425 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1426 1427 UVMHIST_LOG(ubchist, "vp %#jx pgs %#jx npages %jd flags 0x%jx", 1428 (uintptr_t)vp, (uintptr_t)pgs, npages, flags); 1429 1430 off = pgs[0]->offset; 1431 kva = uvm_pagermapin(pgs, npages, 1432 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 1433 len = npages << PAGE_SHIFT; 1434 1435 error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE, 1436 uvm_aio_aiodone); 1437 1438 return error; 1439 } 1440 1441 /* 1442 * Backend routine for doing I/O to vnode pages. Pages are already locked 1443 * and mapped into kernel memory. Here we just look up the underlying 1444 * device block addresses and call the strategy routine. 1445 */ 1446 1447 static int 1448 genfs_do_io(struct vnode *vp, off_t off, vaddr_t kva, size_t len, int flags, 1449 enum uio_rw rw, void (*iodone)(struct buf *)) 1450 { 1451 int s, error; 1452 int fs_bshift, dev_bshift; 1453 off_t eof, offset, startoffset; 1454 size_t bytes, iobytes, skipbytes; 1455 struct buf *mbp, *bp; 1456 const bool async = (flags & PGO_SYNCIO) == 0; 1457 const bool lazy = (flags & PGO_LAZY) == 0; 1458 const bool iowrite = rw == UIO_WRITE; 1459 const int brw = iowrite ? B_WRITE : B_READ; 1460 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1461 1462 UVMHIST_LOG(ubchist, "vp %#jx kva %#jx len 0x%jx flags 0x%jx", 1463 (uintptr_t)vp, (uintptr_t)kva, len, flags); 1464 1465 KASSERT(vp->v_size <= vp->v_writesize); 1466 GOP_SIZE(vp, vp->v_writesize, &eof, 0); 1467 if (vp->v_type != VBLK) { 1468 fs_bshift = vp->v_mount->mnt_fs_bshift; 1469 dev_bshift = vp->v_mount->mnt_dev_bshift; 1470 } else { 1471 fs_bshift = DEV_BSHIFT; 1472 dev_bshift = DEV_BSHIFT; 1473 } 1474 error = 0; 1475 startoffset = off; 1476 bytes = MIN(len, eof - startoffset); 1477 skipbytes = 0; 1478 KASSERT(bytes != 0); 1479 1480 if (iowrite) { 1481 /* 1482 * why += 2? 1483 * 1 for biodone, 1 for uvm_aio_aiodone. 1484 */ 1485 mutex_enter(vp->v_interlock); 1486 vp->v_numoutput += 2; 1487 mutex_exit(vp->v_interlock); 1488 } 1489 mbp = getiobuf(vp, true); 1490 UVMHIST_LOG(ubchist, "vp %#jx mbp %#jx num now %jd bytes 0x%jx", 1491 (uintptr_t)vp, (uintptr_t)mbp, vp->v_numoutput, bytes); 1492 mbp->b_bufsize = len; 1493 mbp->b_data = (void *)kva; 1494 mbp->b_resid = mbp->b_bcount = bytes; 1495 mbp->b_cflags |= BC_BUSY | BC_AGE; 1496 if (async) { 1497 mbp->b_flags = brw | B_ASYNC; 1498 mbp->b_iodone = iodone; 1499 } else { 1500 mbp->b_flags = brw; 1501 mbp->b_iodone = NULL; 1502 } 1503 if (curlwp == uvm.pagedaemon_lwp) 1504 BIO_SETPRIO(mbp, BPRIO_TIMELIMITED); 1505 else if (async || lazy) 1506 BIO_SETPRIO(mbp, BPRIO_TIMENONCRITICAL); 1507 else 1508 BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL); 1509 1510 bp = NULL; 1511 for (offset = startoffset; 1512 bytes > 0; 1513 offset += iobytes, bytes -= iobytes) { 1514 int run; 1515 daddr_t lbn, blkno; 1516 struct vnode *devvp; 1517 1518 /* 1519 * bmap the file to find out the blkno to read from and 1520 * how much we can read in one i/o. if bmap returns an error, 1521 * skip the rest of the top-level i/o. 1522 */ 1523 1524 lbn = offset >> fs_bshift; 1525 error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run); 1526 if (error) { 1527 UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%jx -> %jd\n", 1528 lbn, error, 0, 0); 1529 skipbytes += bytes; 1530 bytes = 0; 1531 goto loopdone; 1532 } 1533 1534 /* 1535 * see how many pages can be read with this i/o. 1536 * reduce the i/o size if necessary to avoid 1537 * overwriting pages with valid data. 1538 */ 1539 1540 iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset, 1541 bytes); 1542 1543 /* 1544 * if this block isn't allocated, zero it instead of 1545 * reading it. unless we are going to allocate blocks, 1546 * mark the pages we zeroed PG_RDONLY. 1547 */ 1548 1549 if (blkno == (daddr_t)-1) { 1550 if (!iowrite) { 1551 memset((char *)kva + (offset - startoffset), 0, 1552 iobytes); 1553 } 1554 skipbytes += iobytes; 1555 continue; 1556 } 1557 1558 /* 1559 * allocate a sub-buf for this piece of the i/o 1560 * (or just use mbp if there's only 1 piece), 1561 * and start it going. 1562 */ 1563 1564 if (offset == startoffset && iobytes == bytes) { 1565 bp = mbp; 1566 } else { 1567 UVMHIST_LOG(ubchist, "vp %#jx bp %#jx num now %jd", 1568 (uintptr_t)vp, (uintptr_t)bp, vp->v_numoutput, 0); 1569 bp = getiobuf(vp, true); 1570 nestiobuf_setup(mbp, bp, offset - startoffset, iobytes); 1571 } 1572 bp->b_lblkno = 0; 1573 1574 /* adjust physical blkno for partial blocks */ 1575 bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >> 1576 dev_bshift); 1577 1578 UVMHIST_LOG(ubchist, 1579 "bp %#jx offset 0x%jx bcount 0x%jx blkno 0x%jx", 1580 (uintptr_t)bp, offset, bp->b_bcount, bp->b_blkno); 1581 1582 VOP_STRATEGY(devvp, bp); 1583 } 1584 1585 loopdone: 1586 if (skipbytes) { 1587 UVMHIST_LOG(ubchist, "skipbytes %jd", skipbytes, 0,0,0); 1588 } 1589 nestiobuf_done(mbp, skipbytes, error); 1590 if (async) { 1591 UVMHIST_LOG(ubchist, "returning 0 (async)", 0,0,0,0); 1592 return (0); 1593 } 1594 UVMHIST_LOG(ubchist, "waiting for mbp %#jx", (uintptr_t)mbp, 0, 0, 0); 1595 error = biowait(mbp); 1596 s = splbio(); 1597 (*iodone)(mbp); 1598 splx(s); 1599 UVMHIST_LOG(ubchist, "returning, error %jd", error, 0, 0, 0); 1600 return (error); 1601 } 1602 1603 int 1604 genfs_compat_getpages(void *v) 1605 { 1606 struct vop_getpages_args /* { 1607 struct vnode *a_vp; 1608 voff_t a_offset; 1609 struct vm_page **a_m; 1610 int *a_count; 1611 int a_centeridx; 1612 vm_prot_t a_access_type; 1613 int a_advice; 1614 int a_flags; 1615 } */ *ap = v; 1616 1617 off_t origoffset; 1618 struct vnode *vp = ap->a_vp; 1619 struct uvm_object *uobj = &vp->v_uobj; 1620 struct vm_page *pg, **pgs; 1621 vaddr_t kva; 1622 int i, error, orignpages, npages; 1623 struct iovec iov; 1624 struct uio uio; 1625 kauth_cred_t cred = curlwp->l_cred; 1626 const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0; 1627 1628 error = 0; 1629 origoffset = ap->a_offset; 1630 orignpages = *ap->a_count; 1631 pgs = ap->a_m; 1632 1633 if (ap->a_flags & PGO_LOCKED) { 1634 uvn_findpages(uobj, origoffset, ap->a_count, ap->a_m, NULL, 1635 UFP_NOWAIT|UFP_NOALLOC| (memwrite ? UFP_NORDONLY : 0)); 1636 1637 error = ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0; 1638 return error; 1639 } 1640 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= vp->v_size) { 1641 rw_exit(uobj->vmobjlock); 1642 return EINVAL; 1643 } 1644 if ((ap->a_flags & PGO_SYNCIO) == 0) { 1645 rw_exit(uobj->vmobjlock); 1646 return 0; 1647 } 1648 npages = orignpages; 1649 uvn_findpages(uobj, origoffset, &npages, pgs, NULL, UFP_ALL); 1650 rw_exit(uobj->vmobjlock); 1651 kva = uvm_pagermapin(pgs, npages, 1652 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 1653 for (i = 0; i < npages; i++) { 1654 pg = pgs[i]; 1655 if ((pg->flags & PG_FAKE) == 0) { 1656 continue; 1657 } 1658 iov.iov_base = (char *)kva + (i << PAGE_SHIFT); 1659 iov.iov_len = PAGE_SIZE; 1660 uio.uio_iov = &iov; 1661 uio.uio_iovcnt = 1; 1662 uio.uio_offset = origoffset + (i << PAGE_SHIFT); 1663 uio.uio_rw = UIO_READ; 1664 uio.uio_resid = PAGE_SIZE; 1665 UIO_SETUP_SYSSPACE(&uio); 1666 /* XXX vn_lock */ 1667 error = VOP_READ(vp, &uio, 0, cred); 1668 if (error) { 1669 break; 1670 } 1671 if (uio.uio_resid) { 1672 memset(iov.iov_base, 0, uio.uio_resid); 1673 } 1674 } 1675 uvm_pagermapout(kva, npages); 1676 rw_enter(uobj->vmobjlock, RW_WRITER); 1677 for (i = 0; i < npages; i++) { 1678 pg = pgs[i]; 1679 if (error && (pg->flags & PG_FAKE) != 0) { 1680 pg->flags |= PG_RELEASED; 1681 } else { 1682 uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_UNKNOWN); 1683 uvm_pagelock(pg); 1684 uvm_pageactivate(pg); 1685 uvm_pageunlock(pg); 1686 } 1687 } 1688 if (error) { 1689 uvm_page_unbusy(pgs, npages); 1690 } 1691 rw_exit(uobj->vmobjlock); 1692 return error; 1693 } 1694 1695 int 1696 genfs_compat_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, 1697 int flags) 1698 { 1699 off_t offset; 1700 struct iovec iov; 1701 struct uio uio; 1702 kauth_cred_t cred = curlwp->l_cred; 1703 struct buf *bp; 1704 vaddr_t kva; 1705 int error; 1706 1707 offset = pgs[0]->offset; 1708 kva = uvm_pagermapin(pgs, npages, 1709 UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK); 1710 1711 iov.iov_base = (void *)kva; 1712 iov.iov_len = npages << PAGE_SHIFT; 1713 uio.uio_iov = &iov; 1714 uio.uio_iovcnt = 1; 1715 uio.uio_offset = offset; 1716 uio.uio_rw = UIO_WRITE; 1717 uio.uio_resid = npages << PAGE_SHIFT; 1718 UIO_SETUP_SYSSPACE(&uio); 1719 /* XXX vn_lock */ 1720 error = VOP_WRITE(vp, &uio, 0, cred); 1721 1722 mutex_enter(vp->v_interlock); 1723 vp->v_numoutput++; 1724 mutex_exit(vp->v_interlock); 1725 1726 bp = getiobuf(vp, true); 1727 bp->b_cflags |= BC_BUSY | BC_AGE; 1728 bp->b_lblkno = offset >> vp->v_mount->mnt_fs_bshift; 1729 bp->b_data = (char *)kva; 1730 bp->b_bcount = npages << PAGE_SHIFT; 1731 bp->b_bufsize = npages << PAGE_SHIFT; 1732 bp->b_resid = 0; 1733 bp->b_error = error; 1734 uvm_aio_aiodone(bp); 1735 return (error); 1736 } 1737 1738 /* 1739 * Process a uio using direct I/O. If we reach a part of the request 1740 * which cannot be processed in this fashion for some reason, just return. 1741 * The caller must handle some additional part of the request using 1742 * buffered I/O before trying direct I/O again. 1743 */ 1744 1745 void 1746 genfs_directio(struct vnode *vp, struct uio *uio, int ioflag) 1747 { 1748 struct vmspace *vs; 1749 struct iovec *iov; 1750 vaddr_t va; 1751 size_t len; 1752 const int mask = DEV_BSIZE - 1; 1753 int error; 1754 bool need_wapbl = (vp->v_mount && vp->v_mount->mnt_wapbl && 1755 (ioflag & IO_JOURNALLOCKED) == 0); 1756 1757 #ifdef DIAGNOSTIC 1758 if ((ioflag & IO_JOURNALLOCKED) && vp->v_mount->mnt_wapbl) 1759 WAPBL_JLOCK_ASSERT(vp->v_mount); 1760 #endif 1761 1762 /* 1763 * We only support direct I/O to user space for now. 1764 */ 1765 1766 if (VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) { 1767 return; 1768 } 1769 1770 /* 1771 * If the vnode is mapped, we would need to get the getpages lock 1772 * to stabilize the bmap, but then we would get into trouble while 1773 * locking the pages if the pages belong to this same vnode (or a 1774 * multi-vnode cascade to the same effect). Just fall back to 1775 * buffered I/O if the vnode is mapped to avoid this mess. 1776 */ 1777 1778 if (vp->v_vflag & VV_MAPPED) { 1779 return; 1780 } 1781 1782 if (need_wapbl) { 1783 error = WAPBL_BEGIN(vp->v_mount); 1784 if (error) 1785 return; 1786 } 1787 1788 /* 1789 * Do as much of the uio as possible with direct I/O. 1790 */ 1791 1792 vs = uio->uio_vmspace; 1793 while (uio->uio_resid) { 1794 iov = uio->uio_iov; 1795 if (iov->iov_len == 0) { 1796 uio->uio_iov++; 1797 uio->uio_iovcnt--; 1798 continue; 1799 } 1800 va = (vaddr_t)iov->iov_base; 1801 len = MIN(iov->iov_len, genfs_maxdio); 1802 len &= ~mask; 1803 1804 /* 1805 * If the next chunk is smaller than DEV_BSIZE or extends past 1806 * the current EOF, then fall back to buffered I/O. 1807 */ 1808 1809 if (len == 0 || uio->uio_offset + len > vp->v_size) { 1810 break; 1811 } 1812 1813 /* 1814 * Check alignment. The file offset must be at least 1815 * sector-aligned. The exact constraint on memory alignment 1816 * is very hardware-dependent, but requiring sector-aligned 1817 * addresses there too is safe. 1818 */ 1819 1820 if (uio->uio_offset & mask || va & mask) { 1821 break; 1822 } 1823 error = genfs_do_directio(vs, va, len, vp, uio->uio_offset, 1824 uio->uio_rw); 1825 if (error) { 1826 break; 1827 } 1828 iov->iov_base = (char *)iov->iov_base + len; 1829 iov->iov_len -= len; 1830 uio->uio_offset += len; 1831 uio->uio_resid -= len; 1832 } 1833 1834 if (need_wapbl) 1835 WAPBL_END(vp->v_mount); 1836 } 1837 1838 /* 1839 * Iodone routine for direct I/O. We don't do much here since the request is 1840 * always synchronous, so the caller will do most of the work after biowait(). 1841 */ 1842 1843 static void 1844 genfs_dio_iodone(struct buf *bp) 1845 { 1846 1847 KASSERT((bp->b_flags & B_ASYNC) == 0); 1848 if ((bp->b_flags & B_READ) == 0 && (bp->b_cflags & BC_AGE) != 0) { 1849 mutex_enter(bp->b_objlock); 1850 vwakeup(bp); 1851 mutex_exit(bp->b_objlock); 1852 } 1853 putiobuf(bp); 1854 } 1855 1856 /* 1857 * Process one chunk of a direct I/O request. 1858 */ 1859 1860 static int 1861 genfs_do_directio(struct vmspace *vs, vaddr_t uva, size_t len, struct vnode *vp, 1862 off_t off, enum uio_rw rw) 1863 { 1864 struct vm_map *map; 1865 struct pmap *upm, *kpm __unused; 1866 size_t klen = round_page(uva + len) - trunc_page(uva); 1867 off_t spoff, epoff; 1868 vaddr_t kva, puva; 1869 paddr_t pa; 1870 vm_prot_t prot; 1871 int error, rv __diagused, poff, koff; 1872 const int pgoflags = PGO_CLEANIT | PGO_SYNCIO | PGO_JOURNALLOCKED | 1873 (rw == UIO_WRITE ? PGO_FREE : 0); 1874 1875 /* 1876 * For writes, verify that this range of the file already has fully 1877 * allocated backing store. If there are any holes, just punt and 1878 * make the caller take the buffered write path. 1879 */ 1880 1881 if (rw == UIO_WRITE) { 1882 daddr_t lbn, elbn, blkno; 1883 int bsize, bshift, run; 1884 1885 bshift = vp->v_mount->mnt_fs_bshift; 1886 bsize = 1 << bshift; 1887 lbn = off >> bshift; 1888 elbn = (off + len + bsize - 1) >> bshift; 1889 while (lbn < elbn) { 1890 error = VOP_BMAP(vp, lbn, NULL, &blkno, &run); 1891 if (error) { 1892 return error; 1893 } 1894 if (blkno == (daddr_t)-1) { 1895 return ENOSPC; 1896 } 1897 lbn += 1 + run; 1898 } 1899 } 1900 1901 /* 1902 * Flush any cached pages for parts of the file that we're about to 1903 * access. If we're writing, invalidate pages as well. 1904 */ 1905 1906 spoff = trunc_page(off); 1907 epoff = round_page(off + len); 1908 rw_enter(vp->v_uobj.vmobjlock, RW_WRITER); 1909 error = VOP_PUTPAGES(vp, spoff, epoff, pgoflags); 1910 if (error) { 1911 return error; 1912 } 1913 1914 /* 1915 * Wire the user pages and remap them into kernel memory. 1916 */ 1917 1918 prot = rw == UIO_READ ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_READ; 1919 error = uvm_vslock(vs, (void *)uva, len, prot); 1920 if (error) { 1921 return error; 1922 } 1923 1924 map = &vs->vm_map; 1925 upm = vm_map_pmap(map); 1926 kpm = vm_map_pmap(kernel_map); 1927 puva = trunc_page(uva); 1928 kva = uvm_km_alloc(kernel_map, klen, atop(puva) & uvmexp.colormask, 1929 UVM_KMF_VAONLY | UVM_KMF_WAITVA | UVM_KMF_COLORMATCH); 1930 for (poff = 0; poff < klen; poff += PAGE_SIZE) { 1931 rv = pmap_extract(upm, puva + poff, &pa); 1932 KASSERT(rv); 1933 pmap_kenter_pa(kva + poff, pa, prot, PMAP_WIRED); 1934 } 1935 pmap_update(kpm); 1936 1937 /* 1938 * Do the I/O. 1939 */ 1940 1941 koff = uva - trunc_page(uva); 1942 error = genfs_do_io(vp, off, kva + koff, len, PGO_SYNCIO, rw, 1943 genfs_dio_iodone); 1944 1945 /* 1946 * Tear down the kernel mapping. 1947 */ 1948 1949 pmap_kremove(kva, klen); 1950 pmap_update(kpm); 1951 uvm_km_free(kernel_map, kva, klen, UVM_KMF_VAONLY); 1952 1953 /* 1954 * Unwire the user pages. 1955 */ 1956 1957 uvm_vsunlock(vs, (void *)uva, len); 1958 return error; 1959 } 1960