1 /* $NetBSD: genfs_io.c,v 1.60 2015/04/12 14:44:06 skrll Exp $ */ 2 3 /* 4 * Copyright (c) 1982, 1986, 1989, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the University nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 */ 32 33 #include <sys/cdefs.h> 34 __KERNEL_RCSID(0, "$NetBSD: genfs_io.c,v 1.60 2015/04/12 14:44:06 skrll Exp $"); 35 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/proc.h> 39 #include <sys/kernel.h> 40 #include <sys/mount.h> 41 #include <sys/vnode.h> 42 #include <sys/kmem.h> 43 #include <sys/kauth.h> 44 #include <sys/fstrans.h> 45 #include <sys/buf.h> 46 47 #include <miscfs/genfs/genfs.h> 48 #include <miscfs/genfs/genfs_node.h> 49 #include <miscfs/specfs/specdev.h> 50 #include <miscfs/syncfs/syncfs.h> 51 52 #include <uvm/uvm.h> 53 #include <uvm/uvm_pager.h> 54 55 static int genfs_do_directio(struct vmspace *, vaddr_t, size_t, struct vnode *, 56 off_t, enum uio_rw); 57 static void genfs_dio_iodone(struct buf *); 58 59 static int genfs_getpages_read(struct vnode *, struct vm_page **, int, off_t, 60 off_t, bool, bool, bool, bool); 61 static int genfs_do_io(struct vnode *, off_t, vaddr_t, size_t, int, enum uio_rw, 62 void (*)(struct buf *)); 63 static void genfs_rel_pages(struct vm_page **, unsigned int); 64 static void genfs_markdirty(struct vnode *); 65 66 int genfs_maxdio = MAXPHYS; 67 68 static void 69 genfs_rel_pages(struct vm_page **pgs, unsigned int npages) 70 { 71 unsigned int i; 72 73 for (i = 0; i < npages; i++) { 74 struct vm_page *pg = pgs[i]; 75 76 if (pg == NULL || pg == PGO_DONTCARE) 77 continue; 78 KASSERT(uvm_page_locked_p(pg)); 79 if (pg->flags & PG_FAKE) { 80 pg->flags |= PG_RELEASED; 81 } 82 } 83 mutex_enter(&uvm_pageqlock); 84 uvm_page_unbusy(pgs, npages); 85 mutex_exit(&uvm_pageqlock); 86 } 87 88 static void 89 genfs_markdirty(struct vnode *vp) 90 { 91 struct genfs_node * const gp = VTOG(vp); 92 93 KASSERT(mutex_owned(vp->v_interlock)); 94 gp->g_dirtygen++; 95 if ((vp->v_iflag & VI_ONWORKLST) == 0) { 96 vn_syncer_add_to_worklist(vp, filedelay); 97 } 98 if ((vp->v_iflag & (VI_WRMAP|VI_WRMAPDIRTY)) == VI_WRMAP) { 99 vp->v_iflag |= VI_WRMAPDIRTY; 100 } 101 } 102 103 /* 104 * generic VM getpages routine. 105 * Return PG_BUSY pages for the given range, 106 * reading from backing store if necessary. 107 */ 108 109 int 110 genfs_getpages(void *v) 111 { 112 struct vop_getpages_args /* { 113 struct vnode *a_vp; 114 voff_t a_offset; 115 struct vm_page **a_m; 116 int *a_count; 117 int a_centeridx; 118 vm_prot_t a_access_type; 119 int a_advice; 120 int a_flags; 121 } */ * const ap = v; 122 123 off_t diskeof, memeof; 124 int i, error, npages; 125 const int flags = ap->a_flags; 126 struct vnode * const vp = ap->a_vp; 127 struct uvm_object * const uobj = &vp->v_uobj; 128 const bool async = (flags & PGO_SYNCIO) == 0; 129 const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0; 130 const bool overwrite = (flags & PGO_OVERWRITE) != 0; 131 const bool blockalloc = memwrite && (flags & PGO_NOBLOCKALLOC) == 0; 132 const bool glocked = (flags & PGO_GLOCKHELD) != 0; 133 const bool need_wapbl = blockalloc && vp->v_mount->mnt_wapbl; 134 bool has_trans_wapbl = false; 135 UVMHIST_FUNC("genfs_getpages"); UVMHIST_CALLED(ubchist); 136 137 UVMHIST_LOG(ubchist, "vp %p off 0x%x/%x count %d", 138 vp, ap->a_offset >> 32, ap->a_offset, *ap->a_count); 139 140 KASSERT(vp->v_type == VREG || vp->v_type == VDIR || 141 vp->v_type == VLNK || vp->v_type == VBLK); 142 143 startover: 144 error = 0; 145 const voff_t origvsize = vp->v_size; 146 const off_t origoffset = ap->a_offset; 147 const int orignpages = *ap->a_count; 148 149 GOP_SIZE(vp, origvsize, &diskeof, 0); 150 if (flags & PGO_PASTEOF) { 151 off_t newsize; 152 #if defined(DIAGNOSTIC) 153 off_t writeeof; 154 #endif /* defined(DIAGNOSTIC) */ 155 156 newsize = MAX(origvsize, 157 origoffset + (orignpages << PAGE_SHIFT)); 158 GOP_SIZE(vp, newsize, &memeof, GOP_SIZE_MEM); 159 #if defined(DIAGNOSTIC) 160 GOP_SIZE(vp, vp->v_writesize, &writeeof, GOP_SIZE_MEM); 161 if (newsize > round_page(writeeof)) { 162 panic("%s: past eof: %" PRId64 " vs. %" PRId64, 163 __func__, newsize, round_page(writeeof)); 164 } 165 #endif /* defined(DIAGNOSTIC) */ 166 } else { 167 GOP_SIZE(vp, origvsize, &memeof, GOP_SIZE_MEM); 168 } 169 KASSERT(ap->a_centeridx >= 0 || ap->a_centeridx <= orignpages); 170 KASSERT((origoffset & (PAGE_SIZE - 1)) == 0 && origoffset >= 0); 171 KASSERT(orignpages > 0); 172 173 /* 174 * Bounds-check the request. 175 */ 176 177 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= memeof) { 178 if ((flags & PGO_LOCKED) == 0) { 179 mutex_exit(uobj->vmobjlock); 180 } 181 UVMHIST_LOG(ubchist, "off 0x%x count %d goes past EOF 0x%x", 182 origoffset, *ap->a_count, memeof,0); 183 error = EINVAL; 184 goto out_err; 185 } 186 187 /* uobj is locked */ 188 189 if ((flags & PGO_NOTIMESTAMP) == 0 && 190 (vp->v_type != VBLK || 191 (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) { 192 int updflags = 0; 193 194 if ((vp->v_mount->mnt_flag & MNT_NOATIME) == 0) { 195 updflags = GOP_UPDATE_ACCESSED; 196 } 197 if (memwrite) { 198 updflags |= GOP_UPDATE_MODIFIED; 199 } 200 if (updflags != 0) { 201 GOP_MARKUPDATE(vp, updflags); 202 } 203 } 204 205 /* 206 * For PGO_LOCKED requests, just return whatever's in memory. 207 */ 208 209 if (flags & PGO_LOCKED) { 210 int nfound; 211 struct vm_page *pg; 212 213 KASSERT(!glocked); 214 npages = *ap->a_count; 215 #if defined(DEBUG) 216 for (i = 0; i < npages; i++) { 217 pg = ap->a_m[i]; 218 KASSERT(pg == NULL || pg == PGO_DONTCARE); 219 } 220 #endif /* defined(DEBUG) */ 221 nfound = uvn_findpages(uobj, origoffset, &npages, 222 ap->a_m, UFP_NOWAIT|UFP_NOALLOC|(memwrite ? UFP_NORDONLY : 0)); 223 KASSERT(npages == *ap->a_count); 224 if (nfound == 0) { 225 error = EBUSY; 226 goto out_err; 227 } 228 if (!genfs_node_rdtrylock(vp)) { 229 genfs_rel_pages(ap->a_m, npages); 230 231 /* 232 * restore the array. 233 */ 234 235 for (i = 0; i < npages; i++) { 236 pg = ap->a_m[i]; 237 238 if (pg != NULL && pg != PGO_DONTCARE) { 239 ap->a_m[i] = NULL; 240 } 241 KASSERT(ap->a_m[i] == NULL || 242 ap->a_m[i] == PGO_DONTCARE); 243 } 244 } else { 245 genfs_node_unlock(vp); 246 } 247 error = (ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0); 248 if (error == 0 && memwrite) { 249 genfs_markdirty(vp); 250 } 251 goto out_err; 252 } 253 mutex_exit(uobj->vmobjlock); 254 255 /* 256 * find the requested pages and make some simple checks. 257 * leave space in the page array for a whole block. 258 */ 259 260 const int fs_bshift = (vp->v_type != VBLK) ? 261 vp->v_mount->mnt_fs_bshift : DEV_BSHIFT; 262 const int fs_bsize = 1 << fs_bshift; 263 #define blk_mask (fs_bsize - 1) 264 #define trunc_blk(x) ((x) & ~blk_mask) 265 #define round_blk(x) (((x) + blk_mask) & ~blk_mask) 266 267 const int orignmempages = MIN(orignpages, 268 round_page(memeof - origoffset) >> PAGE_SHIFT); 269 npages = orignmempages; 270 const off_t startoffset = trunc_blk(origoffset); 271 const off_t endoffset = MIN( 272 round_page(round_blk(origoffset + (npages << PAGE_SHIFT))), 273 round_page(memeof)); 274 const int ridx = (origoffset - startoffset) >> PAGE_SHIFT; 275 276 const int pgs_size = sizeof(struct vm_page *) * 277 ((endoffset - startoffset) >> PAGE_SHIFT); 278 struct vm_page **pgs, *pgs_onstack[UBC_MAX_PAGES]; 279 280 if (pgs_size > sizeof(pgs_onstack)) { 281 pgs = kmem_zalloc(pgs_size, async ? KM_NOSLEEP : KM_SLEEP); 282 if (pgs == NULL) { 283 pgs = pgs_onstack; 284 error = ENOMEM; 285 goto out_err; 286 } 287 } else { 288 pgs = pgs_onstack; 289 (void)memset(pgs, 0, pgs_size); 290 } 291 292 UVMHIST_LOG(ubchist, "ridx %d npages %d startoff %ld endoff %ld", 293 ridx, npages, startoffset, endoffset); 294 295 if (!has_trans_wapbl) { 296 fstrans_start(vp->v_mount, FSTRANS_SHARED); 297 /* 298 * XXX: This assumes that we come here only via 299 * the mmio path 300 */ 301 if (need_wapbl) { 302 error = WAPBL_BEGIN(vp->v_mount); 303 if (error) { 304 fstrans_done(vp->v_mount); 305 goto out_err_free; 306 } 307 } 308 has_trans_wapbl = true; 309 } 310 311 /* 312 * hold g_glock to prevent a race with truncate. 313 * 314 * check if our idea of v_size is still valid. 315 */ 316 317 KASSERT(!glocked || genfs_node_wrlocked(vp)); 318 if (!glocked) { 319 if (blockalloc) { 320 genfs_node_wrlock(vp); 321 } else { 322 genfs_node_rdlock(vp); 323 } 324 } 325 mutex_enter(uobj->vmobjlock); 326 if (vp->v_size < origvsize) { 327 if (!glocked) { 328 genfs_node_unlock(vp); 329 } 330 if (pgs != pgs_onstack) 331 kmem_free(pgs, pgs_size); 332 goto startover; 333 } 334 335 if (uvn_findpages(uobj, origoffset, &npages, &pgs[ridx], 336 async ? UFP_NOWAIT : UFP_ALL) != orignmempages) { 337 if (!glocked) { 338 genfs_node_unlock(vp); 339 } 340 KASSERT(async != 0); 341 genfs_rel_pages(&pgs[ridx], orignmempages); 342 mutex_exit(uobj->vmobjlock); 343 error = EBUSY; 344 goto out_err_free; 345 } 346 347 /* 348 * if the pages are already resident, just return them. 349 */ 350 351 for (i = 0; i < npages; i++) { 352 struct vm_page *pg = pgs[ridx + i]; 353 354 if ((pg->flags & PG_FAKE) || 355 (blockalloc && (pg->flags & PG_RDONLY))) { 356 break; 357 } 358 } 359 if (i == npages) { 360 if (!glocked) { 361 genfs_node_unlock(vp); 362 } 363 UVMHIST_LOG(ubchist, "returning cached pages", 0,0,0,0); 364 npages += ridx; 365 goto out; 366 } 367 368 /* 369 * if PGO_OVERWRITE is set, don't bother reading the pages. 370 */ 371 372 if (overwrite) { 373 if (!glocked) { 374 genfs_node_unlock(vp); 375 } 376 UVMHIST_LOG(ubchist, "PGO_OVERWRITE",0,0,0,0); 377 378 for (i = 0; i < npages; i++) { 379 struct vm_page *pg = pgs[ridx + i]; 380 381 pg->flags &= ~(PG_RDONLY|PG_CLEAN); 382 } 383 npages += ridx; 384 goto out; 385 } 386 387 /* 388 * the page wasn't resident and we're not overwriting, 389 * so we're going to have to do some i/o. 390 * find any additional pages needed to cover the expanded range. 391 */ 392 393 npages = (endoffset - startoffset) >> PAGE_SHIFT; 394 if (startoffset != origoffset || npages != orignmempages) { 395 int npgs; 396 397 /* 398 * we need to avoid deadlocks caused by locking 399 * additional pages at lower offsets than pages we 400 * already have locked. unlock them all and start over. 401 */ 402 403 genfs_rel_pages(&pgs[ridx], orignmempages); 404 memset(pgs, 0, pgs_size); 405 406 UVMHIST_LOG(ubchist, "reset npages start 0x%x end 0x%x", 407 startoffset, endoffset, 0,0); 408 npgs = npages; 409 if (uvn_findpages(uobj, startoffset, &npgs, pgs, 410 async ? UFP_NOWAIT : UFP_ALL) != npages) { 411 if (!glocked) { 412 genfs_node_unlock(vp); 413 } 414 KASSERT(async != 0); 415 genfs_rel_pages(pgs, npages); 416 mutex_exit(uobj->vmobjlock); 417 error = EBUSY; 418 goto out_err_free; 419 } 420 } 421 422 mutex_exit(uobj->vmobjlock); 423 error = genfs_getpages_read(vp, pgs, npages, startoffset, diskeof, 424 async, memwrite, blockalloc, glocked); 425 if (error == 0 && async) 426 goto out_err_free; 427 if (!glocked) { 428 genfs_node_unlock(vp); 429 } 430 mutex_enter(uobj->vmobjlock); 431 432 /* 433 * we're almost done! release the pages... 434 * for errors, we free the pages. 435 * otherwise we activate them and mark them as valid and clean. 436 * also, unbusy pages that were not actually requested. 437 */ 438 439 if (error) { 440 genfs_rel_pages(pgs, npages); 441 mutex_exit(uobj->vmobjlock); 442 UVMHIST_LOG(ubchist, "returning error %d", error,0,0,0); 443 goto out_err_free; 444 } 445 446 out: 447 UVMHIST_LOG(ubchist, "succeeding, npages %d", npages,0,0,0); 448 error = 0; 449 mutex_enter(&uvm_pageqlock); 450 for (i = 0; i < npages; i++) { 451 struct vm_page *pg = pgs[i]; 452 if (pg == NULL) { 453 continue; 454 } 455 UVMHIST_LOG(ubchist, "examining pg %p flags 0x%x", 456 pg, pg->flags, 0,0); 457 if (pg->flags & PG_FAKE && !overwrite) { 458 pg->flags &= ~(PG_FAKE); 459 pmap_clear_modify(pgs[i]); 460 } 461 KASSERT(!memwrite || !blockalloc || (pg->flags & PG_RDONLY) == 0); 462 if (i < ridx || i >= ridx + orignmempages || async) { 463 UVMHIST_LOG(ubchist, "unbusy pg %p offset 0x%x", 464 pg, pg->offset,0,0); 465 if (pg->flags & PG_WANTED) { 466 wakeup(pg); 467 } 468 if (pg->flags & PG_FAKE) { 469 KASSERT(overwrite); 470 uvm_pagezero(pg); 471 } 472 if (pg->flags & PG_RELEASED) { 473 uvm_pagefree(pg); 474 continue; 475 } 476 uvm_pageenqueue(pg); 477 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE); 478 UVM_PAGE_OWN(pg, NULL); 479 } 480 } 481 mutex_exit(&uvm_pageqlock); 482 if (memwrite) { 483 genfs_markdirty(vp); 484 } 485 mutex_exit(uobj->vmobjlock); 486 if (ap->a_m != NULL) { 487 memcpy(ap->a_m, &pgs[ridx], 488 orignmempages * sizeof(struct vm_page *)); 489 } 490 491 out_err_free: 492 if (pgs != NULL && pgs != pgs_onstack) 493 kmem_free(pgs, pgs_size); 494 out_err: 495 if (has_trans_wapbl) { 496 if (need_wapbl) 497 WAPBL_END(vp->v_mount); 498 fstrans_done(vp->v_mount); 499 } 500 return error; 501 } 502 503 /* 504 * genfs_getpages_read: Read the pages in with VOP_BMAP/VOP_STRATEGY. 505 */ 506 static int 507 genfs_getpages_read(struct vnode *vp, struct vm_page **pgs, int npages, 508 off_t startoffset, off_t diskeof, 509 bool async, bool memwrite, bool blockalloc, bool glocked) 510 { 511 struct uvm_object * const uobj = &vp->v_uobj; 512 const int fs_bshift = (vp->v_type != VBLK) ? 513 vp->v_mount->mnt_fs_bshift : DEV_BSHIFT; 514 const int dev_bshift = (vp->v_type != VBLK) ? 515 vp->v_mount->mnt_dev_bshift : DEV_BSHIFT; 516 kauth_cred_t const cred = curlwp->l_cred; /* XXXUBC curlwp */ 517 size_t bytes, iobytes, tailstart, tailbytes, totalbytes, skipbytes; 518 vaddr_t kva; 519 struct buf *bp, *mbp; 520 bool sawhole = false; 521 int i; 522 int error = 0; 523 524 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 525 526 /* 527 * read the desired page(s). 528 */ 529 530 totalbytes = npages << PAGE_SHIFT; 531 bytes = MIN(totalbytes, MAX(diskeof - startoffset, 0)); 532 tailbytes = totalbytes - bytes; 533 skipbytes = 0; 534 535 kva = uvm_pagermapin(pgs, npages, 536 UVMPAGER_MAPIN_READ | (async ? 0 : UVMPAGER_MAPIN_WAITOK)); 537 if (kva == 0) 538 return EBUSY; 539 540 mbp = getiobuf(vp, true); 541 mbp->b_bufsize = totalbytes; 542 mbp->b_data = (void *)kva; 543 mbp->b_resid = mbp->b_bcount = bytes; 544 mbp->b_cflags = BC_BUSY; 545 if (async) { 546 mbp->b_flags = B_READ | B_ASYNC; 547 mbp->b_iodone = uvm_aio_biodone; 548 } else { 549 mbp->b_flags = B_READ; 550 mbp->b_iodone = NULL; 551 } 552 if (async) 553 BIO_SETPRIO(mbp, BPRIO_TIMELIMITED); 554 else 555 BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL); 556 557 /* 558 * if EOF is in the middle of the range, zero the part past EOF. 559 * skip over pages which are not PG_FAKE since in that case they have 560 * valid data that we need to preserve. 561 */ 562 563 tailstart = bytes; 564 while (tailbytes > 0) { 565 const int len = PAGE_SIZE - (tailstart & PAGE_MASK); 566 567 KASSERT(len <= tailbytes); 568 if ((pgs[tailstart >> PAGE_SHIFT]->flags & PG_FAKE) != 0) { 569 memset((void *)(kva + tailstart), 0, len); 570 UVMHIST_LOG(ubchist, "tailbytes %p 0x%x 0x%x", 571 kva, tailstart, len, 0); 572 } 573 tailstart += len; 574 tailbytes -= len; 575 } 576 577 /* 578 * now loop over the pages, reading as needed. 579 */ 580 581 bp = NULL; 582 off_t offset; 583 for (offset = startoffset; 584 bytes > 0; 585 offset += iobytes, bytes -= iobytes) { 586 int run; 587 daddr_t lbn, blkno; 588 int pidx; 589 struct vnode *devvp; 590 591 /* 592 * skip pages which don't need to be read. 593 */ 594 595 pidx = (offset - startoffset) >> PAGE_SHIFT; 596 while ((pgs[pidx]->flags & PG_FAKE) == 0) { 597 size_t b; 598 599 KASSERT((offset & (PAGE_SIZE - 1)) == 0); 600 if ((pgs[pidx]->flags & PG_RDONLY)) { 601 sawhole = true; 602 } 603 b = MIN(PAGE_SIZE, bytes); 604 offset += b; 605 bytes -= b; 606 skipbytes += b; 607 pidx++; 608 UVMHIST_LOG(ubchist, "skipping, new offset 0x%x", 609 offset, 0,0,0); 610 if (bytes == 0) { 611 goto loopdone; 612 } 613 } 614 615 /* 616 * bmap the file to find out the blkno to read from and 617 * how much we can read in one i/o. if bmap returns an error, 618 * skip the rest of the top-level i/o. 619 */ 620 621 lbn = offset >> fs_bshift; 622 error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run); 623 if (error) { 624 UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%x -> %d\n", 625 lbn,error,0,0); 626 skipbytes += bytes; 627 bytes = 0; 628 goto loopdone; 629 } 630 631 /* 632 * see how many pages can be read with this i/o. 633 * reduce the i/o size if necessary to avoid 634 * overwriting pages with valid data. 635 */ 636 637 iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset, 638 bytes); 639 if (offset + iobytes > round_page(offset)) { 640 int pcount; 641 642 pcount = 1; 643 while (pidx + pcount < npages && 644 pgs[pidx + pcount]->flags & PG_FAKE) { 645 pcount++; 646 } 647 iobytes = MIN(iobytes, (pcount << PAGE_SHIFT) - 648 (offset - trunc_page(offset))); 649 } 650 651 /* 652 * if this block isn't allocated, zero it instead of 653 * reading it. unless we are going to allocate blocks, 654 * mark the pages we zeroed PG_RDONLY. 655 */ 656 657 if (blkno == (daddr_t)-1) { 658 int holepages = (round_page(offset + iobytes) - 659 trunc_page(offset)) >> PAGE_SHIFT; 660 UVMHIST_LOG(ubchist, "lbn 0x%x -> HOLE", lbn,0,0,0); 661 662 sawhole = true; 663 memset((char *)kva + (offset - startoffset), 0, 664 iobytes); 665 skipbytes += iobytes; 666 667 mutex_enter(uobj->vmobjlock); 668 for (i = 0; i < holepages; i++) { 669 if (memwrite) { 670 pgs[pidx + i]->flags &= ~PG_CLEAN; 671 } 672 if (!blockalloc) { 673 pgs[pidx + i]->flags |= PG_RDONLY; 674 } 675 } 676 mutex_exit(uobj->vmobjlock); 677 continue; 678 } 679 680 /* 681 * allocate a sub-buf for this piece of the i/o 682 * (or just use mbp if there's only 1 piece), 683 * and start it going. 684 */ 685 686 if (offset == startoffset && iobytes == bytes) { 687 bp = mbp; 688 } else { 689 UVMHIST_LOG(ubchist, "vp %p bp %p num now %d", 690 vp, bp, vp->v_numoutput, 0); 691 bp = getiobuf(vp, true); 692 nestiobuf_setup(mbp, bp, offset - startoffset, iobytes); 693 } 694 bp->b_lblkno = 0; 695 696 /* adjust physical blkno for partial blocks */ 697 bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >> 698 dev_bshift); 699 700 UVMHIST_LOG(ubchist, 701 "bp %p offset 0x%x bcount 0x%x blkno 0x%x", 702 bp, offset, bp->b_bcount, bp->b_blkno); 703 704 VOP_STRATEGY(devvp, bp); 705 } 706 707 loopdone: 708 nestiobuf_done(mbp, skipbytes, error); 709 if (async) { 710 UVMHIST_LOG(ubchist, "returning 0 (async)",0,0,0,0); 711 if (!glocked) { 712 genfs_node_unlock(vp); 713 } 714 return 0; 715 } 716 if (bp != NULL) { 717 error = biowait(mbp); 718 } 719 720 /* Remove the mapping (make KVA available as soon as possible) */ 721 uvm_pagermapout(kva, npages); 722 723 /* 724 * if this we encountered a hole then we have to do a little more work. 725 * for read faults, we marked the page PG_RDONLY so that future 726 * write accesses to the page will fault again. 727 * for write faults, we must make sure that the backing store for 728 * the page is completely allocated while the pages are locked. 729 */ 730 731 if (!error && sawhole && blockalloc) { 732 error = GOP_ALLOC(vp, startoffset, 733 npages << PAGE_SHIFT, 0, cred); 734 UVMHIST_LOG(ubchist, "gop_alloc off 0x%x/0x%x -> %d", 735 startoffset, npages << PAGE_SHIFT, error,0); 736 if (!error) { 737 mutex_enter(uobj->vmobjlock); 738 for (i = 0; i < npages; i++) { 739 struct vm_page *pg = pgs[i]; 740 741 if (pg == NULL) { 742 continue; 743 } 744 pg->flags &= ~(PG_CLEAN|PG_RDONLY); 745 UVMHIST_LOG(ubchist, "mark dirty pg %p", 746 pg,0,0,0); 747 } 748 mutex_exit(uobj->vmobjlock); 749 } 750 } 751 752 putiobuf(mbp); 753 return error; 754 } 755 756 /* 757 * generic VM putpages routine. 758 * Write the given range of pages to backing store. 759 * 760 * => "offhi == 0" means flush all pages at or after "offlo". 761 * => object should be locked by caller. we return with the 762 * object unlocked. 763 * => if PGO_CLEANIT or PGO_SYNCIO is set, we may block (due to I/O). 764 * thus, a caller might want to unlock higher level resources 765 * (e.g. vm_map) before calling flush. 766 * => if neither PGO_CLEANIT nor PGO_SYNCIO is set, we will not block 767 * => if PGO_ALLPAGES is set, then all pages in the object will be processed. 768 * => NOTE: we rely on the fact that the object's memq is a TAILQ and 769 * that new pages are inserted on the tail end of the list. thus, 770 * we can make a complete pass through the object in one go by starting 771 * at the head and working towards the tail (new pages are put in 772 * front of us). 773 * => NOTE: we are allowed to lock the page queues, so the caller 774 * must not be holding the page queue lock. 775 * 776 * note on "cleaning" object and PG_BUSY pages: 777 * this routine is holding the lock on the object. the only time 778 * that it can run into a PG_BUSY page that it does not own is if 779 * some other process has started I/O on the page (e.g. either 780 * a pagein, or a pageout). if the PG_BUSY page is being paged 781 * in, then it can not be dirty (!PG_CLEAN) because no one has 782 * had a chance to modify it yet. if the PG_BUSY page is being 783 * paged out then it means that someone else has already started 784 * cleaning the page for us (how nice!). in this case, if we 785 * have syncio specified, then after we make our pass through the 786 * object we need to wait for the other PG_BUSY pages to clear 787 * off (i.e. we need to do an iosync). also note that once a 788 * page is PG_BUSY it must stay in its object until it is un-busyed. 789 * 790 * note on page traversal: 791 * we can traverse the pages in an object either by going down the 792 * linked list in "uobj->memq", or we can go over the address range 793 * by page doing hash table lookups for each address. depending 794 * on how many pages are in the object it may be cheaper to do one 795 * or the other. we set "by_list" to true if we are using memq. 796 * if the cost of a hash lookup was equal to the cost of the list 797 * traversal we could compare the number of pages in the start->stop 798 * range to the total number of pages in the object. however, it 799 * seems that a hash table lookup is more expensive than the linked 800 * list traversal, so we multiply the number of pages in the 801 * range by an estimate of the relatively higher cost of the hash lookup. 802 */ 803 804 int 805 genfs_putpages(void *v) 806 { 807 struct vop_putpages_args /* { 808 struct vnode *a_vp; 809 voff_t a_offlo; 810 voff_t a_offhi; 811 int a_flags; 812 } */ * const ap = v; 813 814 return genfs_do_putpages(ap->a_vp, ap->a_offlo, ap->a_offhi, 815 ap->a_flags, NULL); 816 } 817 818 int 819 genfs_do_putpages(struct vnode *vp, off_t startoff, off_t endoff, 820 int origflags, struct vm_page **busypg) 821 { 822 struct uvm_object * const uobj = &vp->v_uobj; 823 kmutex_t * const slock = uobj->vmobjlock; 824 off_t off; 825 /* Even for strange MAXPHYS, the shift rounds down to a page */ 826 #define maxpages (MAXPHYS >> PAGE_SHIFT) 827 int i, error, npages, nback; 828 int freeflag; 829 struct vm_page *pgs[maxpages], *pg, *nextpg, *tpg, curmp, endmp; 830 bool wasclean, by_list, needs_clean, yld; 831 bool async = (origflags & PGO_SYNCIO) == 0; 832 bool pagedaemon = curlwp == uvm.pagedaemon_lwp; 833 struct lwp * const l = curlwp ? curlwp : &lwp0; 834 struct genfs_node * const gp = VTOG(vp); 835 int flags; 836 int dirtygen; 837 bool modified; 838 bool need_wapbl; 839 bool has_trans; 840 bool cleanall; 841 bool onworklst; 842 843 UVMHIST_FUNC("genfs_putpages"); UVMHIST_CALLED(ubchist); 844 845 KASSERT(origflags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)); 846 KASSERT((startoff & PAGE_MASK) == 0 && (endoff & PAGE_MASK) == 0); 847 KASSERT(startoff < endoff || endoff == 0); 848 849 UVMHIST_LOG(ubchist, "vp %p pages %d off 0x%x len 0x%x", 850 vp, uobj->uo_npages, startoff, endoff - startoff); 851 852 has_trans = false; 853 need_wapbl = (!pagedaemon && vp->v_mount && vp->v_mount->mnt_wapbl && 854 (origflags & PGO_JOURNALLOCKED) == 0); 855 856 retry: 857 modified = false; 858 flags = origflags; 859 KASSERT((vp->v_iflag & VI_ONWORKLST) != 0 || 860 (vp->v_iflag & VI_WRMAPDIRTY) == 0); 861 if (uobj->uo_npages == 0) { 862 if (vp->v_iflag & VI_ONWORKLST) { 863 vp->v_iflag &= ~VI_WRMAPDIRTY; 864 if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL) 865 vn_syncer_remove_from_worklist(vp); 866 } 867 if (has_trans) { 868 if (need_wapbl) 869 WAPBL_END(vp->v_mount); 870 fstrans_done(vp->v_mount); 871 } 872 mutex_exit(slock); 873 return (0); 874 } 875 876 /* 877 * the vnode has pages, set up to process the request. 878 */ 879 880 if (!has_trans && (flags & PGO_CLEANIT) != 0) { 881 mutex_exit(slock); 882 if (pagedaemon) { 883 error = fstrans_start_nowait(vp->v_mount, FSTRANS_LAZY); 884 if (error) 885 return error; 886 } else 887 fstrans_start(vp->v_mount, FSTRANS_LAZY); 888 if (need_wapbl) { 889 error = WAPBL_BEGIN(vp->v_mount); 890 if (error) { 891 fstrans_done(vp->v_mount); 892 return error; 893 } 894 } 895 has_trans = true; 896 mutex_enter(slock); 897 goto retry; 898 } 899 900 error = 0; 901 wasclean = (vp->v_numoutput == 0); 902 off = startoff; 903 if (endoff == 0 || flags & PGO_ALLPAGES) { 904 endoff = trunc_page(LLONG_MAX); 905 } 906 by_list = (uobj->uo_npages <= 907 ((endoff - startoff) >> PAGE_SHIFT) * UVM_PAGE_TREE_PENALTY); 908 909 /* 910 * if this vnode is known not to have dirty pages, 911 * don't bother to clean it out. 912 */ 913 914 if ((vp->v_iflag & VI_ONWORKLST) == 0) { 915 #if !defined(DEBUG) 916 if ((flags & (PGO_FREE|PGO_DEACTIVATE)) == 0) { 917 goto skip_scan; 918 } 919 #endif /* !defined(DEBUG) */ 920 flags &= ~PGO_CLEANIT; 921 } 922 923 /* 924 * start the loop. when scanning by list, hold the last page 925 * in the list before we start. pages allocated after we start 926 * will be added to the end of the list, so we can stop at the 927 * current last page. 928 */ 929 930 cleanall = (flags & PGO_CLEANIT) != 0 && wasclean && 931 startoff == 0 && endoff == trunc_page(LLONG_MAX) && 932 (vp->v_iflag & VI_ONWORKLST) != 0; 933 dirtygen = gp->g_dirtygen; 934 freeflag = pagedaemon ? PG_PAGEOUT : PG_RELEASED; 935 if (by_list) { 936 curmp.flags = PG_MARKER; 937 endmp.flags = PG_MARKER; 938 pg = TAILQ_FIRST(&uobj->memq); 939 TAILQ_INSERT_TAIL(&uobj->memq, &endmp, listq.queue); 940 } else { 941 pg = uvm_pagelookup(uobj, off); 942 } 943 nextpg = NULL; 944 while (by_list || off < endoff) { 945 946 /* 947 * if the current page is not interesting, move on to the next. 948 */ 949 950 KASSERT(pg == NULL || pg->uobject == uobj || 951 (pg->flags & PG_MARKER) != 0); 952 KASSERT(pg == NULL || 953 (pg->flags & (PG_RELEASED|PG_PAGEOUT)) == 0 || 954 (pg->flags & (PG_BUSY|PG_MARKER)) != 0); 955 if (by_list) { 956 if (pg == &endmp) { 957 break; 958 } 959 if (pg->flags & PG_MARKER) { 960 pg = TAILQ_NEXT(pg, listq.queue); 961 continue; 962 } 963 if (pg->offset < startoff || pg->offset >= endoff || 964 pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 965 if (pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 966 wasclean = false; 967 } 968 pg = TAILQ_NEXT(pg, listq.queue); 969 continue; 970 } 971 off = pg->offset; 972 } else if (pg == NULL || pg->flags & (PG_RELEASED|PG_PAGEOUT)) { 973 if (pg != NULL) { 974 wasclean = false; 975 } 976 off += PAGE_SIZE; 977 if (off < endoff) { 978 pg = uvm_pagelookup(uobj, off); 979 } 980 continue; 981 } 982 983 /* 984 * if the current page needs to be cleaned and it's busy, 985 * wait for it to become unbusy. 986 */ 987 988 yld = (l->l_cpu->ci_schedstate.spc_flags & 989 SPCF_SHOULDYIELD) && !pagedaemon; 990 if (pg->flags & PG_BUSY || yld) { 991 UVMHIST_LOG(ubchist, "busy %p", pg,0,0,0); 992 if (flags & PGO_BUSYFAIL && pg->flags & PG_BUSY) { 993 UVMHIST_LOG(ubchist, "busyfail %p", pg, 0,0,0); 994 error = EDEADLK; 995 if (busypg != NULL) 996 *busypg = pg; 997 break; 998 } 999 if (pagedaemon) { 1000 /* 1001 * someone has taken the page while we 1002 * dropped the lock for fstrans_start. 1003 */ 1004 break; 1005 } 1006 if (by_list) { 1007 TAILQ_INSERT_BEFORE(pg, &curmp, listq.queue); 1008 UVMHIST_LOG(ubchist, "curmp next %p", 1009 TAILQ_NEXT(&curmp, listq.queue), 0,0,0); 1010 } 1011 if (yld) { 1012 mutex_exit(slock); 1013 preempt(); 1014 mutex_enter(slock); 1015 } else { 1016 pg->flags |= PG_WANTED; 1017 UVM_UNLOCK_AND_WAIT(pg, slock, 0, "genput", 0); 1018 mutex_enter(slock); 1019 } 1020 if (by_list) { 1021 UVMHIST_LOG(ubchist, "after next %p", 1022 TAILQ_NEXT(&curmp, listq.queue), 0,0,0); 1023 pg = TAILQ_NEXT(&curmp, listq.queue); 1024 TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue); 1025 } else { 1026 pg = uvm_pagelookup(uobj, off); 1027 } 1028 continue; 1029 } 1030 1031 /* 1032 * if we're freeing, remove all mappings of the page now. 1033 * if we're cleaning, check if the page is needs to be cleaned. 1034 */ 1035 1036 if (flags & PGO_FREE) { 1037 pmap_page_protect(pg, VM_PROT_NONE); 1038 } else if (flags & PGO_CLEANIT) { 1039 1040 /* 1041 * if we still have some hope to pull this vnode off 1042 * from the syncer queue, write-protect the page. 1043 */ 1044 1045 if (cleanall && wasclean && 1046 gp->g_dirtygen == dirtygen) { 1047 1048 /* 1049 * uobj pages get wired only by uvm_fault 1050 * where uobj is locked. 1051 */ 1052 1053 if (pg->wire_count == 0) { 1054 pmap_page_protect(pg, 1055 VM_PROT_READ|VM_PROT_EXECUTE); 1056 } else { 1057 cleanall = false; 1058 } 1059 } 1060 } 1061 1062 if (flags & PGO_CLEANIT) { 1063 needs_clean = pmap_clear_modify(pg) || 1064 (pg->flags & PG_CLEAN) == 0; 1065 pg->flags |= PG_CLEAN; 1066 } else { 1067 needs_clean = false; 1068 } 1069 1070 /* 1071 * if we're cleaning, build a cluster. 1072 * the cluster will consist of pages which are currently dirty, 1073 * but they will be returned to us marked clean. 1074 * if not cleaning, just operate on the one page. 1075 */ 1076 1077 if (needs_clean) { 1078 KDASSERT((vp->v_iflag & VI_ONWORKLST)); 1079 wasclean = false; 1080 memset(pgs, 0, sizeof(pgs)); 1081 pg->flags |= PG_BUSY; 1082 UVM_PAGE_OWN(pg, "genfs_putpages"); 1083 1084 /* 1085 * first look backward. 1086 */ 1087 1088 npages = MIN(maxpages >> 1, off >> PAGE_SHIFT); 1089 nback = npages; 1090 uvn_findpages(uobj, off - PAGE_SIZE, &nback, &pgs[0], 1091 UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY|UFP_BACKWARD); 1092 if (nback) { 1093 memmove(&pgs[0], &pgs[npages - nback], 1094 nback * sizeof(pgs[0])); 1095 if (npages - nback < nback) 1096 memset(&pgs[nback], 0, 1097 (npages - nback) * sizeof(pgs[0])); 1098 else 1099 memset(&pgs[npages - nback], 0, 1100 nback * sizeof(pgs[0])); 1101 } 1102 1103 /* 1104 * then plug in our page of interest. 1105 */ 1106 1107 pgs[nback] = pg; 1108 1109 /* 1110 * then look forward to fill in the remaining space in 1111 * the array of pages. 1112 */ 1113 1114 npages = maxpages - nback - 1; 1115 uvn_findpages(uobj, off + PAGE_SIZE, &npages, 1116 &pgs[nback + 1], 1117 UFP_NOWAIT|UFP_NOALLOC|UFP_DIRTYONLY); 1118 npages += nback + 1; 1119 } else { 1120 pgs[0] = pg; 1121 npages = 1; 1122 nback = 0; 1123 } 1124 1125 /* 1126 * apply FREE or DEACTIVATE options if requested. 1127 */ 1128 1129 if (flags & (PGO_DEACTIVATE|PGO_FREE)) { 1130 mutex_enter(&uvm_pageqlock); 1131 } 1132 for (i = 0; i < npages; i++) { 1133 tpg = pgs[i]; 1134 KASSERT(tpg->uobject == uobj); 1135 if (by_list && tpg == TAILQ_NEXT(pg, listq.queue)) 1136 pg = tpg; 1137 if (tpg->offset < startoff || tpg->offset >= endoff) 1138 continue; 1139 if (flags & PGO_DEACTIVATE && tpg->wire_count == 0) { 1140 uvm_pagedeactivate(tpg); 1141 } else if (flags & PGO_FREE) { 1142 pmap_page_protect(tpg, VM_PROT_NONE); 1143 if (tpg->flags & PG_BUSY) { 1144 tpg->flags |= freeflag; 1145 if (pagedaemon) { 1146 uvm_pageout_start(1); 1147 uvm_pagedequeue(tpg); 1148 } 1149 } else { 1150 1151 /* 1152 * ``page is not busy'' 1153 * implies that npages is 1 1154 * and needs_clean is false. 1155 */ 1156 1157 nextpg = TAILQ_NEXT(tpg, listq.queue); 1158 uvm_pagefree(tpg); 1159 if (pagedaemon) 1160 uvmexp.pdfreed++; 1161 } 1162 } 1163 } 1164 if (flags & (PGO_DEACTIVATE|PGO_FREE)) { 1165 mutex_exit(&uvm_pageqlock); 1166 } 1167 if (needs_clean) { 1168 modified = true; 1169 1170 /* 1171 * start the i/o. if we're traversing by list, 1172 * keep our place in the list with a marker page. 1173 */ 1174 1175 if (by_list) { 1176 TAILQ_INSERT_AFTER(&uobj->memq, pg, &curmp, 1177 listq.queue); 1178 } 1179 mutex_exit(slock); 1180 error = GOP_WRITE(vp, pgs, npages, flags); 1181 mutex_enter(slock); 1182 if (by_list) { 1183 pg = TAILQ_NEXT(&curmp, listq.queue); 1184 TAILQ_REMOVE(&uobj->memq, &curmp, listq.queue); 1185 } 1186 if (error) { 1187 break; 1188 } 1189 if (by_list) { 1190 continue; 1191 } 1192 } 1193 1194 /* 1195 * find the next page and continue if there was no error. 1196 */ 1197 1198 if (by_list) { 1199 if (nextpg) { 1200 pg = nextpg; 1201 nextpg = NULL; 1202 } else { 1203 pg = TAILQ_NEXT(pg, listq.queue); 1204 } 1205 } else { 1206 off += (npages - nback) << PAGE_SHIFT; 1207 if (off < endoff) { 1208 pg = uvm_pagelookup(uobj, off); 1209 } 1210 } 1211 } 1212 if (by_list) { 1213 TAILQ_REMOVE(&uobj->memq, &endmp, listq.queue); 1214 } 1215 1216 if (modified && (vp->v_iflag & VI_WRMAPDIRTY) != 0 && 1217 (vp->v_type != VBLK || 1218 (vp->v_mount->mnt_flag & MNT_NODEVMTIME) == 0)) { 1219 GOP_MARKUPDATE(vp, GOP_UPDATE_MODIFIED); 1220 } 1221 1222 /* 1223 * if we're cleaning and there was nothing to clean, 1224 * take us off the syncer list. if we started any i/o 1225 * and we're doing sync i/o, wait for all writes to finish. 1226 */ 1227 1228 if (cleanall && wasclean && gp->g_dirtygen == dirtygen && 1229 (vp->v_iflag & VI_ONWORKLST) != 0) { 1230 #if defined(DEBUG) 1231 TAILQ_FOREACH(pg, &uobj->memq, listq.queue) { 1232 if ((pg->flags & (PG_FAKE | PG_MARKER)) != 0) { 1233 continue; 1234 } 1235 if ((pg->flags & PG_CLEAN) == 0) { 1236 printf("%s: %p: !CLEAN\n", __func__, pg); 1237 } 1238 if (pmap_is_modified(pg)) { 1239 printf("%s: %p: modified\n", __func__, pg); 1240 } 1241 } 1242 #endif /* defined(DEBUG) */ 1243 vp->v_iflag &= ~VI_WRMAPDIRTY; 1244 if (LIST_FIRST(&vp->v_dirtyblkhd) == NULL) 1245 vn_syncer_remove_from_worklist(vp); 1246 } 1247 1248 #if !defined(DEBUG) 1249 skip_scan: 1250 #endif /* !defined(DEBUG) */ 1251 1252 /* Wait for output to complete. */ 1253 if (!wasclean && !async && vp->v_numoutput != 0) { 1254 while (vp->v_numoutput != 0) 1255 cv_wait(&vp->v_cv, slock); 1256 } 1257 onworklst = (vp->v_iflag & VI_ONWORKLST) != 0; 1258 mutex_exit(slock); 1259 1260 if ((flags & PGO_RECLAIM) != 0 && onworklst) { 1261 /* 1262 * in the case of PGO_RECLAIM, ensure to make the vnode clean. 1263 * retrying is not a big deal because, in many cases, 1264 * uobj->uo_npages is already 0 here. 1265 */ 1266 mutex_enter(slock); 1267 goto retry; 1268 } 1269 1270 if (has_trans) { 1271 if (need_wapbl) 1272 WAPBL_END(vp->v_mount); 1273 fstrans_done(vp->v_mount); 1274 } 1275 1276 return (error); 1277 } 1278 1279 int 1280 genfs_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, int flags) 1281 { 1282 off_t off; 1283 vaddr_t kva; 1284 size_t len; 1285 int error; 1286 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1287 1288 UVMHIST_LOG(ubchist, "vp %p pgs %p npages %d flags 0x%x", 1289 vp, pgs, npages, flags); 1290 1291 off = pgs[0]->offset; 1292 kva = uvm_pagermapin(pgs, npages, 1293 UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK); 1294 len = npages << PAGE_SHIFT; 1295 1296 error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE, 1297 uvm_aio_biodone); 1298 1299 return error; 1300 } 1301 1302 int 1303 genfs_gop_write_rwmap(struct vnode *vp, struct vm_page **pgs, int npages, int flags) 1304 { 1305 off_t off; 1306 vaddr_t kva; 1307 size_t len; 1308 int error; 1309 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1310 1311 UVMHIST_LOG(ubchist, "vp %p pgs %p npages %d flags 0x%x", 1312 vp, pgs, npages, flags); 1313 1314 off = pgs[0]->offset; 1315 kva = uvm_pagermapin(pgs, npages, 1316 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 1317 len = npages << PAGE_SHIFT; 1318 1319 error = genfs_do_io(vp, off, kva, len, flags, UIO_WRITE, 1320 uvm_aio_biodone); 1321 1322 return error; 1323 } 1324 1325 /* 1326 * Backend routine for doing I/O to vnode pages. Pages are already locked 1327 * and mapped into kernel memory. Here we just look up the underlying 1328 * device block addresses and call the strategy routine. 1329 */ 1330 1331 static int 1332 genfs_do_io(struct vnode *vp, off_t off, vaddr_t kva, size_t len, int flags, 1333 enum uio_rw rw, void (*iodone)(struct buf *)) 1334 { 1335 int s, error; 1336 int fs_bshift, dev_bshift; 1337 off_t eof, offset, startoffset; 1338 size_t bytes, iobytes, skipbytes; 1339 struct buf *mbp, *bp; 1340 const bool async = (flags & PGO_SYNCIO) == 0; 1341 const bool lazy = (flags & PGO_LAZY) == 0; 1342 const bool iowrite = rw == UIO_WRITE; 1343 const int brw = iowrite ? B_WRITE : B_READ; 1344 UVMHIST_FUNC(__func__); UVMHIST_CALLED(ubchist); 1345 1346 UVMHIST_LOG(ubchist, "vp %p kva %p len 0x%x flags 0x%x", 1347 vp, kva, len, flags); 1348 1349 KASSERT(vp->v_size <= vp->v_writesize); 1350 GOP_SIZE(vp, vp->v_writesize, &eof, 0); 1351 if (vp->v_type != VBLK) { 1352 fs_bshift = vp->v_mount->mnt_fs_bshift; 1353 dev_bshift = vp->v_mount->mnt_dev_bshift; 1354 } else { 1355 fs_bshift = DEV_BSHIFT; 1356 dev_bshift = DEV_BSHIFT; 1357 } 1358 error = 0; 1359 startoffset = off; 1360 bytes = MIN(len, eof - startoffset); 1361 skipbytes = 0; 1362 KASSERT(bytes != 0); 1363 1364 if (iowrite) { 1365 mutex_enter(vp->v_interlock); 1366 vp->v_numoutput += 2; 1367 mutex_exit(vp->v_interlock); 1368 } 1369 mbp = getiobuf(vp, true); 1370 UVMHIST_LOG(ubchist, "vp %p mbp %p num now %d bytes 0x%x", 1371 vp, mbp, vp->v_numoutput, bytes); 1372 mbp->b_bufsize = len; 1373 mbp->b_data = (void *)kva; 1374 mbp->b_resid = mbp->b_bcount = bytes; 1375 mbp->b_cflags = BC_BUSY | BC_AGE; 1376 if (async) { 1377 mbp->b_flags = brw | B_ASYNC; 1378 mbp->b_iodone = iodone; 1379 } else { 1380 mbp->b_flags = brw; 1381 mbp->b_iodone = NULL; 1382 } 1383 if (curlwp == uvm.pagedaemon_lwp) 1384 BIO_SETPRIO(mbp, BPRIO_TIMELIMITED); 1385 else if (async || lazy) 1386 BIO_SETPRIO(mbp, BPRIO_TIMENONCRITICAL); 1387 else 1388 BIO_SETPRIO(mbp, BPRIO_TIMECRITICAL); 1389 1390 bp = NULL; 1391 for (offset = startoffset; 1392 bytes > 0; 1393 offset += iobytes, bytes -= iobytes) { 1394 int run; 1395 daddr_t lbn, blkno; 1396 struct vnode *devvp; 1397 1398 /* 1399 * bmap the file to find out the blkno to read from and 1400 * how much we can read in one i/o. if bmap returns an error, 1401 * skip the rest of the top-level i/o. 1402 */ 1403 1404 lbn = offset >> fs_bshift; 1405 error = VOP_BMAP(vp, lbn, &devvp, &blkno, &run); 1406 if (error) { 1407 UVMHIST_LOG(ubchist, "VOP_BMAP lbn 0x%x -> %d\n", 1408 lbn,error,0,0); 1409 skipbytes += bytes; 1410 bytes = 0; 1411 goto loopdone; 1412 } 1413 1414 /* 1415 * see how many pages can be read with this i/o. 1416 * reduce the i/o size if necessary to avoid 1417 * overwriting pages with valid data. 1418 */ 1419 1420 iobytes = MIN((((off_t)lbn + 1 + run) << fs_bshift) - offset, 1421 bytes); 1422 1423 /* 1424 * if this block isn't allocated, zero it instead of 1425 * reading it. unless we are going to allocate blocks, 1426 * mark the pages we zeroed PG_RDONLY. 1427 */ 1428 1429 if (blkno == (daddr_t)-1) { 1430 if (!iowrite) { 1431 memset((char *)kva + (offset - startoffset), 0, 1432 iobytes); 1433 } 1434 skipbytes += iobytes; 1435 continue; 1436 } 1437 1438 /* 1439 * allocate a sub-buf for this piece of the i/o 1440 * (or just use mbp if there's only 1 piece), 1441 * and start it going. 1442 */ 1443 1444 if (offset == startoffset && iobytes == bytes) { 1445 bp = mbp; 1446 } else { 1447 UVMHIST_LOG(ubchist, "vp %p bp %p num now %d", 1448 vp, bp, vp->v_numoutput, 0); 1449 bp = getiobuf(vp, true); 1450 nestiobuf_setup(mbp, bp, offset - startoffset, iobytes); 1451 } 1452 bp->b_lblkno = 0; 1453 1454 /* adjust physical blkno for partial blocks */ 1455 bp->b_blkno = blkno + ((offset - ((off_t)lbn << fs_bshift)) >> 1456 dev_bshift); 1457 1458 UVMHIST_LOG(ubchist, 1459 "bp %p offset 0x%x bcount 0x%x blkno 0x%x", 1460 bp, offset, bp->b_bcount, bp->b_blkno); 1461 1462 VOP_STRATEGY(devvp, bp); 1463 } 1464 1465 loopdone: 1466 if (skipbytes) { 1467 UVMHIST_LOG(ubchist, "skipbytes %d", skipbytes, 0,0,0); 1468 } 1469 nestiobuf_done(mbp, skipbytes, error); 1470 if (async) { 1471 UVMHIST_LOG(ubchist, "returning 0 (async)", 0,0,0,0); 1472 return (0); 1473 } 1474 UVMHIST_LOG(ubchist, "waiting for mbp %p", mbp,0,0,0); 1475 error = biowait(mbp); 1476 s = splbio(); 1477 (*iodone)(mbp); 1478 splx(s); 1479 UVMHIST_LOG(ubchist, "returning, error %d", error,0,0,0); 1480 return (error); 1481 } 1482 1483 int 1484 genfs_compat_getpages(void *v) 1485 { 1486 struct vop_getpages_args /* { 1487 struct vnode *a_vp; 1488 voff_t a_offset; 1489 struct vm_page **a_m; 1490 int *a_count; 1491 int a_centeridx; 1492 vm_prot_t a_access_type; 1493 int a_advice; 1494 int a_flags; 1495 } */ *ap = v; 1496 1497 off_t origoffset; 1498 struct vnode *vp = ap->a_vp; 1499 struct uvm_object *uobj = &vp->v_uobj; 1500 struct vm_page *pg, **pgs; 1501 vaddr_t kva; 1502 int i, error, orignpages, npages; 1503 struct iovec iov; 1504 struct uio uio; 1505 kauth_cred_t cred = curlwp->l_cred; 1506 const bool memwrite = (ap->a_access_type & VM_PROT_WRITE) != 0; 1507 1508 error = 0; 1509 origoffset = ap->a_offset; 1510 orignpages = *ap->a_count; 1511 pgs = ap->a_m; 1512 1513 if (ap->a_flags & PGO_LOCKED) { 1514 uvn_findpages(uobj, origoffset, ap->a_count, ap->a_m, 1515 UFP_NOWAIT|UFP_NOALLOC| (memwrite ? UFP_NORDONLY : 0)); 1516 1517 error = ap->a_m[ap->a_centeridx] == NULL ? EBUSY : 0; 1518 if (error == 0 && memwrite) { 1519 genfs_markdirty(vp); 1520 } 1521 return error; 1522 } 1523 if (origoffset + (ap->a_centeridx << PAGE_SHIFT) >= vp->v_size) { 1524 mutex_exit(uobj->vmobjlock); 1525 return EINVAL; 1526 } 1527 if ((ap->a_flags & PGO_SYNCIO) == 0) { 1528 mutex_exit(uobj->vmobjlock); 1529 return 0; 1530 } 1531 npages = orignpages; 1532 uvn_findpages(uobj, origoffset, &npages, pgs, UFP_ALL); 1533 mutex_exit(uobj->vmobjlock); 1534 kva = uvm_pagermapin(pgs, npages, 1535 UVMPAGER_MAPIN_READ | UVMPAGER_MAPIN_WAITOK); 1536 for (i = 0; i < npages; i++) { 1537 pg = pgs[i]; 1538 if ((pg->flags & PG_FAKE) == 0) { 1539 continue; 1540 } 1541 iov.iov_base = (char *)kva + (i << PAGE_SHIFT); 1542 iov.iov_len = PAGE_SIZE; 1543 uio.uio_iov = &iov; 1544 uio.uio_iovcnt = 1; 1545 uio.uio_offset = origoffset + (i << PAGE_SHIFT); 1546 uio.uio_rw = UIO_READ; 1547 uio.uio_resid = PAGE_SIZE; 1548 UIO_SETUP_SYSSPACE(&uio); 1549 /* XXX vn_lock */ 1550 error = VOP_READ(vp, &uio, 0, cred); 1551 if (error) { 1552 break; 1553 } 1554 if (uio.uio_resid) { 1555 memset(iov.iov_base, 0, uio.uio_resid); 1556 } 1557 } 1558 uvm_pagermapout(kva, npages); 1559 mutex_enter(uobj->vmobjlock); 1560 mutex_enter(&uvm_pageqlock); 1561 for (i = 0; i < npages; i++) { 1562 pg = pgs[i]; 1563 if (error && (pg->flags & PG_FAKE) != 0) { 1564 pg->flags |= PG_RELEASED; 1565 } else { 1566 pmap_clear_modify(pg); 1567 uvm_pageactivate(pg); 1568 } 1569 } 1570 if (error) { 1571 uvm_page_unbusy(pgs, npages); 1572 } 1573 mutex_exit(&uvm_pageqlock); 1574 if (error == 0 && memwrite) { 1575 genfs_markdirty(vp); 1576 } 1577 mutex_exit(uobj->vmobjlock); 1578 return error; 1579 } 1580 1581 int 1582 genfs_compat_gop_write(struct vnode *vp, struct vm_page **pgs, int npages, 1583 int flags) 1584 { 1585 off_t offset; 1586 struct iovec iov; 1587 struct uio uio; 1588 kauth_cred_t cred = curlwp->l_cred; 1589 struct buf *bp; 1590 vaddr_t kva; 1591 int error; 1592 1593 offset = pgs[0]->offset; 1594 kva = uvm_pagermapin(pgs, npages, 1595 UVMPAGER_MAPIN_WRITE | UVMPAGER_MAPIN_WAITOK); 1596 1597 iov.iov_base = (void *)kva; 1598 iov.iov_len = npages << PAGE_SHIFT; 1599 uio.uio_iov = &iov; 1600 uio.uio_iovcnt = 1; 1601 uio.uio_offset = offset; 1602 uio.uio_rw = UIO_WRITE; 1603 uio.uio_resid = npages << PAGE_SHIFT; 1604 UIO_SETUP_SYSSPACE(&uio); 1605 /* XXX vn_lock */ 1606 error = VOP_WRITE(vp, &uio, 0, cred); 1607 1608 mutex_enter(vp->v_interlock); 1609 vp->v_numoutput++; 1610 mutex_exit(vp->v_interlock); 1611 1612 bp = getiobuf(vp, true); 1613 bp->b_cflags = BC_BUSY | BC_AGE; 1614 bp->b_lblkno = offset >> vp->v_mount->mnt_fs_bshift; 1615 bp->b_data = (char *)kva; 1616 bp->b_bcount = npages << PAGE_SHIFT; 1617 bp->b_bufsize = npages << PAGE_SHIFT; 1618 bp->b_resid = 0; 1619 bp->b_error = error; 1620 uvm_aio_aiodone(bp); 1621 return (error); 1622 } 1623 1624 /* 1625 * Process a uio using direct I/O. If we reach a part of the request 1626 * which cannot be processed in this fashion for some reason, just return. 1627 * The caller must handle some additional part of the request using 1628 * buffered I/O before trying direct I/O again. 1629 */ 1630 1631 void 1632 genfs_directio(struct vnode *vp, struct uio *uio, int ioflag) 1633 { 1634 struct vmspace *vs; 1635 struct iovec *iov; 1636 vaddr_t va; 1637 size_t len; 1638 const int mask = DEV_BSIZE - 1; 1639 int error; 1640 bool need_wapbl = (vp->v_mount && vp->v_mount->mnt_wapbl && 1641 (ioflag & IO_JOURNALLOCKED) == 0); 1642 1643 /* 1644 * We only support direct I/O to user space for now. 1645 */ 1646 1647 if (VMSPACE_IS_KERNEL_P(uio->uio_vmspace)) { 1648 return; 1649 } 1650 1651 /* 1652 * If the vnode is mapped, we would need to get the getpages lock 1653 * to stabilize the bmap, but then we would get into trouble while 1654 * locking the pages if the pages belong to this same vnode (or a 1655 * multi-vnode cascade to the same effect). Just fall back to 1656 * buffered I/O if the vnode is mapped to avoid this mess. 1657 */ 1658 1659 if (vp->v_vflag & VV_MAPPED) { 1660 return; 1661 } 1662 1663 if (need_wapbl) { 1664 error = WAPBL_BEGIN(vp->v_mount); 1665 if (error) 1666 return; 1667 } 1668 1669 /* 1670 * Do as much of the uio as possible with direct I/O. 1671 */ 1672 1673 vs = uio->uio_vmspace; 1674 while (uio->uio_resid) { 1675 iov = uio->uio_iov; 1676 if (iov->iov_len == 0) { 1677 uio->uio_iov++; 1678 uio->uio_iovcnt--; 1679 continue; 1680 } 1681 va = (vaddr_t)iov->iov_base; 1682 len = MIN(iov->iov_len, genfs_maxdio); 1683 len &= ~mask; 1684 1685 /* 1686 * If the next chunk is smaller than DEV_BSIZE or extends past 1687 * the current EOF, then fall back to buffered I/O. 1688 */ 1689 1690 if (len == 0 || uio->uio_offset + len > vp->v_size) { 1691 break; 1692 } 1693 1694 /* 1695 * Check alignment. The file offset must be at least 1696 * sector-aligned. The exact constraint on memory alignment 1697 * is very hardware-dependent, but requiring sector-aligned 1698 * addresses there too is safe. 1699 */ 1700 1701 if (uio->uio_offset & mask || va & mask) { 1702 break; 1703 } 1704 error = genfs_do_directio(vs, va, len, vp, uio->uio_offset, 1705 uio->uio_rw); 1706 if (error) { 1707 break; 1708 } 1709 iov->iov_base = (char *)iov->iov_base + len; 1710 iov->iov_len -= len; 1711 uio->uio_offset += len; 1712 uio->uio_resid -= len; 1713 } 1714 1715 if (need_wapbl) 1716 WAPBL_END(vp->v_mount); 1717 } 1718 1719 /* 1720 * Iodone routine for direct I/O. We don't do much here since the request is 1721 * always synchronous, so the caller will do most of the work after biowait(). 1722 */ 1723 1724 static void 1725 genfs_dio_iodone(struct buf *bp) 1726 { 1727 1728 KASSERT((bp->b_flags & B_ASYNC) == 0); 1729 if ((bp->b_flags & B_READ) == 0 && (bp->b_cflags & BC_AGE) != 0) { 1730 mutex_enter(bp->b_objlock); 1731 vwakeup(bp); 1732 mutex_exit(bp->b_objlock); 1733 } 1734 putiobuf(bp); 1735 } 1736 1737 /* 1738 * Process one chunk of a direct I/O request. 1739 */ 1740 1741 static int 1742 genfs_do_directio(struct vmspace *vs, vaddr_t uva, size_t len, struct vnode *vp, 1743 off_t off, enum uio_rw rw) 1744 { 1745 struct vm_map *map; 1746 struct pmap *upm, *kpm __unused; 1747 size_t klen = round_page(uva + len) - trunc_page(uva); 1748 off_t spoff, epoff; 1749 vaddr_t kva, puva; 1750 paddr_t pa; 1751 vm_prot_t prot; 1752 int error, rv __diagused, poff, koff; 1753 const int pgoflags = PGO_CLEANIT | PGO_SYNCIO | PGO_JOURNALLOCKED | 1754 (rw == UIO_WRITE ? PGO_FREE : 0); 1755 1756 /* 1757 * For writes, verify that this range of the file already has fully 1758 * allocated backing store. If there are any holes, just punt and 1759 * make the caller take the buffered write path. 1760 */ 1761 1762 if (rw == UIO_WRITE) { 1763 daddr_t lbn, elbn, blkno; 1764 int bsize, bshift, run; 1765 1766 bshift = vp->v_mount->mnt_fs_bshift; 1767 bsize = 1 << bshift; 1768 lbn = off >> bshift; 1769 elbn = (off + len + bsize - 1) >> bshift; 1770 while (lbn < elbn) { 1771 error = VOP_BMAP(vp, lbn, NULL, &blkno, &run); 1772 if (error) { 1773 return error; 1774 } 1775 if (blkno == (daddr_t)-1) { 1776 return ENOSPC; 1777 } 1778 lbn += 1 + run; 1779 } 1780 } 1781 1782 /* 1783 * Flush any cached pages for parts of the file that we're about to 1784 * access. If we're writing, invalidate pages as well. 1785 */ 1786 1787 spoff = trunc_page(off); 1788 epoff = round_page(off + len); 1789 mutex_enter(vp->v_interlock); 1790 error = VOP_PUTPAGES(vp, spoff, epoff, pgoflags); 1791 if (error) { 1792 return error; 1793 } 1794 1795 /* 1796 * Wire the user pages and remap them into kernel memory. 1797 */ 1798 1799 prot = rw == UIO_READ ? VM_PROT_READ | VM_PROT_WRITE : VM_PROT_READ; 1800 error = uvm_vslock(vs, (void *)uva, len, prot); 1801 if (error) { 1802 return error; 1803 } 1804 1805 map = &vs->vm_map; 1806 upm = vm_map_pmap(map); 1807 kpm = vm_map_pmap(kernel_map); 1808 puva = trunc_page(uva); 1809 kva = uvm_km_alloc(kernel_map, klen, atop(puva) & uvmexp.colormask, 1810 UVM_KMF_VAONLY | UVM_KMF_WAITVA | UVM_KMF_COLORMATCH); 1811 for (poff = 0; poff < klen; poff += PAGE_SIZE) { 1812 rv = pmap_extract(upm, puva + poff, &pa); 1813 KASSERT(rv); 1814 pmap_kenter_pa(kva + poff, pa, prot, PMAP_WIRED); 1815 } 1816 pmap_update(kpm); 1817 1818 /* 1819 * Do the I/O. 1820 */ 1821 1822 koff = uva - trunc_page(uva); 1823 error = genfs_do_io(vp, off, kva + koff, len, PGO_SYNCIO, rw, 1824 genfs_dio_iodone); 1825 1826 /* 1827 * Tear down the kernel mapping. 1828 */ 1829 1830 pmap_kremove(kva, klen); 1831 pmap_update(kpm); 1832 uvm_km_free(kernel_map, kva, klen, UVM_KMF_VAONLY); 1833 1834 /* 1835 * Unwire the user pages. 1836 */ 1837 1838 uvm_vsunlock(vs, (void *)uva, len); 1839 return error; 1840 } 1841