1 /* 2 * Copyright (c) 1990 University of Utah. 3 * Copyright (c) 1991 The Regents of the University of California. 4 * All rights reserved. 5 * Copyright (c) 1993, 1994 John S. Dyson 6 * Copyright (c) 1995, David Greenman 7 * 8 * This code is derived from software contributed to Berkeley by 9 * the Systems Programming Group of the University of Utah Computer 10 * Science Department. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. All advertising materials mentioning features or use of this software 21 * must display the following acknowledgement: 22 * This product includes software developed by the University of 23 * California, Berkeley and its contributors. 24 * 4. Neither the name of the University nor the names of its contributors 25 * may be used to endorse or promote products derived from this software 26 * without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 31 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 38 * SUCH DAMAGE. 39 * 40 * from: @(#)vnode_pager.c 7.5 (Berkeley) 4/20/91 41 * $FreeBSD: src/sys/vm/vnode_pager.c,v 1.116.2.7 2002/12/31 09:34:51 dillon Exp $ 42 * $DragonFly: src/sys/vm/vnode_pager.c,v 1.35 2007/07/30 21:41:30 dillon Exp $ 43 */ 44 45 /* 46 * Page to/from files (vnodes). 47 */ 48 49 /* 50 * TODO: 51 * Implement VOP_GETPAGES/PUTPAGES interface for filesystems. Will 52 * greatly re-simplify the vnode_pager. 53 */ 54 55 #include <sys/param.h> 56 #include <sys/systm.h> 57 #include <sys/kernel.h> 58 #include <sys/proc.h> 59 #include <sys/vnode.h> 60 #include <sys/mount.h> 61 #include <sys/buf.h> 62 #include <sys/vmmeter.h> 63 #include <sys/conf.h> 64 #include <sys/sfbuf.h> 65 #include <sys/thread2.h> 66 67 #include <vm/vm.h> 68 #include <vm/vm_object.h> 69 #include <vm/vm_page.h> 70 #include <vm/vm_pager.h> 71 #include <vm/vm_map.h> 72 #include <vm/vnode_pager.h> 73 #include <vm/vm_extern.h> 74 75 static off_t vnode_pager_addr (struct vnode *vp, off_t loffset, int *run); 76 static void vnode_pager_iodone (struct bio *bio); 77 static int vnode_pager_input_smlfs (vm_object_t object, vm_page_t m); 78 static int vnode_pager_input_old (vm_object_t object, vm_page_t m); 79 static void vnode_pager_dealloc (vm_object_t); 80 static int vnode_pager_getpages (vm_object_t, vm_page_t *, int, int); 81 static void vnode_pager_putpages (vm_object_t, vm_page_t *, int, boolean_t, int *); 82 static boolean_t vnode_pager_haspage (vm_object_t, vm_pindex_t, int *, int *); 83 84 struct pagerops vnodepagerops = { 85 NULL, 86 vnode_pager_alloc, 87 vnode_pager_dealloc, 88 vnode_pager_getpages, 89 vnode_pager_putpages, 90 vnode_pager_haspage, 91 NULL 92 }; 93 94 static struct krate vbadrate = { 1 }; 95 static struct krate vresrate = { 1 }; 96 97 int vnode_pbuf_freecnt = -1; /* start out unlimited */ 98 99 /* 100 * Allocate (or lookup) pager for a vnode. 101 * Handle is a vnode pointer. 102 */ 103 vm_object_t 104 vnode_pager_alloc(void *handle, off_t size, vm_prot_t prot, off_t offset) 105 { 106 vm_object_t object; 107 struct vnode *vp; 108 109 /* 110 * Pageout to vnode, no can do yet. 111 */ 112 if (handle == NULL) 113 return (NULL); 114 115 /* 116 * XXX hack - This initialization should be put somewhere else. 117 */ 118 if (vnode_pbuf_freecnt < 0) { 119 vnode_pbuf_freecnt = nswbuf / 2 + 1; 120 } 121 122 vp = (struct vnode *) handle; 123 124 /* 125 * Prevent race condition when allocating the object. This 126 * can happen with NFS vnodes since the nfsnode isn't locked. 127 */ 128 while (vp->v_flag & VOLOCK) { 129 vp->v_flag |= VOWANT; 130 tsleep(vp, 0, "vnpobj", 0); 131 } 132 vp->v_flag |= VOLOCK; 133 134 /* 135 * If the object is being terminated, wait for it to 136 * go away. 137 */ 138 while (((object = vp->v_object) != NULL) && 139 (object->flags & OBJ_DEAD)) { 140 vm_object_dead_sleep(object, "vadead"); 141 } 142 143 if (vp->v_sysref.refcnt <= 0) 144 panic("vnode_pager_alloc: no vnode reference"); 145 146 if (object == NULL) { 147 /* 148 * And an object of the appropriate size 149 */ 150 object = vm_object_allocate(OBJT_VNODE, OFF_TO_IDX(round_page(size))); 151 object->flags = 0; 152 object->handle = handle; 153 vp->v_object = object; 154 vp->v_filesize = size; 155 } else { 156 object->ref_count++; 157 if (vp->v_filesize != size) 158 kprintf("vnode_pager_alloc: Warning, filesize mismatch %lld/%lld\n", vp->v_filesize, size); 159 } 160 vref(vp); 161 162 vp->v_flag &= ~VOLOCK; 163 if (vp->v_flag & VOWANT) { 164 vp->v_flag &= ~VOWANT; 165 wakeup(vp); 166 } 167 return (object); 168 } 169 170 static void 171 vnode_pager_dealloc(vm_object_t object) 172 { 173 struct vnode *vp = object->handle; 174 175 if (vp == NULL) 176 panic("vnode_pager_dealloc: pager already dealloced"); 177 178 vm_object_pip_wait(object, "vnpdea"); 179 180 object->handle = NULL; 181 object->type = OBJT_DEAD; 182 vp->v_object = NULL; 183 vp->v_filesize = NOOFFSET; 184 vp->v_flag &= ~(VTEXT | VOBJBUF); 185 } 186 187 /* 188 * Return whether the vnode pager has the requested page. Return the 189 * number of disk-contiguous pages before and after the requested page, 190 * not including the requested page. 191 */ 192 static boolean_t 193 vnode_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, 194 int *after) 195 { 196 struct vnode *vp = object->handle; 197 off_t loffset; 198 off_t doffset; 199 int voff; 200 int bsize; 201 int error; 202 203 /* 204 * If no vp or vp is doomed or marked transparent to VM, we do not 205 * have the page. 206 */ 207 if ((vp == NULL) || (vp->v_flag & VRECLAIMED)) 208 return FALSE; 209 210 /* 211 * If filesystem no longer mounted or offset beyond end of file we do 212 * not have the page. 213 */ 214 loffset = IDX_TO_OFF(pindex); 215 216 if (vp->v_mount == NULL || loffset >= vp->v_filesize) 217 return FALSE; 218 219 bsize = vp->v_mount->mnt_stat.f_iosize; 220 voff = loffset % bsize; 221 222 error = VOP_BMAP(vp, loffset - voff, NULL, &doffset, after, before); 223 if (error) 224 return TRUE; 225 if (doffset == NOOFFSET) 226 return FALSE; 227 228 if (before) { 229 *before = (*before + voff) >> PAGE_SHIFT; 230 } 231 if (after) { 232 *after -= voff; 233 if (loffset + *after > vp->v_filesize) 234 *after = vp->v_filesize - loffset; 235 *after >>= PAGE_SHIFT; 236 if (*after < 0) 237 *after = 0; 238 } 239 return TRUE; 240 } 241 242 /* 243 * Lets the VM system know about a change in size for a file. 244 * We adjust our own internal size and flush any cached pages in 245 * the associated object that are affected by the size change. 246 * 247 * NOTE: This routine may be invoked as a result of a pager put 248 * operation (possibly at object termination time), so we must be careful. 249 * 250 * NOTE: vp->v_filesize is initialized to NOOFFSET (-1), be sure that 251 * we do not blow up on the case. nsize will always be >= 0, however. 252 */ 253 void 254 vnode_pager_setsize(struct vnode *vp, vm_ooffset_t nsize) 255 { 256 vm_pindex_t nobjsize; 257 vm_pindex_t oobjsize; 258 vm_object_t object = vp->v_object; 259 260 if (object == NULL) 261 return; 262 263 /* 264 * Hasn't changed size 265 */ 266 if (nsize == vp->v_filesize) 267 return; 268 269 /* 270 * Has changed size. Adjust the VM object's size and v_filesize 271 * before we start scanning pages to prevent new pages from being 272 * allocated during the scan. 273 */ 274 nobjsize = OFF_TO_IDX(nsize + PAGE_MASK); 275 oobjsize = object->size; 276 object->size = nobjsize; 277 278 /* 279 * File has shrunk. Toss any cached pages beyond the new EOF. 280 */ 281 if (nsize < vp->v_filesize) { 282 vp->v_filesize = nsize; 283 if (nobjsize < oobjsize) { 284 vm_object_page_remove(object, nobjsize, oobjsize, 285 FALSE); 286 } 287 /* 288 * This gets rid of garbage at the end of a page that is now 289 * only partially backed by the vnode. Since we are setting 290 * the entire page valid & clean after we are done we have 291 * to be sure that the portion of the page within the file 292 * bounds is already valid. If it isn't then making it 293 * valid would create a corrupt block. 294 */ 295 if (nsize & PAGE_MASK) { 296 vm_offset_t kva; 297 vm_page_t m; 298 299 m = vm_page_lookup(object, OFF_TO_IDX(nsize)); 300 if (m && m->valid) { 301 int base = (int)nsize & PAGE_MASK; 302 int size = PAGE_SIZE - base; 303 struct sf_buf *sf; 304 305 /* 306 * Clear out partial-page garbage in case 307 * the page has been mapped. 308 */ 309 sf = sf_buf_alloc(m, SFB_CPUPRIVATE); 310 kva = sf_buf_kva(sf); 311 bzero((caddr_t)kva + base, size); 312 sf_buf_free(sf); 313 314 /* 315 * XXX work around SMP data integrity race 316 * by unmapping the page from user processes. 317 * The garbage we just cleared may be mapped 318 * to a user process running on another cpu 319 * and this code is not running through normal 320 * I/O channels which handle SMP issues for 321 * us, so unmap page to synchronize all cpus. 322 * 323 * XXX should vm_pager_unmap_page() have 324 * dealt with this? 325 */ 326 vm_page_protect(m, VM_PROT_NONE); 327 328 /* 329 * Clear out partial-page dirty bits. This 330 * has the side effect of setting the valid 331 * bits, but that is ok. There are a bunch 332 * of places in the VM system where we expected 333 * m->dirty == VM_PAGE_BITS_ALL. The file EOF 334 * case is one of them. If the page is still 335 * partially dirty, make it fully dirty. 336 * 337 * note that we do not clear out the valid 338 * bits. This would prevent bogus_page 339 * replacement from working properly. 340 */ 341 vm_page_set_validclean(m, base, size); 342 if (m->dirty != 0) 343 m->dirty = VM_PAGE_BITS_ALL; 344 } 345 } 346 } else { 347 vp->v_filesize = nsize; 348 } 349 } 350 351 void 352 vnode_pager_freepage(vm_page_t m) 353 { 354 vm_page_free(m); 355 } 356 357 /* 358 * calculate the disk byte address of specified logical byte offset. The 359 * logical offset will be block-aligned. Return the number of contiguous 360 * pages that may be read from the underlying block device in *run. If 361 * *run is non-NULL, it will be set to a value of at least 1. 362 */ 363 static off_t 364 vnode_pager_addr(struct vnode *vp, off_t loffset, int *run) 365 { 366 struct vnode *rtvp; 367 off_t doffset; 368 int bsize; 369 int error; 370 int voff; 371 372 if (loffset < 0) 373 return -1; 374 375 if (vp->v_mount == NULL) 376 return -1; 377 378 /* 379 * Align loffset to a block boundary for the BMAP, then adjust the 380 * returned disk address appropriately. 381 */ 382 bsize = vp->v_mount->mnt_stat.f_iosize; 383 voff = loffset % bsize; 384 385 /* 386 * Map the block, adjust the disk offset so it represents the 387 * passed loffset rather then the block containing loffset. 388 */ 389 error = VOP_BMAP(vp, loffset - voff, &rtvp, &doffset, run, NULL); 390 if (error || doffset == NOOFFSET) { 391 doffset = NOOFFSET; 392 } else { 393 doffset += voff; 394 395 /* 396 * When calculating *run, which is the number of pages 397 * worth of data which can be read linearly from disk, 398 * the minimum return value is 1 page. 399 */ 400 if (run) { 401 *run = (*run - voff) >> PAGE_SHIFT; 402 if (*run < 1) 403 *run = 1; 404 } 405 406 } 407 return (doffset); 408 } 409 410 /* 411 * interrupt routine for I/O completion 412 */ 413 static void 414 vnode_pager_iodone(struct bio *bio) 415 { 416 struct buf *bp = bio->bio_buf; 417 418 bp->b_cmd = BUF_CMD_DONE; 419 wakeup(bp); 420 } 421 422 /* 423 * small block file system vnode pager input 424 */ 425 static int 426 vnode_pager_input_smlfs(vm_object_t object, vm_page_t m) 427 { 428 int i; 429 struct vnode *dp, *vp; 430 struct buf *bp; 431 vm_offset_t kva; 432 struct sf_buf *sf; 433 off_t doffset; 434 vm_offset_t bsize; 435 int error = 0; 436 437 vp = object->handle; 438 if (vp->v_mount == NULL) 439 return VM_PAGER_BAD; 440 441 bsize = vp->v_mount->mnt_stat.f_iosize; 442 443 444 VOP_BMAP(vp, (off_t)0, &dp, NULL, NULL, NULL); 445 446 sf = sf_buf_alloc(m, 0); 447 kva = sf_buf_kva(sf); 448 449 for (i = 0; i < PAGE_SIZE / bsize; i++) { 450 off_t loffset; 451 452 if (vm_page_bits(i * bsize, bsize) & m->valid) 453 continue; 454 455 loffset = IDX_TO_OFF(m->pindex) + i * bsize; 456 if (loffset >= vp->v_filesize) { 457 doffset = NOOFFSET; 458 } else { 459 doffset = vnode_pager_addr(vp, loffset, NULL); 460 } 461 if (doffset != NOOFFSET) { 462 bp = getpbuf(&vnode_pbuf_freecnt); 463 464 /* build a minimal buffer header */ 465 bp->b_data = (caddr_t) kva + i * bsize; 466 bp->b_bio1.bio_done = vnode_pager_iodone; 467 bp->b_bio1.bio_offset = doffset; 468 bp->b_bcount = bsize; 469 bp->b_runningbufspace = bsize; 470 runningbufspace += bp->b_runningbufspace; 471 bp->b_cmd = BUF_CMD_READ; 472 473 /* do the input */ 474 vn_strategy(dp, &bp->b_bio1); 475 476 /* we definitely need to be at splvm here */ 477 478 crit_enter(); 479 while (bp->b_cmd != BUF_CMD_DONE) 480 tsleep(bp, 0, "vnsrd", 0); 481 crit_exit(); 482 if ((bp->b_flags & B_ERROR) != 0) 483 error = EIO; 484 485 /* 486 * free the buffer header back to the swap buffer pool 487 */ 488 relpbuf(bp, &vnode_pbuf_freecnt); 489 if (error) 490 break; 491 492 vm_page_set_validclean(m, (i * bsize) & PAGE_MASK, bsize); 493 } else { 494 vm_page_set_validclean(m, (i * bsize) & PAGE_MASK, bsize); 495 bzero((caddr_t) kva + i * bsize, bsize); 496 } 497 } 498 sf_buf_free(sf); 499 pmap_clear_modify(m); 500 vm_page_flag_clear(m, PG_ZERO); 501 if (error) { 502 return VM_PAGER_ERROR; 503 } 504 return VM_PAGER_OK; 505 506 } 507 508 509 /* 510 * old style vnode pager output routine 511 */ 512 static int 513 vnode_pager_input_old(vm_object_t object, vm_page_t m) 514 { 515 struct uio auio; 516 struct iovec aiov; 517 int error; 518 int size; 519 vm_offset_t kva; 520 struct sf_buf *sf; 521 struct vnode *vp; 522 523 error = 0; 524 vp = object->handle; 525 526 /* 527 * Return failure if beyond current EOF 528 */ 529 if (IDX_TO_OFF(m->pindex) >= vp->v_filesize) { 530 return VM_PAGER_BAD; 531 } else { 532 size = PAGE_SIZE; 533 if (IDX_TO_OFF(m->pindex) + size > vp->v_filesize) 534 size = vp->v_filesize - IDX_TO_OFF(m->pindex); 535 536 /* 537 * Allocate a kernel virtual address and initialize so that 538 * we can use VOP_READ/WRITE routines. 539 */ 540 sf = sf_buf_alloc(m, 0); 541 kva = sf_buf_kva(sf); 542 543 aiov.iov_base = (caddr_t) kva; 544 aiov.iov_len = size; 545 auio.uio_iov = &aiov; 546 auio.uio_iovcnt = 1; 547 auio.uio_offset = IDX_TO_OFF(m->pindex); 548 auio.uio_segflg = UIO_SYSSPACE; 549 auio.uio_rw = UIO_READ; 550 auio.uio_resid = size; 551 auio.uio_td = curthread; 552 553 error = VOP_READ(((struct vnode *)object->handle), 554 &auio, 0, proc0.p_ucred); 555 if (!error) { 556 int count = size - auio.uio_resid; 557 558 if (count == 0) 559 error = EINVAL; 560 else if (count != PAGE_SIZE) 561 bzero((caddr_t) kva + count, PAGE_SIZE - count); 562 } 563 sf_buf_free(sf); 564 } 565 pmap_clear_modify(m); 566 vm_page_undirty(m); 567 vm_page_flag_clear(m, PG_ZERO); 568 if (!error) 569 m->valid = VM_PAGE_BITS_ALL; 570 return error ? VM_PAGER_ERROR : VM_PAGER_OK; 571 } 572 573 /* 574 * generic vnode pager input routine 575 */ 576 577 /* 578 * EOPNOTSUPP is no longer legal. For local media VFS's that do not 579 * implement their own VOP_GETPAGES, their VOP_GETPAGES should call to 580 * vnode_pager_generic_getpages() to implement the previous behaviour. 581 * 582 * All other FS's should use the bypass to get to the local media 583 * backing vp's VOP_GETPAGES. 584 */ 585 static int 586 vnode_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage) 587 { 588 int rtval; 589 struct vnode *vp; 590 int bytes = count * PAGE_SIZE; 591 592 vp = object->handle; 593 /* 594 * XXX temporary diagnostic message to help track stale FS code, 595 * Returning EOPNOTSUPP from here may make things unhappy. 596 */ 597 rtval = VOP_GETPAGES(vp, m, bytes, reqpage, 0); 598 if (rtval == EOPNOTSUPP) { 599 kprintf("vnode_pager: *** WARNING *** stale FS getpages\n"); 600 rtval = vnode_pager_generic_getpages( vp, m, bytes, reqpage); 601 } 602 return rtval; 603 } 604 605 606 /* 607 * This is now called from local media FS's to operate against their 608 * own vnodes if they fail to implement VOP_GETPAGES. 609 */ 610 int 611 vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int bytecount, 612 int reqpage) 613 { 614 vm_object_t object; 615 vm_offset_t kva; 616 off_t foff, tfoff, nextoff; 617 int i, size, bsize, first; 618 off_t firstaddr; 619 struct vnode *dp; 620 int runpg; 621 int runend; 622 struct buf *bp; 623 int count; 624 int error = 0; 625 626 object = vp->v_object; 627 count = bytecount / PAGE_SIZE; 628 629 if (vp->v_mount == NULL) 630 return VM_PAGER_BAD; 631 632 bsize = vp->v_mount->mnt_stat.f_iosize; 633 634 /* get the UNDERLYING device for the file with VOP_BMAP() */ 635 636 /* 637 * originally, we did not check for an error return value -- assuming 638 * an fs always has a bmap entry point -- that assumption is wrong!!! 639 */ 640 foff = IDX_TO_OFF(m[reqpage]->pindex); 641 642 /* 643 * if we can't bmap, use old VOP code 644 */ 645 if (VOP_BMAP(vp, (off_t)0, &dp, NULL, NULL, NULL)) { 646 for (i = 0; i < count; i++) { 647 if (i != reqpage) { 648 vnode_pager_freepage(m[i]); 649 } 650 } 651 mycpu->gd_cnt.v_vnodein++; 652 mycpu->gd_cnt.v_vnodepgsin++; 653 return vnode_pager_input_old(object, m[reqpage]); 654 655 /* 656 * if the blocksize is smaller than a page size, then use 657 * special small filesystem code. NFS sometimes has a small 658 * blocksize, but it can handle large reads itself. 659 */ 660 } else if ((PAGE_SIZE / bsize) > 1 && 661 (vp->v_mount->mnt_stat.f_type != nfs_mount_type)) { 662 for (i = 0; i < count; i++) { 663 if (i != reqpage) { 664 vnode_pager_freepage(m[i]); 665 } 666 } 667 mycpu->gd_cnt.v_vnodein++; 668 mycpu->gd_cnt.v_vnodepgsin++; 669 return vnode_pager_input_smlfs(object, m[reqpage]); 670 } 671 672 /* 673 * If we have a completely valid page available to us, we can 674 * clean up and return. Otherwise we have to re-read the 675 * media. 676 * 677 * Note that this does not work with NFS, so NFS has its own 678 * getpages routine. The problem is that NFS can have partially 679 * valid pages associated with the buffer cache due to the piecemeal 680 * write support. If we were to fall through and re-read the media 681 * as we do here, dirty data could be lost. 682 */ 683 684 if (m[reqpage]->valid == VM_PAGE_BITS_ALL) { 685 for (i = 0; i < count; i++) { 686 if (i != reqpage) 687 vnode_pager_freepage(m[i]); 688 } 689 return VM_PAGER_OK; 690 } 691 m[reqpage]->valid = 0; 692 693 /* 694 * here on direct device I/O 695 */ 696 697 firstaddr = -1; 698 /* 699 * calculate the run that includes the required page 700 */ 701 for(first = 0, i = 0; i < count; i = runend) { 702 firstaddr = vnode_pager_addr(vp, IDX_TO_OFF(m[i]->pindex), 703 &runpg); 704 if (firstaddr == -1) { 705 if (i == reqpage && foff < vp->v_filesize) { 706 /* XXX no %qd in kernel. */ 707 panic("vnode_pager_getpages: unexpected missing page: firstaddr: %012llx, foff: 0x%012llx, v_filesize: 0x%012llx", 708 firstaddr, foff, vp->v_filesize); 709 } 710 vnode_pager_freepage(m[i]); 711 runend = i + 1; 712 first = runend; 713 continue; 714 } 715 runend = i + runpg; 716 if (runend <= reqpage) { 717 int j; 718 for (j = i; j < runend; j++) { 719 vnode_pager_freepage(m[j]); 720 } 721 } else { 722 if (runpg < (count - first)) { 723 for (i = first + runpg; i < count; i++) 724 vnode_pager_freepage(m[i]); 725 count = first + runpg; 726 } 727 break; 728 } 729 first = runend; 730 } 731 732 /* 733 * the first and last page have been calculated now, move input pages 734 * to be zero based... 735 */ 736 if (first != 0) { 737 for (i = first; i < count; i++) { 738 m[i - first] = m[i]; 739 } 740 count -= first; 741 reqpage -= first; 742 } 743 744 /* 745 * calculate the file virtual address for the transfer 746 */ 747 foff = IDX_TO_OFF(m[0]->pindex); 748 749 /* 750 * calculate the size of the transfer 751 */ 752 size = count * PAGE_SIZE; 753 if ((foff + size) > vp->v_filesize) 754 size = vp->v_filesize - foff; 755 756 /* 757 * round up physical size for real devices. 758 */ 759 if (dp->v_type == VBLK || dp->v_type == VCHR) { 760 int secmask = dp->v_rdev->si_bsize_phys - 1; 761 KASSERT(secmask < PAGE_SIZE, ("vnode_pager_generic_getpages: sector size %d too large\n", secmask + 1)); 762 size = (size + secmask) & ~secmask; 763 } 764 765 bp = getpbuf(&vnode_pbuf_freecnt); 766 kva = (vm_offset_t) bp->b_data; 767 768 /* 769 * and map the pages to be read into the kva 770 */ 771 pmap_qenter(kva, m, count); 772 773 /* build a minimal buffer header */ 774 bp->b_bio1.bio_done = vnode_pager_iodone; 775 bp->b_bio1.bio_offset = firstaddr; 776 bp->b_bcount = size; 777 bp->b_runningbufspace = size; 778 runningbufspace += bp->b_runningbufspace; 779 bp->b_cmd = BUF_CMD_READ; 780 781 mycpu->gd_cnt.v_vnodein++; 782 mycpu->gd_cnt.v_vnodepgsin += count; 783 784 /* do the input */ 785 vn_strategy(dp, &bp->b_bio1); 786 787 crit_enter(); 788 /* we definitely need to be at splvm here */ 789 790 while (bp->b_cmd != BUF_CMD_DONE) 791 tsleep(bp, 0, "vnread", 0); 792 crit_exit(); 793 if ((bp->b_flags & B_ERROR) != 0) 794 error = EIO; 795 796 if (!error) { 797 if (size != count * PAGE_SIZE) 798 bzero((caddr_t) kva + size, PAGE_SIZE * count - size); 799 } 800 pmap_qremove(kva, count); 801 802 /* 803 * free the buffer header back to the swap buffer pool 804 */ 805 relpbuf(bp, &vnode_pbuf_freecnt); 806 807 for (i = 0, tfoff = foff; i < count; i++, tfoff = nextoff) { 808 vm_page_t mt; 809 810 nextoff = tfoff + PAGE_SIZE; 811 mt = m[i]; 812 813 if (nextoff <= vp->v_filesize) { 814 /* 815 * Read filled up entire page. 816 */ 817 mt->valid = VM_PAGE_BITS_ALL; 818 vm_page_undirty(mt); /* should be an assert? XXX */ 819 pmap_clear_modify(mt); 820 } else { 821 /* 822 * Read did not fill up entire page. Since this 823 * is getpages, the page may be mapped, so we have 824 * to zero the invalid portions of the page even 825 * though we aren't setting them valid. 826 * 827 * Currently we do not set the entire page valid, 828 * we just try to clear the piece that we couldn't 829 * read. 830 */ 831 vm_page_set_validclean(mt, 0, vp->v_filesize - tfoff); 832 /* handled by vm_fault now */ 833 /* vm_page_zero_invalid(mt, FALSE); */ 834 } 835 836 vm_page_flag_clear(mt, PG_ZERO); 837 if (i != reqpage) { 838 839 /* 840 * whether or not to leave the page activated is up in 841 * the air, but we should put the page on a page queue 842 * somewhere. (it already is in the object). Result: 843 * It appears that empirical results show that 844 * deactivating pages is best. 845 */ 846 847 /* 848 * just in case someone was asking for this page we 849 * now tell them that it is ok to use 850 */ 851 if (!error) { 852 if (mt->flags & PG_WANTED) 853 vm_page_activate(mt); 854 else 855 vm_page_deactivate(mt); 856 vm_page_wakeup(mt); 857 } else { 858 vnode_pager_freepage(mt); 859 } 860 } 861 } 862 if (error) { 863 kprintf("vnode_pager_getpages: I/O read error\n"); 864 } 865 return (error ? VM_PAGER_ERROR : VM_PAGER_OK); 866 } 867 868 /* 869 * EOPNOTSUPP is no longer legal. For local media VFS's that do not 870 * implement their own VOP_PUTPAGES, their VOP_PUTPAGES should call to 871 * vnode_pager_generic_putpages() to implement the previous behaviour. 872 * 873 * All other FS's should use the bypass to get to the local media 874 * backing vp's VOP_PUTPAGES. 875 */ 876 static void 877 vnode_pager_putpages(vm_object_t object, vm_page_t *m, int count, 878 boolean_t sync, int *rtvals) 879 { 880 int rtval; 881 struct vnode *vp; 882 int bytes = count * PAGE_SIZE; 883 884 /* 885 * Force synchronous operation if we are extremely low on memory 886 * to prevent a low-memory deadlock. VOP operations often need to 887 * allocate more memory to initiate the I/O ( i.e. do a BMAP 888 * operation ). The swapper handles the case by limiting the amount 889 * of asynchronous I/O, but that sort of solution doesn't scale well 890 * for the vnode pager without a lot of work. 891 * 892 * Also, the backing vnode's iodone routine may not wake the pageout 893 * daemon up. This should be probably be addressed XXX. 894 */ 895 896 if ((vmstats.v_free_count + vmstats.v_cache_count) < vmstats.v_pageout_free_min) 897 sync |= OBJPC_SYNC; 898 899 /* 900 * Call device-specific putpages function 901 */ 902 903 vp = object->handle; 904 rtval = VOP_PUTPAGES(vp, m, bytes, sync, rtvals, 0); 905 if (rtval == EOPNOTSUPP) { 906 kprintf("vnode_pager: *** WARNING *** stale FS putpages\n"); 907 rtval = vnode_pager_generic_putpages( vp, m, bytes, sync, rtvals); 908 } 909 } 910 911 912 /* 913 * This is now called from local media FS's to operate against their 914 * own vnodes if they fail to implement VOP_PUTPAGES. 915 * 916 * This is typically called indirectly via the pageout daemon and 917 * clustering has already typically occured, so in general we ask the 918 * underlying filesystem to write the data out asynchronously rather 919 * then delayed. 920 */ 921 int 922 vnode_pager_generic_putpages(struct vnode *vp, vm_page_t *m, int bytecount, 923 int flags, int *rtvals) 924 { 925 int i; 926 vm_object_t object; 927 int count; 928 929 int maxsize, ncount; 930 vm_ooffset_t poffset; 931 struct uio auio; 932 struct iovec aiov; 933 int error; 934 int ioflags; 935 936 object = vp->v_object; 937 count = bytecount / PAGE_SIZE; 938 939 for (i = 0; i < count; i++) 940 rtvals[i] = VM_PAGER_AGAIN; 941 942 if ((int) m[0]->pindex < 0) { 943 kprintf("vnode_pager_putpages: attempt to write meta-data!!! -- 0x%lx(%x)\n", 944 (long)m[0]->pindex, m[0]->dirty); 945 rtvals[0] = VM_PAGER_BAD; 946 return VM_PAGER_BAD; 947 } 948 949 maxsize = count * PAGE_SIZE; 950 ncount = count; 951 952 poffset = IDX_TO_OFF(m[0]->pindex); 953 954 /* 955 * If the page-aligned write is larger then the actual file we 956 * have to invalidate pages occuring beyond the file EOF. However, 957 * there is an edge case where a file may not be page-aligned where 958 * the last page is partially invalid. In this case the filesystem 959 * may not properly clear the dirty bits for the entire page (which 960 * could be VM_PAGE_BITS_ALL due to the page having been mmap()d). 961 * With the page locked we are free to fix-up the dirty bits here. 962 * 963 * We do not under any circumstances truncate the valid bits, as 964 * this will screw up bogus page replacement. 965 */ 966 if (maxsize + poffset > vp->v_filesize) { 967 if (vp->v_filesize > poffset) { 968 int pgoff; 969 970 maxsize = vp->v_filesize - poffset; 971 ncount = btoc(maxsize); 972 if ((pgoff = (int)maxsize & PAGE_MASK) != 0) { 973 vm_page_clear_dirty(m[ncount - 1], pgoff, 974 PAGE_SIZE - pgoff); 975 } 976 } else { 977 maxsize = 0; 978 ncount = 0; 979 } 980 if (ncount < count) { 981 for (i = ncount; i < count; i++) { 982 rtvals[i] = VM_PAGER_BAD; 983 } 984 } 985 } 986 987 /* 988 * pageouts are already clustered, use IO_ASYNC to force a bawrite() 989 * rather then a bdwrite() to prevent paging I/O from saturating 990 * the buffer cache. Dummy-up the sequential heuristic to cause 991 * large ranges to cluster. If neither IO_SYNC or IO_ASYNC is set, 992 * the system decides how to cluster. 993 */ 994 ioflags = IO_VMIO; 995 if (flags & (VM_PAGER_PUT_SYNC | VM_PAGER_PUT_INVAL)) 996 ioflags |= IO_SYNC; 997 else if ((flags & VM_PAGER_CLUSTER_OK) == 0) 998 ioflags |= IO_ASYNC; 999 ioflags |= (flags & VM_PAGER_PUT_INVAL) ? IO_INVAL: 0; 1000 ioflags |= IO_SEQMAX << IO_SEQSHIFT; 1001 1002 aiov.iov_base = (caddr_t) 0; 1003 aiov.iov_len = maxsize; 1004 auio.uio_iov = &aiov; 1005 auio.uio_iovcnt = 1; 1006 auio.uio_offset = poffset; 1007 auio.uio_segflg = UIO_NOCOPY; 1008 auio.uio_rw = UIO_WRITE; 1009 auio.uio_resid = maxsize; 1010 auio.uio_td = NULL; 1011 error = VOP_WRITE(vp, &auio, ioflags, proc0.p_ucred); 1012 mycpu->gd_cnt.v_vnodeout++; 1013 mycpu->gd_cnt.v_vnodepgsout += ncount; 1014 1015 if (error) { 1016 krateprintf(&vbadrate, 1017 "vnode_pager_putpages: I/O error %d\n", error); 1018 } 1019 if (auio.uio_resid) { 1020 krateprintf(&vresrate, 1021 "vnode_pager_putpages: residual I/O %d at %lu\n", 1022 auio.uio_resid, (u_long)m[0]->pindex); 1023 } 1024 for (i = 0; i < ncount; i++) { 1025 rtvals[i] = VM_PAGER_OK; 1026 } 1027 return rtvals[0]; 1028 } 1029 1030 struct vnode * 1031 vnode_pager_lock(vm_object_t object) 1032 { 1033 struct thread *td = curthread; /* XXX */ 1034 int error; 1035 1036 for (; object != NULL; object = object->backing_object) { 1037 if (object->type != OBJT_VNODE) 1038 continue; 1039 if (object->flags & OBJ_DEAD) 1040 return NULL; 1041 1042 for (;;) { 1043 struct vnode *vp = object->handle; 1044 error = vget(vp, LK_SHARED | LK_RETRY | LK_CANRECURSE); 1045 if (error == 0) { 1046 if (object->handle != vp) { 1047 vput(vp); 1048 continue; 1049 } 1050 return (vp); 1051 } 1052 if ((object->flags & OBJ_DEAD) || 1053 (object->type != OBJT_VNODE)) { 1054 return NULL; 1055 } 1056 kprintf("vnode_pager_lock: vp %p error %d lockstatus %d, retrying\n", vp, error, lockstatus(&vp->v_lock, td)); 1057 tsleep(object->handle, 0, "vnpgrl", hz); 1058 } 1059 } 1060 return NULL; 1061 } 1062