1 /* $NetBSD: uvm_vnode.c,v 1.28 2000/01/28 08:02:48 chs Exp $ */ 2 3 /* 4 * Copyright (c) 1997 Charles D. Cranor and Washington University. 5 * Copyright (c) 1991, 1993 6 * The Regents of the University of California. 7 * Copyright (c) 1990 University of Utah. 8 * 9 * All rights reserved. 10 * 11 * This code is derived from software contributed to Berkeley by 12 * the Systems Programming Group of the University of Utah Computer 13 * Science Department. 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 3. All advertising materials mentioning features or use of this software 24 * must display the following acknowledgement: 25 * This product includes software developed by Charles D. Cranor, 26 * Washington University, the University of California, Berkeley and 27 * its contributors. 28 * 4. Neither the name of the University nor the names of its contributors 29 * may be used to endorse or promote products derived from this software 30 * without specific prior written permission. 31 * 32 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 33 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 34 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 35 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 36 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 37 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 38 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 39 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 40 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 41 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 42 * SUCH DAMAGE. 43 * 44 * @(#)vnode_pager.c 8.8 (Berkeley) 2/13/94 45 * from: Id: uvm_vnode.c,v 1.1.2.26 1998/02/02 20:38:07 chuck Exp 46 */ 47 48 #include "fs_nfs.h" 49 #include "opt_uvmhist.h" 50 51 /* 52 * uvm_vnode.c: the vnode pager. 53 */ 54 55 #include <sys/param.h> 56 #include <sys/systm.h> 57 #include <sys/proc.h> 58 #include <sys/malloc.h> 59 #include <sys/vnode.h> 60 #include <sys/disklabel.h> 61 #include <sys/ioctl.h> 62 #include <sys/fcntl.h> 63 #include <sys/conf.h> 64 65 #include <miscfs/specfs/specdev.h> 66 67 #include <vm/vm.h> 68 #include <vm/vm_page.h> 69 #include <vm/vm_kern.h> 70 71 #include <uvm/uvm.h> 72 #include <uvm/uvm_vnode.h> 73 74 /* 75 * private global data structure 76 * 77 * we keep a list of writeable active vnode-backed VM objects for sync op. 78 * we keep a simpleq of vnodes that are currently being sync'd. 79 */ 80 81 LIST_HEAD(uvn_list_struct, uvm_vnode); 82 static struct uvn_list_struct uvn_wlist; /* writeable uvns */ 83 static simple_lock_data_t uvn_wl_lock; /* locks uvn_wlist */ 84 85 SIMPLEQ_HEAD(uvn_sq_struct, uvm_vnode); 86 static struct uvn_sq_struct uvn_sync_q; /* sync'ing uvns */ 87 lock_data_t uvn_sync_lock; /* locks sync operation */ 88 89 /* 90 * functions 91 */ 92 93 static int uvn_asyncget __P((struct uvm_object *, vaddr_t, 94 int)); 95 struct uvm_object *uvn_attach __P((void *, vm_prot_t)); 96 static void uvn_cluster __P((struct uvm_object *, vaddr_t, 97 vaddr_t *, vaddr_t *)); 98 static void uvn_detach __P((struct uvm_object *)); 99 static boolean_t uvn_flush __P((struct uvm_object *, vaddr_t, 100 vaddr_t, int)); 101 static int uvn_get __P((struct uvm_object *, vaddr_t, 102 vm_page_t *, int *, int, 103 vm_prot_t, int, int)); 104 static void uvn_init __P((void)); 105 static int uvn_io __P((struct uvm_vnode *, vm_page_t *, 106 int, int, int)); 107 static int uvn_put __P((struct uvm_object *, vm_page_t *, 108 int, boolean_t)); 109 static void uvn_reference __P((struct uvm_object *)); 110 static boolean_t uvn_releasepg __P((struct vm_page *, 111 struct vm_page **)); 112 113 /* 114 * master pager structure 115 */ 116 117 struct uvm_pagerops uvm_vnodeops = { 118 uvn_init, 119 uvn_reference, 120 uvn_detach, 121 NULL, /* no specialized fault routine required */ 122 uvn_flush, 123 uvn_get, 124 uvn_asyncget, 125 uvn_put, 126 uvn_cluster, 127 uvm_mk_pcluster, /* use generic version of this: see uvm_pager.c */ 128 uvm_shareprot, /* !NULL: allow us in share maps */ 129 NULL, /* AIO-DONE function (not until we have asyncio) */ 130 uvn_releasepg, 131 }; 132 133 /* 134 * the ops! 135 */ 136 137 /* 138 * uvn_init 139 * 140 * init pager private data structures. 141 */ 142 143 static void 144 uvn_init() 145 { 146 147 LIST_INIT(&uvn_wlist); 148 simple_lock_init(&uvn_wl_lock); 149 /* note: uvn_sync_q init'd in uvm_vnp_sync() */ 150 lockinit(&uvn_sync_lock, PVM, "uvnsync", 0, 0); 151 } 152 153 /* 154 * uvn_attach 155 * 156 * attach a vnode structure to a VM object. if the vnode is already 157 * attached, then just bump the reference count by one and return the 158 * VM object. if not already attached, attach and return the new VM obj. 159 * the "accessprot" tells the max access the attaching thread wants to 160 * our pages. 161 * 162 * => caller must _not_ already be holding the lock on the uvm_object. 163 * => in fact, nothing should be locked so that we can sleep here. 164 * => note that uvm_object is first thing in vnode structure, so their 165 * pointers are equiv. 166 */ 167 168 struct uvm_object * 169 uvn_attach(arg, accessprot) 170 void *arg; 171 vm_prot_t accessprot; 172 { 173 struct vnode *vp = arg; 174 struct uvm_vnode *uvn = &vp->v_uvm; 175 struct vattr vattr; 176 int oldflags, result; 177 struct partinfo pi; 178 u_quad_t used_vnode_size; 179 UVMHIST_FUNC("uvn_attach"); UVMHIST_CALLED(maphist); 180 181 UVMHIST_LOG(maphist, "(vn=0x%x)", arg,0,0,0); 182 183 used_vnode_size = (u_quad_t)0; /* XXX gcc -Wuninitialized */ 184 185 /* 186 * first get a lock on the uvn. 187 */ 188 simple_lock(&uvn->u_obj.vmobjlock); 189 while (uvn->u_flags & UVM_VNODE_BLOCKED) { 190 uvn->u_flags |= UVM_VNODE_WANTED; 191 UVMHIST_LOG(maphist, " SLEEPING on blocked vn",0,0,0,0); 192 UVM_UNLOCK_AND_WAIT(uvn, &uvn->u_obj.vmobjlock, FALSE, 193 "uvn_attach", 0); 194 simple_lock(&uvn->u_obj.vmobjlock); 195 UVMHIST_LOG(maphist," WOKE UP",0,0,0,0); 196 } 197 198 /* 199 * if we're mapping a BLK device, make sure it is a disk. 200 */ 201 if (vp->v_type == VBLK && bdevsw[major(vp->v_rdev)].d_type != D_DISK) { 202 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock */ 203 UVMHIST_LOG(maphist,"<- done (VBLK not D_DISK!)", 0,0,0,0); 204 return(NULL); 205 } 206 207 /* 208 * now we have lock and uvn must not be in a blocked state. 209 * first check to see if it is already active, in which case 210 * we can bump the reference count, check to see if we need to 211 * add it to the writeable list, and then return. 212 */ 213 if (uvn->u_flags & UVM_VNODE_VALID) { /* already active? */ 214 215 /* regain VREF if we were persisting */ 216 if (uvn->u_obj.uo_refs == 0) { 217 VREF(vp); 218 UVMHIST_LOG(maphist," VREF (reclaim persisting vnode)", 219 0,0,0,0); 220 } 221 uvn->u_obj.uo_refs++; /* bump uvn ref! */ 222 223 /* check for new writeable uvn */ 224 if ((accessprot & VM_PROT_WRITE) != 0 && 225 (uvn->u_flags & UVM_VNODE_WRITEABLE) == 0) { 226 simple_lock(&uvn_wl_lock); 227 LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist); 228 simple_unlock(&uvn_wl_lock); 229 /* we are now on wlist! */ 230 uvn->u_flags |= UVM_VNODE_WRITEABLE; 231 } 232 233 /* unlock and return */ 234 simple_unlock(&uvn->u_obj.vmobjlock); 235 UVMHIST_LOG(maphist,"<- done, refcnt=%d", uvn->u_obj.uo_refs, 236 0, 0, 0); 237 return (&uvn->u_obj); 238 } 239 240 /* 241 * need to call VOP_GETATTR() to get the attributes, but that could 242 * block (due to I/O), so we want to unlock the object before calling. 243 * however, we want to keep anyone else from playing with the object 244 * while it is unlocked. to do this we set UVM_VNODE_ALOCK which 245 * prevents anyone from attaching to the vnode until we are done with 246 * it. 247 */ 248 uvn->u_flags = UVM_VNODE_ALOCK; 249 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock in case we sleep */ 250 /* XXX: curproc? */ 251 252 if (vp->v_type == VBLK) { 253 /* 254 * We could implement this as a specfs getattr call, but: 255 * 256 * (1) VOP_GETATTR() would get the file system 257 * vnode operation, not the specfs operation. 258 * 259 * (2) All we want is the size, anyhow. 260 */ 261 result = (*bdevsw[major(vp->v_rdev)].d_ioctl)(vp->v_rdev, 262 DIOCGPART, (caddr_t)&pi, FREAD, curproc); 263 if (result == 0) { 264 /* XXX should remember blocksize */ 265 used_vnode_size = (u_quad_t)pi.disklab->d_secsize * 266 (u_quad_t)pi.part->p_size; 267 } 268 } else { 269 result = VOP_GETATTR(vp, &vattr, curproc->p_ucred, curproc); 270 if (result == 0) 271 used_vnode_size = vattr.va_size; 272 } 273 274 /* relock object */ 275 simple_lock(&uvn->u_obj.vmobjlock); 276 277 if (result != 0) { 278 if (uvn->u_flags & UVM_VNODE_WANTED) 279 wakeup(uvn); 280 uvn->u_flags = 0; 281 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock */ 282 UVMHIST_LOG(maphist,"<- done (VOP_GETATTR FAILED!)", 0,0,0,0); 283 return(NULL); 284 } 285 286 /* 287 * make sure that the newsize fits within a vaddr_t 288 * XXX: need to revise addressing data types 289 */ 290 #ifdef DEBUG 291 if (vp->v_type == VBLK) 292 printf("used_vnode_size = %qu\n", (long long)used_vnode_size); 293 #endif 294 if (used_vnode_size > (vaddr_t) -PAGE_SIZE) { 295 #ifdef DEBUG 296 printf("uvn_attach: vn %p size truncated %qx->%x\n", vp, 297 (long long)used_vnode_size, -PAGE_SIZE); 298 #endif 299 used_vnode_size = (vaddr_t) -PAGE_SIZE; 300 } 301 302 /* 303 * now set up the uvn. 304 */ 305 uvn->u_obj.pgops = &uvm_vnodeops; 306 TAILQ_INIT(&uvn->u_obj.memq); 307 uvn->u_obj.uo_npages = 0; 308 uvn->u_obj.uo_refs = 1; /* just us... */ 309 oldflags = uvn->u_flags; 310 uvn->u_flags = UVM_VNODE_VALID|UVM_VNODE_CANPERSIST; 311 uvn->u_nio = 0; 312 uvn->u_size = used_vnode_size; 313 314 /* if write access, we need to add it to the wlist */ 315 if (accessprot & VM_PROT_WRITE) { 316 simple_lock(&uvn_wl_lock); 317 LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist); 318 simple_unlock(&uvn_wl_lock); 319 uvn->u_flags |= UVM_VNODE_WRITEABLE; /* we are on wlist! */ 320 } 321 322 /* 323 * add a reference to the vnode. this reference will stay as long 324 * as there is a valid mapping of the vnode. dropped when the 325 * reference count goes to zero [and we either free or persist]. 326 */ 327 VREF(vp); 328 simple_unlock(&uvn->u_obj.vmobjlock); 329 if (oldflags & UVM_VNODE_WANTED) 330 wakeup(uvn); 331 332 UVMHIST_LOG(maphist,"<- done/VREF, ret 0x%x", &uvn->u_obj,0,0,0); 333 return(&uvn->u_obj); 334 } 335 336 337 /* 338 * uvn_reference 339 * 340 * duplicate a reference to a VM object. Note that the reference 341 * count must already be at least one (the passed in reference) so 342 * there is no chance of the uvn being killed or locked out here. 343 * 344 * => caller must call with object unlocked. 345 * => caller must be using the same accessprot as was used at attach time 346 */ 347 348 349 static void 350 uvn_reference(uobj) 351 struct uvm_object *uobj; 352 { 353 #ifdef DEBUG 354 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 355 #endif 356 UVMHIST_FUNC("uvn_reference"); UVMHIST_CALLED(maphist); 357 358 simple_lock(&uobj->vmobjlock); 359 #ifdef DEBUG 360 if ((uvn->u_flags & UVM_VNODE_VALID) == 0) { 361 printf("uvn_reference: ref=%d, flags=0x%x\n", uvn->u_flags, 362 uobj->uo_refs); 363 panic("uvn_reference: invalid state"); 364 } 365 #endif 366 uobj->uo_refs++; 367 UVMHIST_LOG(maphist, "<- done (uobj=0x%x, ref = %d)", 368 uobj, uobj->uo_refs,0,0); 369 simple_unlock(&uobj->vmobjlock); 370 } 371 372 /* 373 * uvn_detach 374 * 375 * remove a reference to a VM object. 376 * 377 * => caller must call with object unlocked and map locked. 378 * => this starts the detach process, but doesn't have to finish it 379 * (async i/o could still be pending). 380 */ 381 static void 382 uvn_detach(uobj) 383 struct uvm_object *uobj; 384 { 385 struct uvm_vnode *uvn; 386 struct vnode *vp; 387 int oldflags; 388 UVMHIST_FUNC("uvn_detach"); UVMHIST_CALLED(maphist); 389 390 simple_lock(&uobj->vmobjlock); 391 392 UVMHIST_LOG(maphist," (uobj=0x%x) ref=%d", uobj,uobj->uo_refs,0,0); 393 uobj->uo_refs--; /* drop ref! */ 394 if (uobj->uo_refs) { /* still more refs */ 395 simple_unlock(&uobj->vmobjlock); 396 UVMHIST_LOG(maphist, "<- done (rc>0)", 0,0,0,0); 397 return; 398 } 399 400 /* 401 * get other pointers ... 402 */ 403 404 uvn = (struct uvm_vnode *) uobj; 405 vp = (struct vnode *) uobj; 406 407 /* 408 * clear VTEXT flag now that there are no mappings left (VTEXT is used 409 * to keep an active text file from being overwritten). 410 */ 411 vp->v_flag &= ~VTEXT; 412 413 /* 414 * we just dropped the last reference to the uvn. see if we can 415 * let it "stick around". 416 */ 417 418 if (uvn->u_flags & UVM_VNODE_CANPERSIST) { 419 /* won't block */ 420 uvn_flush(uobj, 0, 0, PGO_DEACTIVATE|PGO_ALLPAGES); 421 simple_unlock(&uobj->vmobjlock); 422 vrele(vp); /* drop vnode reference */ 423 UVMHIST_LOG(maphist,"<- done/vrele! (persist)", 0,0,0,0); 424 return; 425 } 426 427 /* 428 * its a goner! 429 */ 430 431 UVMHIST_LOG(maphist," its a goner (flushing)!", 0,0,0,0); 432 433 uvn->u_flags |= UVM_VNODE_DYING; 434 435 /* 436 * even though we may unlock in flush, no one can gain a reference 437 * to us until we clear the "dying" flag [because it blocks 438 * attaches]. we will not do that until after we've disposed of all 439 * the pages with uvn_flush(). note that before the flush the only 440 * pages that could be marked PG_BUSY are ones that are in async 441 * pageout by the daemon. (there can't be any pending "get"'s 442 * because there are no references to the object). 443 */ 444 445 (void) uvn_flush(uobj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES); 446 447 UVMHIST_LOG(maphist," its a goner (done flush)!", 0,0,0,0); 448 449 /* 450 * given the structure of this pager, the above flush request will 451 * create the following state: all the pages that were in the object 452 * have either been free'd or they are marked PG_BUSY|PG_RELEASED. 453 * the PG_BUSY bit was set either by us or the daemon for async I/O. 454 * in either case, if we have pages left we can't kill the object 455 * yet because i/o is pending. in this case we set the "relkill" 456 * flag which will cause pgo_releasepg to kill the object once all 457 * the I/O's are done [pgo_releasepg will be called from the aiodone 458 * routine or from the page daemon]. 459 */ 460 461 if (uobj->uo_npages) { /* I/O pending. iodone will free */ 462 #ifdef DEBUG 463 /* 464 * XXXCDC: very unlikely to happen until we have async i/o 465 * so print a little info message in case it does. 466 */ 467 printf("uvn_detach: vn %p has pages left after flush - " 468 "relkill mode\n", uobj); 469 #endif 470 uvn->u_flags |= UVM_VNODE_RELKILL; 471 simple_unlock(&uobj->vmobjlock); 472 UVMHIST_LOG(maphist,"<- done! (releasepg will kill obj)", 0, 0, 473 0, 0); 474 return; 475 } 476 477 /* 478 * kill object now. note that we can't be on the sync q because 479 * all references are gone. 480 */ 481 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 482 simple_lock(&uvn_wl_lock); /* protect uvn_wlist */ 483 LIST_REMOVE(uvn, u_wlist); 484 simple_unlock(&uvn_wl_lock); 485 } 486 #ifdef DIAGNOSTIC 487 if (uobj->memq.tqh_first != NULL) 488 panic("uvn_deref: vnode VM object still has pages afer " 489 "syncio/free flush"); 490 #endif 491 oldflags = uvn->u_flags; 492 uvn->u_flags = 0; 493 simple_unlock(&uobj->vmobjlock); 494 495 /* wake up any sleepers */ 496 if (oldflags & UVM_VNODE_WANTED) 497 wakeup(uvn); 498 499 /* 500 * drop our reference to the vnode. 501 */ 502 vrele(vp); 503 UVMHIST_LOG(maphist,"<- done (vrele) final", 0,0,0,0); 504 505 return; 506 } 507 508 /* 509 * uvm_vnp_terminate: external hook to clear out a vnode's VM 510 * 511 * called in two cases: 512 * [1] when a persisting vnode vm object (i.e. one with a zero reference 513 * count) needs to be freed so that a vnode can be reused. this 514 * happens under "getnewvnode" in vfs_subr.c. if the vnode from 515 * the free list is still attached (i.e. not VBAD) then vgone is 516 * called. as part of the vgone trace this should get called to 517 * free the vm object. this is the common case. 518 * [2] when a filesystem is being unmounted by force (MNT_FORCE, 519 * "umount -f") the vgone() function is called on active vnodes 520 * on the mounted file systems to kill their data (the vnodes become 521 * "dead" ones [see src/sys/miscfs/deadfs/...]). that results in a 522 * call here (even if the uvn is still in use -- i.e. has a non-zero 523 * reference count). this case happens at "umount -f" and during a 524 * "reboot/halt" operation. 525 * 526 * => the caller must XLOCK and VOP_LOCK the vnode before calling us 527 * [protects us from getting a vnode that is already in the DYING 528 * state...] 529 * => unlike uvn_detach, this function must not return until all the 530 * uvn's pages are disposed of. 531 * => in case [2] the uvn is still alive after this call, but all I/O 532 * ops will fail (due to the backing vnode now being "dead"). this 533 * will prob. kill any process using the uvn due to pgo_get failing. 534 */ 535 536 void 537 uvm_vnp_terminate(vp) 538 struct vnode *vp; 539 { 540 struct uvm_vnode *uvn = &vp->v_uvm; 541 int oldflags; 542 UVMHIST_FUNC("uvm_vnp_terminate"); UVMHIST_CALLED(maphist); 543 544 /* 545 * lock object and check if it is valid 546 */ 547 simple_lock(&uvn->u_obj.vmobjlock); 548 UVMHIST_LOG(maphist, " vp=0x%x, ref=%d, flag=0x%x", vp, 549 uvn->u_obj.uo_refs, uvn->u_flags, 0); 550 if ((uvn->u_flags & UVM_VNODE_VALID) == 0) { 551 simple_unlock(&uvn->u_obj.vmobjlock); 552 UVMHIST_LOG(maphist, "<- done (not active)", 0, 0, 0, 0); 553 return; 554 } 555 556 /* 557 * must be a valid uvn that is not already dying (because XLOCK 558 * protects us from that). the uvn can't in the the ALOCK state 559 * because it is valid, and uvn's that are in the ALOCK state haven't 560 * been marked valid yet. 561 */ 562 563 #ifdef DEBUG 564 /* 565 * debug check: are we yanking the vnode out from under our uvn? 566 */ 567 if (uvn->u_obj.uo_refs) { 568 printf("uvm_vnp_terminate(%p): terminating active vnode " 569 "(refs=%d)\n", uvn, uvn->u_obj.uo_refs); 570 } 571 #endif 572 573 /* 574 * it is possible that the uvn was detached and is in the relkill 575 * state [i.e. waiting for async i/o to finish so that releasepg can 576 * kill object]. we take over the vnode now and cancel the relkill. 577 * we want to know when the i/o is done so we can recycle right 578 * away. note that a uvn can only be in the RELKILL state if it 579 * has a zero reference count. 580 */ 581 582 if (uvn->u_flags & UVM_VNODE_RELKILL) 583 uvn->u_flags &= ~UVM_VNODE_RELKILL; /* cancel RELKILL */ 584 585 /* 586 * block the uvn by setting the dying flag, and then flush the 587 * pages. (note that flush may unlock object while doing I/O, but 588 * it will re-lock it before it returns control here). 589 * 590 * also, note that we tell I/O that we are already VOP_LOCK'd so 591 * that uvn_io doesn't attempt to VOP_LOCK again. 592 * 593 * XXXCDC: setting VNISLOCKED on an active uvn which is being terminated 594 * due to a forceful unmount might not be a good idea. maybe we 595 * need a way to pass in this info to uvn_flush through a 596 * pager-defined PGO_ constant [currently there are none]. 597 */ 598 uvn->u_flags |= UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED; 599 600 (void) uvn_flush(&uvn->u_obj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES); 601 602 /* 603 * as we just did a flush we expect all the pages to be gone or in 604 * the process of going. sleep to wait for the rest to go [via iosync]. 605 */ 606 607 while (uvn->u_obj.uo_npages) { 608 #ifdef DEBUG 609 struct vm_page *pp; 610 for (pp = uvn->u_obj.memq.tqh_first ; pp != NULL ; 611 pp = pp->listq.tqe_next) { 612 if ((pp->flags & PG_BUSY) == 0) 613 panic("uvm_vnp_terminate: detected unbusy pg"); 614 } 615 if (uvn->u_nio == 0) 616 panic("uvm_vnp_terminate: no I/O to wait for?"); 617 printf("uvm_vnp_terminate: waiting for I/O to fin.\n"); 618 /* 619 * XXXCDC: this is unlikely to happen without async i/o so we 620 * put a printf in just to keep an eye on it. 621 */ 622 #endif 623 uvn->u_flags |= UVM_VNODE_IOSYNC; 624 UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock, FALSE, 625 "uvn_term",0); 626 simple_lock(&uvn->u_obj.vmobjlock); 627 } 628 629 /* 630 * done. now we free the uvn if its reference count is zero 631 * (true if we are zapping a persisting uvn). however, if we are 632 * terminating a uvn with active mappings we let it live ... future 633 * calls down to the vnode layer will fail. 634 */ 635 636 oldflags = uvn->u_flags; 637 if (uvn->u_obj.uo_refs) { 638 639 /* 640 * uvn must live on it is dead-vnode state until all references 641 * are gone. restore flags. clear CANPERSIST state. 642 */ 643 644 uvn->u_flags &= ~(UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED| 645 UVM_VNODE_WANTED|UVM_VNODE_CANPERSIST); 646 647 } else { 648 649 /* 650 * free the uvn now. note that the VREF reference is already 651 * gone [it is dropped when we enter the persist state]. 652 */ 653 if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED) 654 panic("uvm_vnp_terminate: io sync wanted bit set"); 655 656 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 657 simple_lock(&uvn_wl_lock); 658 LIST_REMOVE(uvn, u_wlist); 659 simple_unlock(&uvn_wl_lock); 660 } 661 uvn->u_flags = 0; /* uvn is history, clear all bits */ 662 } 663 664 if (oldflags & UVM_VNODE_WANTED) 665 wakeup(uvn); /* object lock still held */ 666 667 simple_unlock(&uvn->u_obj.vmobjlock); 668 UVMHIST_LOG(maphist, "<- done", 0, 0, 0, 0); 669 670 } 671 672 /* 673 * uvn_releasepg: handled a released page in a uvn 674 * 675 * => "pg" is a PG_BUSY [caller owns it], PG_RELEASED page that we need 676 * to dispose of. 677 * => caller must handled PG_WANTED case 678 * => called with page's object locked, pageq's unlocked 679 * => returns TRUE if page's object is still alive, FALSE if we 680 * killed the page's object. if we return TRUE, then we 681 * return with the object locked. 682 * => if (nextpgp != NULL) => we return pageq.tqe_next here, and return 683 * with the page queues locked [for pagedaemon] 684 * => if (nextpgp == NULL) => we return with page queues unlocked [normal case] 685 * => we kill the uvn if it is not referenced and we are suppose to 686 * kill it ("relkill"). 687 */ 688 689 boolean_t 690 uvn_releasepg(pg, nextpgp) 691 struct vm_page *pg; 692 struct vm_page **nextpgp; /* OUT */ 693 { 694 struct uvm_vnode *uvn = (struct uvm_vnode *) pg->uobject; 695 #ifdef DIAGNOSTIC 696 if ((pg->flags & PG_RELEASED) == 0) 697 panic("uvn_releasepg: page not released!"); 698 #endif 699 700 /* 701 * dispose of the page [caller handles PG_WANTED] 702 */ 703 pmap_page_protect(pg, VM_PROT_NONE); 704 uvm_lock_pageq(); 705 if (nextpgp) 706 *nextpgp = pg->pageq.tqe_next; /* next page for daemon */ 707 uvm_pagefree(pg); 708 if (!nextpgp) 709 uvm_unlock_pageq(); 710 711 /* 712 * now see if we need to kill the object 713 */ 714 if (uvn->u_flags & UVM_VNODE_RELKILL) { 715 if (uvn->u_obj.uo_refs) 716 panic("uvn_releasepg: kill flag set on referenced " 717 "object!"); 718 if (uvn->u_obj.uo_npages == 0) { 719 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 720 simple_lock(&uvn_wl_lock); 721 LIST_REMOVE(uvn, u_wlist); 722 simple_unlock(&uvn_wl_lock); 723 } 724 #ifdef DIAGNOSTIC 725 if (uvn->u_obj.memq.tqh_first) 726 panic("uvn_releasepg: pages in object with npages == 0"); 727 #endif 728 if (uvn->u_flags & UVM_VNODE_WANTED) 729 /* still holding object lock */ 730 wakeup(uvn); 731 732 uvn->u_flags = 0; /* DEAD! */ 733 simple_unlock(&uvn->u_obj.vmobjlock); 734 return (FALSE); 735 } 736 } 737 return (TRUE); 738 } 739 740 /* 741 * NOTE: currently we have to use VOP_READ/VOP_WRITE because they go 742 * through the buffer cache and allow I/O in any size. These VOPs use 743 * synchronous i/o. [vs. VOP_STRATEGY which can be async, but doesn't 744 * go through the buffer cache or allow I/O sizes larger than a 745 * block]. we will eventually want to change this. 746 * 747 * issues to consider: 748 * uvm provides the uvm_aiodesc structure for async i/o management. 749 * there are two tailq's in the uvm. structure... one for pending async 750 * i/o and one for "done" async i/o. to do an async i/o one puts 751 * an aiodesc on the "pending" list (protected by splbio()), starts the 752 * i/o and returns VM_PAGER_PEND. when the i/o is done, we expect 753 * some sort of "i/o done" function to be called (at splbio(), interrupt 754 * time). this function should remove the aiodesc from the pending list 755 * and place it on the "done" list and wakeup the daemon. the daemon 756 * will run at normal spl() and will remove all items from the "done" 757 * list and call the "aiodone" hook for each done request (see uvm_pager.c). 758 * [in the old vm code, this was done by calling the "put" routine with 759 * null arguments which made the code harder to read and understand because 760 * you had one function ("put") doing two things.] 761 * 762 * so the current pager needs: 763 * int uvn_aiodone(struct uvm_aiodesc *) 764 * 765 * => return KERN_SUCCESS (aio finished, free it). otherwise requeue for 766 * later collection. 767 * => called with pageq's locked by the daemon. 768 * 769 * general outline: 770 * - "try" to lock object. if fail, just return (will try again later) 771 * - drop "u_nio" (this req is done!) 772 * - if (object->iosync && u_naio == 0) { wakeup &uvn->u_naio } 773 * - get "page" structures (atop?). 774 * - handle "wanted" pages 775 * - handle "released" pages [using pgo_releasepg] 776 * >>> pgo_releasepg may kill the object 777 * dont forget to look at "object" wanted flag in all cases. 778 */ 779 780 781 /* 782 * uvn_flush: flush pages out of a uvm object. 783 * 784 * => object should be locked by caller. we may _unlock_ the object 785 * if (and only if) we need to clean a page (PGO_CLEANIT). 786 * we return with the object locked. 787 * => if PGO_CLEANIT is set, we may block (due to I/O). thus, a caller 788 * might want to unlock higher level resources (e.g. vm_map) 789 * before calling flush. 790 * => if PGO_CLEANIT is not set, then we will neither unlock the object 791 * or block. 792 * => if PGO_ALLPAGE is set, then all pages in the object are valid targets 793 * for flushing. 794 * => NOTE: we rely on the fact that the object's memq is a TAILQ and 795 * that new pages are inserted on the tail end of the list. thus, 796 * we can make a complete pass through the object in one go by starting 797 * at the head and working towards the tail (new pages are put in 798 * front of us). 799 * => NOTE: we are allowed to lock the page queues, so the caller 800 * must not be holding the lock on them [e.g. pagedaemon had 801 * better not call us with the queues locked] 802 * => we return TRUE unless we encountered some sort of I/O error 803 * 804 * comment on "cleaning" object and PG_BUSY pages: 805 * this routine is holding the lock on the object. the only time 806 * that it can run into a PG_BUSY page that it does not own is if 807 * some other process has started I/O on the page (e.g. either 808 * a pagein, or a pageout). if the PG_BUSY page is being paged 809 * in, then it can not be dirty (!PG_CLEAN) because no one has 810 * had a chance to modify it yet. if the PG_BUSY page is being 811 * paged out then it means that someone else has already started 812 * cleaning the page for us (how nice!). in this case, if we 813 * have syncio specified, then after we make our pass through the 814 * object we need to wait for the other PG_BUSY pages to clear 815 * off (i.e. we need to do an iosync). also note that once a 816 * page is PG_BUSY it must stay in its object until it is un-busyed. 817 * 818 * note on page traversal: 819 * we can traverse the pages in an object either by going down the 820 * linked list in "uobj->memq", or we can go over the address range 821 * by page doing hash table lookups for each address. depending 822 * on how many pages are in the object it may be cheaper to do one 823 * or the other. we set "by_list" to true if we are using memq. 824 * if the cost of a hash lookup was equal to the cost of the list 825 * traversal we could compare the number of pages in the start->stop 826 * range to the total number of pages in the object. however, it 827 * seems that a hash table lookup is more expensive than the linked 828 * list traversal, so we multiply the number of pages in the 829 * start->stop range by a penalty which we define below. 830 */ 831 832 #define UVN_HASH_PENALTY 4 /* XXX: a guess */ 833 834 static boolean_t 835 uvn_flush(uobj, start, stop, flags) 836 struct uvm_object *uobj; 837 vaddr_t start, stop; 838 int flags; 839 { 840 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 841 struct vm_page *pp, *ppnext, *ptmp; 842 struct vm_page *pps[MAXBSIZE >> PAGE_SHIFT], **ppsp; 843 int npages, result, lcv; 844 boolean_t retval, need_iosync, by_list, needs_clean; 845 vaddr_t curoff; 846 u_short pp_version; 847 UVMHIST_FUNC("uvn_flush"); UVMHIST_CALLED(maphist); 848 849 curoff = 0; /* XXX: shut up gcc */ 850 /* 851 * get init vals and determine how we are going to traverse object 852 */ 853 854 need_iosync = FALSE; 855 retval = TRUE; /* return value */ 856 if (flags & PGO_ALLPAGES) { 857 start = 0; 858 stop = round_page(uvn->u_size); 859 by_list = TRUE; /* always go by the list */ 860 } else { 861 start = trunc_page(start); 862 stop = round_page(stop); 863 #ifdef DEBUG 864 if (stop > round_page(uvn->u_size)) 865 printf("uvn_flush: strange, got an out of range " 866 "flush (fixed)\n"); 867 #endif 868 by_list = (uobj->uo_npages <= 869 ((stop - start) >> PAGE_SHIFT) * UVN_HASH_PENALTY); 870 } 871 872 UVMHIST_LOG(maphist, 873 " flush start=0x%x, stop=0x%x, by_list=%d, flags=0x%x", 874 start, stop, by_list, flags); 875 876 /* 877 * PG_CLEANCHK: this bit is used by the pgo_mk_pcluster function as 878 * a _hint_ as to how up to date the PG_CLEAN bit is. if the hint 879 * is wrong it will only prevent us from clustering... it won't break 880 * anything. we clear all PG_CLEANCHK bits here, and pgo_mk_pcluster 881 * will set them as it syncs PG_CLEAN. This is only an issue if we 882 * are looking at non-inactive pages (because inactive page's PG_CLEAN 883 * bit is always up to date since there are no mappings). 884 * [borrowed PG_CLEANCHK idea from FreeBSD VM] 885 */ 886 887 if ((flags & PGO_CLEANIT) != 0 && 888 uobj->pgops->pgo_mk_pcluster != NULL) { 889 if (by_list) { 890 for (pp = uobj->memq.tqh_first ; pp != NULL ; 891 pp = pp->listq.tqe_next) { 892 if (pp->offset < start || pp->offset >= stop) 893 continue; 894 pp->flags &= ~PG_CLEANCHK; 895 } 896 897 } else { /* by hash */ 898 for (curoff = start ; curoff < stop; 899 curoff += PAGE_SIZE) { 900 pp = uvm_pagelookup(uobj, curoff); 901 if (pp) 902 pp->flags &= ~PG_CLEANCHK; 903 } 904 } 905 } 906 907 /* 908 * now do it. note: we must update ppnext in body of loop or we 909 * will get stuck. we need to use ppnext because we may free "pp" 910 * before doing the next loop. 911 */ 912 913 if (by_list) { 914 pp = uobj->memq.tqh_first; 915 } else { 916 curoff = start; 917 pp = uvm_pagelookup(uobj, curoff); 918 } 919 920 ppnext = NULL; /* XXX: shut up gcc */ 921 ppsp = NULL; /* XXX: shut up gcc */ 922 uvm_lock_pageq(); /* page queues locked */ 923 924 /* locked: both page queues and uobj */ 925 for ( ; (by_list && pp != NULL) || 926 (!by_list && curoff < stop) ; pp = ppnext) { 927 928 if (by_list) { 929 930 /* 931 * range check 932 */ 933 934 if (pp->offset < start || pp->offset >= stop) { 935 ppnext = pp->listq.tqe_next; 936 continue; 937 } 938 939 } else { 940 941 /* 942 * null check 943 */ 944 945 curoff += PAGE_SIZE; 946 if (pp == NULL) { 947 if (curoff < stop) 948 ppnext = uvm_pagelookup(uobj, curoff); 949 continue; 950 } 951 952 } 953 954 /* 955 * handle case where we do not need to clean page (either 956 * because we are not clean or because page is not dirty or 957 * is busy): 958 * 959 * NOTE: we are allowed to deactivate a non-wired active 960 * PG_BUSY page, but once a PG_BUSY page is on the inactive 961 * queue it must stay put until it is !PG_BUSY (so as not to 962 * confuse pagedaemon). 963 */ 964 965 if ((flags & PGO_CLEANIT) == 0 || (pp->flags & PG_BUSY) != 0) { 966 needs_clean = FALSE; 967 if ((pp->flags & PG_BUSY) != 0 && 968 (flags & (PGO_CLEANIT|PGO_SYNCIO)) == 969 (PGO_CLEANIT|PGO_SYNCIO)) 970 need_iosync = TRUE; 971 } else { 972 /* 973 * freeing: nuke all mappings so we can sync 974 * PG_CLEAN bit with no race 975 */ 976 if ((pp->flags & PG_CLEAN) != 0 && 977 (flags & PGO_FREE) != 0 && 978 (pp->pqflags & PQ_ACTIVE) != 0) 979 pmap_page_protect(pp, VM_PROT_NONE); 980 if ((pp->flags & PG_CLEAN) != 0 && 981 pmap_is_modified(pp)) 982 pp->flags &= ~(PG_CLEAN); 983 pp->flags |= PG_CLEANCHK; /* update "hint" */ 984 985 needs_clean = ((pp->flags & PG_CLEAN) == 0); 986 } 987 988 /* 989 * if we don't need a clean... load ppnext and dispose of pp 990 */ 991 if (!needs_clean) { 992 /* load ppnext */ 993 if (by_list) 994 ppnext = pp->listq.tqe_next; 995 else { 996 if (curoff < stop) 997 ppnext = uvm_pagelookup(uobj, curoff); 998 } 999 1000 /* now dispose of pp */ 1001 if (flags & PGO_DEACTIVATE) { 1002 if ((pp->pqflags & PQ_INACTIVE) == 0 && 1003 pp->wire_count == 0) { 1004 pmap_page_protect(pp, VM_PROT_NONE); 1005 uvm_pagedeactivate(pp); 1006 } 1007 1008 } else if (flags & PGO_FREE) { 1009 if (pp->flags & PG_BUSY) { 1010 /* release busy pages */ 1011 pp->flags |= PG_RELEASED; 1012 } else { 1013 pmap_page_protect(pp, VM_PROT_NONE); 1014 /* removed page from object */ 1015 uvm_pagefree(pp); 1016 } 1017 } 1018 /* ppnext is valid so we can continue... */ 1019 continue; 1020 } 1021 1022 /* 1023 * pp points to a page in the locked object that we are 1024 * working on. if it is !PG_CLEAN,!PG_BUSY and we asked 1025 * for cleaning (PGO_CLEANIT). we clean it now. 1026 * 1027 * let uvm_pager_put attempted a clustered page out. 1028 * note: locked: uobj and page queues. 1029 */ 1030 1031 pp->flags |= PG_BUSY; /* we 'own' page now */ 1032 UVM_PAGE_OWN(pp, "uvn_flush"); 1033 pmap_page_protect(pp, VM_PROT_READ); 1034 pp_version = pp->version; 1035 ReTry: 1036 ppsp = pps; 1037 npages = sizeof(pps) / sizeof(struct vm_page *); 1038 1039 /* locked: page queues, uobj */ 1040 result = uvm_pager_put(uobj, pp, &ppsp, &npages, 1041 flags | PGO_DOACTCLUST, start, stop); 1042 /* unlocked: page queues, uobj */ 1043 1044 /* 1045 * at this point nothing is locked. if we did an async I/O 1046 * it is remotely possible for the async i/o to complete and 1047 * the page "pp" be freed or what not before we get a chance 1048 * to relock the object. in order to detect this, we have 1049 * saved the version number of the page in "pp_version". 1050 */ 1051 1052 /* relock! */ 1053 simple_lock(&uobj->vmobjlock); 1054 uvm_lock_pageq(); 1055 1056 /* 1057 * VM_PAGER_AGAIN: given the structure of this pager, this 1058 * can only happen when we are doing async I/O and can't 1059 * map the pages into kernel memory (pager_map) due to lack 1060 * of vm space. if this happens we drop back to sync I/O. 1061 */ 1062 1063 if (result == VM_PAGER_AGAIN) { 1064 /* 1065 * it is unlikely, but page could have been released 1066 * while we had the object lock dropped. we ignore 1067 * this now and retry the I/O. we will detect and 1068 * handle the released page after the syncio I/O 1069 * completes. 1070 */ 1071 #ifdef DIAGNOSTIC 1072 if (flags & PGO_SYNCIO) 1073 panic("uvn_flush: PGO_SYNCIO return 'try again' error (impossible)"); 1074 #endif 1075 flags |= PGO_SYNCIO; 1076 goto ReTry; 1077 } 1078 1079 /* 1080 * the cleaning operation is now done. finish up. note that 1081 * on error (!OK, !PEND) uvm_pager_put drops the cluster for us. 1082 * if success (OK, PEND) then uvm_pager_put returns the cluster 1083 * to us in ppsp/npages. 1084 */ 1085 1086 /* 1087 * for pending async i/o if we are not deactivating/freeing 1088 * we can move on to the next page. 1089 */ 1090 1091 if (result == VM_PAGER_PEND) { 1092 1093 if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) { 1094 /* 1095 * no per-page ops: refresh ppnext and continue 1096 */ 1097 if (by_list) { 1098 if (pp->version == pp_version) 1099 ppnext = pp->listq.tqe_next; 1100 else 1101 /* reset */ 1102 ppnext = uobj->memq.tqh_first; 1103 } else { 1104 if (curoff < stop) 1105 ppnext = uvm_pagelookup(uobj, 1106 curoff); 1107 } 1108 continue; 1109 } 1110 1111 /* need to do anything here? */ 1112 } 1113 1114 /* 1115 * need to look at each page of the I/O operation. we defer 1116 * processing "pp" until the last trip through this "for" loop 1117 * so that we can load "ppnext" for the main loop after we 1118 * play with the cluster pages [thus the "npages + 1" in the 1119 * loop below]. 1120 */ 1121 1122 for (lcv = 0 ; lcv < npages + 1 ; lcv++) { 1123 1124 /* 1125 * handle ppnext for outside loop, and saving pp 1126 * until the end. 1127 */ 1128 if (lcv < npages) { 1129 if (ppsp[lcv] == pp) 1130 continue; /* skip pp until the end */ 1131 ptmp = ppsp[lcv]; 1132 } else { 1133 ptmp = pp; 1134 1135 /* set up next page for outer loop */ 1136 if (by_list) { 1137 if (pp->version == pp_version) 1138 ppnext = pp->listq.tqe_next; 1139 else 1140 /* reset */ 1141 ppnext = uobj->memq.tqh_first; 1142 } else { 1143 if (curoff < stop) 1144 ppnext = uvm_pagelookup(uobj, curoff); 1145 } 1146 } 1147 1148 /* 1149 * verify the page didn't get moved while obj was 1150 * unlocked 1151 */ 1152 if (result == VM_PAGER_PEND && ptmp->uobject != uobj) 1153 continue; 1154 1155 /* 1156 * unbusy the page if I/O is done. note that for 1157 * pending I/O it is possible that the I/O op 1158 * finished before we relocked the object (in 1159 * which case the page is no longer busy). 1160 */ 1161 1162 if (result != VM_PAGER_PEND) { 1163 if (ptmp->flags & PG_WANTED) 1164 /* still holding object lock */ 1165 wakeup(ptmp); 1166 1167 ptmp->flags &= ~(PG_WANTED|PG_BUSY); 1168 UVM_PAGE_OWN(ptmp, NULL); 1169 if (ptmp->flags & PG_RELEASED) { 1170 1171 /* pgo_releasepg wants this */ 1172 uvm_unlock_pageq(); 1173 if (!uvn_releasepg(ptmp, NULL)) 1174 return (TRUE); 1175 1176 uvm_lock_pageq(); /* relock */ 1177 continue; /* next page */ 1178 1179 } else { 1180 ptmp->flags |= (PG_CLEAN|PG_CLEANCHK); 1181 if ((flags & PGO_FREE) == 0) 1182 pmap_clear_modify(ptmp); 1183 } 1184 } 1185 1186 /* 1187 * dispose of page 1188 */ 1189 1190 if (flags & PGO_DEACTIVATE) { 1191 if ((pp->pqflags & PQ_INACTIVE) == 0 && 1192 pp->wire_count == 0) { 1193 pmap_page_protect(ptmp, VM_PROT_NONE); 1194 uvm_pagedeactivate(ptmp); 1195 } 1196 1197 } else if (flags & PGO_FREE) { 1198 if (result == VM_PAGER_PEND) { 1199 if ((ptmp->flags & PG_BUSY) != 0) 1200 /* signal for i/o done */ 1201 ptmp->flags |= PG_RELEASED; 1202 } else { 1203 if (result != VM_PAGER_OK) { 1204 printf("uvn_flush: obj=%p, " 1205 "offset=0x%lx. error " 1206 "during pageout.\n", 1207 pp->uobject, pp->offset); 1208 printf("uvn_flush: WARNING: " 1209 "changes to page may be " 1210 "lost!\n"); 1211 retval = FALSE; 1212 } 1213 pmap_page_protect(ptmp, VM_PROT_NONE); 1214 uvm_pagefree(ptmp); 1215 } 1216 } 1217 1218 } /* end of "lcv" for loop */ 1219 1220 } /* end of "pp" for loop */ 1221 1222 /* 1223 * done with pagequeues: unlock 1224 */ 1225 uvm_unlock_pageq(); 1226 1227 /* 1228 * now wait for all I/O if required. 1229 */ 1230 if (need_iosync) { 1231 1232 UVMHIST_LOG(maphist," <<DOING IOSYNC>>",0,0,0,0); 1233 while (uvn->u_nio != 0) { 1234 uvn->u_flags |= UVM_VNODE_IOSYNC; 1235 UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock, 1236 FALSE, "uvn_flush",0); 1237 simple_lock(&uvn->u_obj.vmobjlock); 1238 } 1239 if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED) 1240 wakeup(&uvn->u_flags); 1241 uvn->u_flags &= ~(UVM_VNODE_IOSYNC|UVM_VNODE_IOSYNCWANTED); 1242 } 1243 1244 /* return, with object locked! */ 1245 UVMHIST_LOG(maphist,"<- done (retval=0x%x)",retval,0,0,0); 1246 return(retval); 1247 } 1248 1249 /* 1250 * uvn_cluster 1251 * 1252 * we are about to do I/O in an object at offset. this function is called 1253 * to establish a range of offsets around "offset" in which we can cluster 1254 * I/O. 1255 * 1256 * - currently doesn't matter if obj locked or not. 1257 */ 1258 1259 static void 1260 uvn_cluster(uobj, offset, loffset, hoffset) 1261 struct uvm_object *uobj; 1262 vaddr_t offset; 1263 vaddr_t *loffset, *hoffset; /* OUT */ 1264 { 1265 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 1266 *loffset = offset; 1267 1268 if (*loffset >= uvn->u_size) 1269 panic("uvn_cluster: offset out of range"); 1270 1271 /* 1272 * XXX: old pager claims we could use VOP_BMAP to get maxcontig value. 1273 */ 1274 *hoffset = *loffset + MAXBSIZE; 1275 if (*hoffset > round_page(uvn->u_size)) /* past end? */ 1276 *hoffset = round_page(uvn->u_size); 1277 1278 return; 1279 } 1280 1281 /* 1282 * uvn_put: flush page data to backing store. 1283 * 1284 * => prefer map unlocked (not required) 1285 * => object must be locked! we will _unlock_ it before starting I/O. 1286 * => flags: PGO_SYNCIO -- use sync. I/O 1287 * => note: caller must set PG_CLEAN and pmap_clear_modify (if needed) 1288 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync. 1289 * [thus we never do async i/o! see iodone comment] 1290 */ 1291 1292 static int 1293 uvn_put(uobj, pps, npages, flags) 1294 struct uvm_object *uobj; 1295 struct vm_page **pps; 1296 int npages, flags; 1297 { 1298 int retval; 1299 1300 /* note: object locked */ 1301 retval = uvn_io((struct uvm_vnode*)uobj, pps, npages, flags, UIO_WRITE); 1302 /* note: object unlocked */ 1303 1304 return(retval); 1305 } 1306 1307 1308 /* 1309 * uvn_get: get pages (synchronously) from backing store 1310 * 1311 * => prefer map unlocked (not required) 1312 * => object must be locked! we will _unlock_ it before starting any I/O. 1313 * => flags: PGO_ALLPAGES: get all of the pages 1314 * PGO_LOCKED: fault data structures are locked 1315 * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx] 1316 * => NOTE: caller must check for released pages!! 1317 */ 1318 1319 static int 1320 uvn_get(uobj, offset, pps, npagesp, centeridx, access_type, advice, flags) 1321 struct uvm_object *uobj; 1322 vaddr_t offset; 1323 struct vm_page **pps; /* IN/OUT */ 1324 int *npagesp; /* IN (OUT if PGO_LOCKED) */ 1325 int centeridx, advice, flags; 1326 vm_prot_t access_type; 1327 { 1328 vaddr_t current_offset; 1329 struct vm_page *ptmp; 1330 int lcv, result, gotpages; 1331 boolean_t done; 1332 UVMHIST_FUNC("uvn_get"); UVMHIST_CALLED(maphist); 1333 UVMHIST_LOG(maphist, "flags=%d", flags,0,0,0); 1334 1335 /* 1336 * step 1: handled the case where fault data structures are locked. 1337 */ 1338 1339 if (flags & PGO_LOCKED) { 1340 1341 /* 1342 * gotpages is the current number of pages we've gotten (which 1343 * we pass back up to caller via *npagesp. 1344 */ 1345 1346 gotpages = 0; 1347 1348 /* 1349 * step 1a: get pages that are already resident. only do this 1350 * if the data structures are locked (i.e. the first time 1351 * through). 1352 */ 1353 1354 done = TRUE; /* be optimistic */ 1355 1356 for (lcv = 0, current_offset = offset ; lcv < *npagesp ; 1357 lcv++, current_offset += PAGE_SIZE) { 1358 1359 /* do we care about this page? if not, skip it */ 1360 if (pps[lcv] == PGO_DONTCARE) 1361 continue; 1362 1363 /* lookup page */ 1364 ptmp = uvm_pagelookup(uobj, current_offset); 1365 1366 /* to be useful must get a non-busy, non-released pg */ 1367 if (ptmp == NULL || 1368 (ptmp->flags & (PG_BUSY|PG_RELEASED)) != 0) { 1369 if (lcv == centeridx || (flags & PGO_ALLPAGES) 1370 != 0) 1371 done = FALSE; /* need to do a wait or I/O! */ 1372 continue; 1373 } 1374 1375 /* 1376 * useful page: busy/lock it and plug it in our 1377 * result array 1378 */ 1379 ptmp->flags |= PG_BUSY; /* loan up to caller */ 1380 UVM_PAGE_OWN(ptmp, "uvn_get1"); 1381 pps[lcv] = ptmp; 1382 gotpages++; 1383 1384 } /* "for" lcv loop */ 1385 1386 /* 1387 * XXX: given the "advice", should we consider async read-ahead? 1388 * XXX: fault current does deactive of pages behind us. is 1389 * this good (other callers might now). 1390 */ 1391 /* 1392 * XXX: read-ahead currently handled by buffer cache (bread) 1393 * level. 1394 * XXX: no async i/o available. 1395 * XXX: so we don't do anything now. 1396 */ 1397 1398 /* 1399 * step 1c: now we've either done everything needed or we to 1400 * unlock and do some waiting or I/O. 1401 */ 1402 1403 *npagesp = gotpages; /* let caller know */ 1404 if (done) 1405 return(VM_PAGER_OK); /* bingo! */ 1406 else 1407 /* EEK! Need to unlock and I/O */ 1408 return(VM_PAGER_UNLOCK); 1409 } 1410 1411 /* 1412 * step 2: get non-resident or busy pages. 1413 * object is locked. data structures are unlocked. 1414 * 1415 * XXX: because we can't do async I/O at this level we get things 1416 * page at a time (otherwise we'd chunk). the VOP_READ() will do 1417 * async-read-ahead for us at a lower level. 1418 */ 1419 1420 for (lcv = 0, current_offset = offset ; 1421 lcv < *npagesp ; lcv++, current_offset += PAGE_SIZE) { 1422 1423 /* skip over pages we've already gotten or don't want */ 1424 /* skip over pages we don't _have_ to get */ 1425 if (pps[lcv] != NULL || (lcv != centeridx && 1426 (flags & PGO_ALLPAGES) == 0)) 1427 continue; 1428 1429 /* 1430 * we have yet to locate the current page (pps[lcv]). we first 1431 * look for a page that is already at the current offset. if 1432 * we fine a page, we check to see if it is busy or released. 1433 * if that is the case, then we sleep on the page until it is 1434 * no longer busy or released and repeat the lookup. if the 1435 * page we found is neither busy nor released, then we busy it 1436 * (so we own it) and plug it into pps[lcv]. this breaks the 1437 * following while loop and indicates we are ready to move on 1438 * to the next page in the "lcv" loop above. 1439 * 1440 * if we exit the while loop with pps[lcv] still set to NULL, 1441 * then it means that we allocated a new busy/fake/clean page 1442 * ptmp in the object and we need to do I/O to fill in the data. 1443 */ 1444 1445 while (pps[lcv] == NULL) { /* top of "pps" while loop */ 1446 1447 /* look for a current page */ 1448 ptmp = uvm_pagelookup(uobj, current_offset); 1449 1450 /* nope? allocate one now (if we can) */ 1451 if (ptmp == NULL) { 1452 1453 ptmp = uvm_pagealloc(uobj, current_offset, 1454 NULL, 0); 1455 1456 /* out of RAM? */ 1457 if (ptmp == NULL) { 1458 simple_unlock(&uobj->vmobjlock); 1459 uvm_wait("uvn_getpage"); 1460 simple_lock(&uobj->vmobjlock); 1461 1462 /* goto top of pps while loop */ 1463 continue; 1464 } 1465 1466 /* 1467 * got new page ready for I/O. break pps 1468 * while loop. pps[lcv] is still NULL. 1469 */ 1470 break; 1471 } 1472 1473 /* page is there, see if we need to wait on it */ 1474 if ((ptmp->flags & (PG_BUSY|PG_RELEASED)) != 0) { 1475 ptmp->flags |= PG_WANTED; 1476 UVM_UNLOCK_AND_WAIT(ptmp, 1477 &uobj->vmobjlock, FALSE, "uvn_get",0); 1478 simple_lock(&uobj->vmobjlock); 1479 continue; /* goto top of pps while loop */ 1480 } 1481 1482 /* 1483 * if we get here then the page has become resident 1484 * and unbusy between steps 1 and 2. we busy it 1485 * now (so we own it) and set pps[lcv] (so that we 1486 * exit the while loop). 1487 */ 1488 ptmp->flags |= PG_BUSY; 1489 UVM_PAGE_OWN(ptmp, "uvn_get2"); 1490 pps[lcv] = ptmp; 1491 } 1492 1493 /* 1494 * if we own the a valid page at the correct offset, pps[lcv] 1495 * will point to it. nothing more to do except go to the 1496 * next page. 1497 */ 1498 1499 if (pps[lcv]) 1500 continue; /* next lcv */ 1501 1502 /* 1503 * we have a "fake/busy/clean" page that we just allocated. do 1504 * I/O to fill it with valid data. note that object must be 1505 * locked going into uvn_io, but will be unlocked afterwards. 1506 */ 1507 1508 result = uvn_io((struct uvm_vnode *) uobj, &ptmp, 1, 1509 PGO_SYNCIO, UIO_READ); 1510 1511 /* 1512 * I/O done. object is unlocked (by uvn_io). because we used 1513 * syncio the result can not be PEND or AGAIN. we must relock 1514 * and check for errors. 1515 */ 1516 1517 /* lock object. check for errors. */ 1518 simple_lock(&uobj->vmobjlock); 1519 if (result != VM_PAGER_OK) { 1520 if (ptmp->flags & PG_WANTED) 1521 /* object lock still held */ 1522 wakeup(ptmp); 1523 1524 ptmp->flags &= ~(PG_WANTED|PG_BUSY); 1525 UVM_PAGE_OWN(ptmp, NULL); 1526 uvm_lock_pageq(); 1527 uvm_pagefree(ptmp); 1528 uvm_unlock_pageq(); 1529 simple_unlock(&uobj->vmobjlock); 1530 return(result); 1531 } 1532 1533 /* 1534 * we got the page! clear the fake flag (indicates valid 1535 * data now in page) and plug into our result array. note 1536 * that page is still busy. 1537 * 1538 * it is the callers job to: 1539 * => check if the page is released 1540 * => unbusy the page 1541 * => activate the page 1542 */ 1543 1544 ptmp->flags &= ~PG_FAKE; /* data is valid ... */ 1545 pmap_clear_modify(ptmp); /* ... and clean */ 1546 pps[lcv] = ptmp; 1547 1548 } /* lcv loop */ 1549 1550 /* 1551 * finally, unlock object and return. 1552 */ 1553 1554 simple_unlock(&uobj->vmobjlock); 1555 return (VM_PAGER_OK); 1556 } 1557 1558 /* 1559 * uvn_asyncget: start async I/O to bring pages into ram 1560 * 1561 * => caller must lock object(???XXX: see if this is best) 1562 * => could be called from uvn_get or a madvise() fault-ahead. 1563 * => if it fails, it doesn't matter. 1564 */ 1565 1566 static int 1567 uvn_asyncget(uobj, offset, npages) 1568 struct uvm_object *uobj; 1569 vaddr_t offset; 1570 int npages; 1571 { 1572 1573 /* 1574 * XXXCDC: we can't do async I/O yet 1575 */ 1576 printf("uvn_asyncget called\n"); 1577 return (KERN_SUCCESS); 1578 } 1579 1580 /* 1581 * uvn_io: do I/O to a vnode 1582 * 1583 * => prefer map unlocked (not required) 1584 * => object must be locked! we will _unlock_ it before starting I/O. 1585 * => flags: PGO_SYNCIO -- use sync. I/O 1586 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync. 1587 * [thus we never do async i/o! see iodone comment] 1588 */ 1589 1590 static int 1591 uvn_io(uvn, pps, npages, flags, rw) 1592 struct uvm_vnode *uvn; 1593 vm_page_t *pps; 1594 int npages, flags, rw; 1595 { 1596 struct vnode *vn; 1597 struct uio uio; 1598 struct iovec iov; 1599 vaddr_t kva, file_offset; 1600 int waitf, result, got, wanted; 1601 UVMHIST_FUNC("uvn_io"); UVMHIST_CALLED(maphist); 1602 1603 UVMHIST_LOG(maphist, "rw=%d", rw,0,0,0); 1604 1605 /* 1606 * init values 1607 */ 1608 1609 waitf = (flags & PGO_SYNCIO) ? M_WAITOK : M_NOWAIT; 1610 vn = (struct vnode *) uvn; 1611 file_offset = pps[0]->offset; 1612 1613 /* 1614 * check for sync'ing I/O. 1615 */ 1616 1617 while (uvn->u_flags & UVM_VNODE_IOSYNC) { 1618 if (waitf == M_NOWAIT) { 1619 simple_unlock(&uvn->u_obj.vmobjlock); 1620 UVMHIST_LOG(maphist,"<- try again (iosync)",0,0,0,0); 1621 return(VM_PAGER_AGAIN); 1622 } 1623 uvn->u_flags |= UVM_VNODE_IOSYNCWANTED; 1624 UVM_UNLOCK_AND_WAIT(&uvn->u_flags, &uvn->u_obj.vmobjlock, 1625 FALSE, "uvn_iosync",0); 1626 simple_lock(&uvn->u_obj.vmobjlock); 1627 } 1628 1629 /* 1630 * check size 1631 */ 1632 1633 if (file_offset >= uvn->u_size) { 1634 simple_unlock(&uvn->u_obj.vmobjlock); 1635 UVMHIST_LOG(maphist,"<- BAD (size check)",0,0,0,0); 1636 return(VM_PAGER_BAD); 1637 } 1638 1639 /* 1640 * first try and map the pages in (without waiting) 1641 */ 1642 1643 kva = uvm_pagermapin(pps, npages, NULL, M_NOWAIT); 1644 if (kva == NULL && waitf == M_NOWAIT) { 1645 simple_unlock(&uvn->u_obj.vmobjlock); 1646 UVMHIST_LOG(maphist,"<- mapin failed (try again)",0,0,0,0); 1647 return(VM_PAGER_AGAIN); 1648 } 1649 1650 /* 1651 * ok, now bump u_nio up. at this point we are done with uvn 1652 * and can unlock it. if we still don't have a kva, try again 1653 * (this time with sleep ok). 1654 */ 1655 1656 uvn->u_nio++; /* we have an I/O in progress! */ 1657 simple_unlock(&uvn->u_obj.vmobjlock); 1658 /* NOTE: object now unlocked */ 1659 if (kva == NULL) { 1660 kva = uvm_pagermapin(pps, npages, NULL, M_WAITOK); 1661 } 1662 1663 /* 1664 * ok, mapped in. our pages are PG_BUSY so they are not going to 1665 * get touched (so we can look at "offset" without having to lock 1666 * the object). set up for I/O. 1667 */ 1668 1669 /* 1670 * fill out uio/iov 1671 */ 1672 1673 iov.iov_base = (caddr_t) kva; 1674 wanted = npages << PAGE_SHIFT; 1675 if (file_offset + wanted > uvn->u_size) 1676 wanted = uvn->u_size - file_offset; /* XXX: needed? */ 1677 iov.iov_len = wanted; 1678 uio.uio_iov = &iov; 1679 uio.uio_iovcnt = 1; 1680 uio.uio_offset = file_offset; 1681 uio.uio_segflg = UIO_SYSSPACE; 1682 uio.uio_rw = rw; 1683 uio.uio_resid = wanted; 1684 uio.uio_procp = NULL; 1685 1686 /* 1687 * do the I/O! (XXX: curproc?) 1688 */ 1689 1690 UVMHIST_LOG(maphist, "calling VOP",0,0,0,0); 1691 1692 /* 1693 * This process may already have this vnode locked, if we faulted in 1694 * copyin() or copyout() on a region backed by this vnode 1695 * while doing I/O to the vnode. If this is the case, don't 1696 * panic.. instead, return the error to the user. 1697 * 1698 * XXX this is a stopgap to prevent a panic. 1699 * Ideally, this kind of operation *should* work. 1700 */ 1701 result = 0; 1702 if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0) 1703 result = vn_lock(vn, LK_EXCLUSIVE | LK_RETRY | LK_RECURSEFAIL); 1704 1705 if (result == 0) { 1706 /* NOTE: vnode now locked! */ 1707 1708 if (rw == UIO_READ) 1709 result = VOP_READ(vn, &uio, 0, curproc->p_ucred); 1710 else 1711 result = VOP_WRITE(vn, &uio, 0, curproc->p_ucred); 1712 1713 if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0) 1714 VOP_UNLOCK(vn, 0); 1715 } 1716 1717 /* NOTE: vnode now unlocked (unless vnislocked) */ 1718 1719 UVMHIST_LOG(maphist, "done calling VOP",0,0,0,0); 1720 1721 /* 1722 * result == unix style errno (0 == OK!) 1723 * 1724 * zero out rest of buffer (if needed) 1725 */ 1726 1727 if (result == 0) { 1728 got = wanted - uio.uio_resid; 1729 1730 if (wanted && got == 0) { 1731 result = EIO; /* XXX: error? */ 1732 } else if (got < PAGE_SIZE * npages && rw == UIO_READ) { 1733 memset((void *) (kva + got), 0, 1734 (npages << PAGE_SHIFT) - got); 1735 } 1736 } 1737 1738 /* 1739 * now remove pager mapping 1740 */ 1741 uvm_pagermapout(kva, npages); 1742 1743 /* 1744 * now clean up the object (i.e. drop I/O count) 1745 */ 1746 1747 simple_lock(&uvn->u_obj.vmobjlock); 1748 /* NOTE: object now locked! */ 1749 1750 uvn->u_nio--; /* I/O DONE! */ 1751 if ((uvn->u_flags & UVM_VNODE_IOSYNC) != 0 && uvn->u_nio == 0) { 1752 wakeup(&uvn->u_nio); 1753 } 1754 simple_unlock(&uvn->u_obj.vmobjlock); 1755 /* NOTE: object now unlocked! */ 1756 1757 /* 1758 * done! 1759 */ 1760 1761 UVMHIST_LOG(maphist, "<- done (result %d)", result,0,0,0); 1762 if (result == 0) 1763 return(VM_PAGER_OK); 1764 else 1765 return(VM_PAGER_ERROR); 1766 } 1767 1768 /* 1769 * uvm_vnp_uncache: disable "persisting" in a vnode... when last reference 1770 * is gone we will kill the object (flushing dirty pages back to the vnode 1771 * if needed). 1772 * 1773 * => returns TRUE if there was no uvm_object attached or if there was 1774 * one and we killed it [i.e. if there is no active uvn] 1775 * => called with the vnode VOP_LOCK'd [we will unlock it for I/O, if 1776 * needed] 1777 * 1778 * => XXX: given that we now kill uvn's when a vnode is recycled (without 1779 * having to hold a reference on the vnode) and given a working 1780 * uvm_vnp_sync(), how does that effect the need for this function? 1781 * [XXXCDC: seems like it can die?] 1782 * 1783 * => XXX: this function should DIE once we merge the VM and buffer 1784 * cache. 1785 * 1786 * research shows that this is called in the following places: 1787 * ext2fs_truncate, ffs_truncate, detrunc[msdosfs]: called when vnode 1788 * changes sizes 1789 * ext2fs_write, WRITE [ufs_readwrite], msdosfs_write: called when we 1790 * are written to 1791 * ex2fs_chmod, ufs_chmod: called if VTEXT vnode and the sticky bit 1792 * is off 1793 * ffs_realloccg: when we can't extend the current block and have 1794 * to allocate a new one we call this [XXX: why?] 1795 * nfsrv_rename, rename_files: called when the target filename is there 1796 * and we want to remove it 1797 * nfsrv_remove, sys_unlink: called on file we are removing 1798 * nfsrv_access: if VTEXT and we want WRITE access and we don't uncache 1799 * then return "text busy" 1800 * nfs_open: seems to uncache any file opened with nfs 1801 * vn_writechk: if VTEXT vnode and can't uncache return "text busy" 1802 */ 1803 1804 boolean_t 1805 uvm_vnp_uncache(vp) 1806 struct vnode *vp; 1807 { 1808 struct uvm_vnode *uvn = &vp->v_uvm; 1809 1810 /* 1811 * lock uvn part of the vnode and check to see if we need to do anything 1812 */ 1813 1814 simple_lock(&uvn->u_obj.vmobjlock); 1815 if ((uvn->u_flags & UVM_VNODE_VALID) == 0 || 1816 (uvn->u_flags & UVM_VNODE_BLOCKED) != 0) { 1817 simple_unlock(&uvn->u_obj.vmobjlock); 1818 return(TRUE); 1819 } 1820 1821 /* 1822 * we have a valid, non-blocked uvn. clear persist flag. 1823 * if uvn is currently active we can return now. 1824 */ 1825 1826 uvn->u_flags &= ~UVM_VNODE_CANPERSIST; 1827 if (uvn->u_obj.uo_refs) { 1828 simple_unlock(&uvn->u_obj.vmobjlock); 1829 return(FALSE); 1830 } 1831 1832 /* 1833 * uvn is currently persisting! we have to gain a reference to 1834 * it so that we can call uvn_detach to kill the uvn. 1835 */ 1836 1837 VREF(vp); /* seems ok, even with VOP_LOCK */ 1838 uvn->u_obj.uo_refs++; /* value is now 1 */ 1839 simple_unlock(&uvn->u_obj.vmobjlock); 1840 1841 1842 #ifdef DEBUG 1843 /* 1844 * carry over sanity check from old vnode pager: the vnode should 1845 * be VOP_LOCK'd, and we confirm it here. 1846 */ 1847 if (!VOP_ISLOCKED(vp)) { 1848 boolean_t is_ok_anyway = FALSE; 1849 #ifdef NFS 1850 extern int (**nfsv2_vnodeop_p) __P((void *)); 1851 extern int (**spec_nfsv2nodeop_p) __P((void *)); 1852 extern int (**fifo_nfsv2nodeop_p) __P((void *)); 1853 1854 /* vnode is NOT VOP_LOCKed: some vnode types _never_ lock */ 1855 if (vp->v_op == nfsv2_vnodeop_p || 1856 vp->v_op == spec_nfsv2nodeop_p) { 1857 is_ok_anyway = TRUE; 1858 } 1859 if (vp->v_op == fifo_nfsv2nodeop_p) { 1860 is_ok_anyway = TRUE; 1861 } 1862 #endif /* NFS */ 1863 if (!is_ok_anyway) 1864 panic("uvm_vnp_uncache: vnode not locked!"); 1865 } 1866 #endif /* DEBUG */ 1867 1868 /* 1869 * now drop our reference to the vnode. if we have the sole 1870 * reference to the vnode then this will cause it to die [as we 1871 * just cleared the persist flag]. we have to unlock the vnode 1872 * while we are doing this as it may trigger I/O. 1873 * 1874 * XXX: it might be possible for uvn to get reclaimed while we are 1875 * unlocked causing us to return TRUE when we should not. we ignore 1876 * this as a false-positive return value doesn't hurt us. 1877 */ 1878 VOP_UNLOCK(vp, 0); 1879 uvn_detach(&uvn->u_obj); 1880 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 1881 1882 /* 1883 * and return... 1884 */ 1885 1886 return(TRUE); 1887 } 1888 1889 /* 1890 * uvm_vnp_setsize: grow or shrink a vnode uvn 1891 * 1892 * grow => just update size value 1893 * shrink => toss un-needed pages 1894 * 1895 * => we assume that the caller has a reference of some sort to the 1896 * vnode in question so that it will not be yanked out from under 1897 * us. 1898 * 1899 * called from: 1900 * => truncate fns (ext2fs_truncate, ffs_truncate, detrunc[msdos]) 1901 * => "write" fns (ext2fs_write, WRITE [ufs/ufs], msdosfs_write, nfs_write) 1902 * => ffs_balloc [XXX: why? doesn't WRITE handle?] 1903 * => NFS: nfs_loadattrcache, nfs_getattrcache, nfs_setattr 1904 * => union fs: union_newsize 1905 */ 1906 1907 void 1908 uvm_vnp_setsize(vp, newsize) 1909 struct vnode *vp; 1910 u_quad_t newsize; 1911 { 1912 struct uvm_vnode *uvn = &vp->v_uvm; 1913 1914 /* 1915 * lock uvn and check for valid object, and if valid: do it! 1916 */ 1917 simple_lock(&uvn->u_obj.vmobjlock); 1918 if (uvn->u_flags & UVM_VNODE_VALID) { 1919 1920 /* 1921 * make sure that the newsize fits within a vaddr_t 1922 * XXX: need to revise addressing data types 1923 */ 1924 1925 if (newsize > (vaddr_t) -PAGE_SIZE) { 1926 #ifdef DEBUG 1927 printf("uvm_vnp_setsize: vn %p size truncated " 1928 "%qx->%lx\n", vp, (long long)newsize, 1929 (vaddr_t)-PAGE_SIZE); 1930 #endif 1931 newsize = (vaddr_t)-PAGE_SIZE; 1932 } 1933 1934 /* 1935 * now check if the size has changed: if we shrink we had better 1936 * toss some pages... 1937 */ 1938 1939 if (uvn->u_size > newsize) { 1940 (void)uvn_flush(&uvn->u_obj, (vaddr_t) newsize, 1941 uvn->u_size, PGO_FREE); 1942 } 1943 uvn->u_size = (vaddr_t)newsize; 1944 } 1945 simple_unlock(&uvn->u_obj.vmobjlock); 1946 1947 /* 1948 * done 1949 */ 1950 return; 1951 } 1952 1953 /* 1954 * uvm_vnp_sync: flush all dirty VM pages back to their backing vnodes. 1955 * 1956 * => called from sys_sync with no VM structures locked 1957 * => only one process can do a sync at a time (because the uvn 1958 * structure only has one queue for sync'ing). we ensure this 1959 * by holding the uvn_sync_lock while the sync is in progress. 1960 * other processes attempting a sync will sleep on this lock 1961 * until we are done. 1962 */ 1963 1964 void 1965 uvm_vnp_sync(mp) 1966 struct mount *mp; 1967 { 1968 struct uvm_vnode *uvn; 1969 struct vnode *vp; 1970 boolean_t got_lock; 1971 1972 /* 1973 * step 1: ensure we are only ones using the uvn_sync_q by locking 1974 * our lock... 1975 */ 1976 lockmgr(&uvn_sync_lock, LK_EXCLUSIVE, (void *)0); 1977 1978 /* 1979 * step 2: build up a simpleq of uvns of interest based on the 1980 * write list. we gain a reference to uvns of interest. must 1981 * be careful about locking uvn's since we will be holding uvn_wl_lock 1982 * in the body of the loop. 1983 */ 1984 SIMPLEQ_INIT(&uvn_sync_q); 1985 simple_lock(&uvn_wl_lock); 1986 for (uvn = uvn_wlist.lh_first ; uvn != NULL ; 1987 uvn = uvn->u_wlist.le_next) { 1988 1989 vp = (struct vnode *) uvn; 1990 if (mp && vp->v_mount != mp) 1991 continue; 1992 1993 /* attempt to gain reference */ 1994 while ((got_lock = simple_lock_try(&uvn->u_obj.vmobjlock)) == 1995 FALSE && 1996 (uvn->u_flags & UVM_VNODE_BLOCKED) == 0) 1997 /* spin */ ; 1998 1999 /* 2000 * we will exit the loop if either if the following are true: 2001 * - we got the lock [always true if NCPU == 1] 2002 * - we failed to get the lock but noticed the vnode was 2003 * "blocked" -- in this case the vnode must be a dying 2004 * vnode, and since dying vnodes are in the process of 2005 * being flushed out, we can safely skip this one 2006 * 2007 * we want to skip over the vnode if we did not get the lock, 2008 * or if the vnode is already dying (due to the above logic). 2009 * 2010 * note that uvn must already be valid because we found it on 2011 * the wlist (this also means it can't be ALOCK'd). 2012 */ 2013 if (!got_lock || (uvn->u_flags & UVM_VNODE_BLOCKED) != 0) { 2014 if (got_lock) 2015 simple_unlock(&uvn->u_obj.vmobjlock); 2016 continue; /* skip it */ 2017 } 2018 2019 /* 2020 * gain reference. watch out for persisting uvns (need to 2021 * regain vnode REF). 2022 */ 2023 if (uvn->u_obj.uo_refs == 0) 2024 VREF(vp); 2025 uvn->u_obj.uo_refs++; 2026 simple_unlock(&uvn->u_obj.vmobjlock); 2027 2028 /* 2029 * got it! 2030 */ 2031 SIMPLEQ_INSERT_HEAD(&uvn_sync_q, uvn, u_syncq); 2032 } 2033 simple_unlock(&uvn_wl_lock); 2034 2035 /* 2036 * step 3: we now have a list of uvn's that may need cleaning. 2037 * we are holding the uvn_sync_lock, but have dropped the uvn_wl_lock 2038 * (so we can now safely lock uvn's again). 2039 */ 2040 2041 for (uvn = uvn_sync_q.sqh_first ; uvn ; uvn = uvn->u_syncq.sqe_next) { 2042 simple_lock(&uvn->u_obj.vmobjlock); 2043 #ifdef DEBUG 2044 if (uvn->u_flags & UVM_VNODE_DYING) { 2045 printf("uvm_vnp_sync: dying vnode on sync list\n"); 2046 } 2047 #endif 2048 uvn_flush(&uvn->u_obj, 0, 0, 2049 PGO_CLEANIT|PGO_ALLPAGES|PGO_DOACTCLUST); 2050 2051 /* 2052 * if we have the only reference and we just cleaned the uvn, 2053 * then we can pull it out of the UVM_VNODE_WRITEABLE state 2054 * thus allowing us to avoid thinking about flushing it again 2055 * on later sync ops. 2056 */ 2057 if (uvn->u_obj.uo_refs == 1 && 2058 (uvn->u_flags & UVM_VNODE_WRITEABLE)) { 2059 LIST_REMOVE(uvn, u_wlist); 2060 uvn->u_flags &= ~UVM_VNODE_WRITEABLE; 2061 } 2062 2063 simple_unlock(&uvn->u_obj.vmobjlock); 2064 2065 /* now drop our reference to the uvn */ 2066 uvn_detach(&uvn->u_obj); 2067 } 2068 2069 /* 2070 * done! release sync lock 2071 */ 2072 lockmgr(&uvn_sync_lock, LK_RELEASE, (void *)0); 2073 } 2074