1 /* $OpenBSD: uvm_vnode.c,v 1.54 2009/04/05 18:11:03 oga Exp $ */ 2 /* $NetBSD: uvm_vnode.c,v 1.36 2000/11/24 20:34:01 chs Exp $ */ 3 4 /* 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * Copyright (c) 1991, 1993 7 * The Regents of the University of California. 8 * Copyright (c) 1990 University of Utah. 9 * 10 * All rights reserved. 11 * 12 * This code is derived from software contributed to Berkeley by 13 * the Systems Programming Group of the University of Utah Computer 14 * Science Department. 15 * 16 * Redistribution and use in source and binary forms, with or without 17 * modification, are permitted provided that the following conditions 18 * are met: 19 * 1. Redistributions of source code must retain the above copyright 20 * notice, this list of conditions and the following disclaimer. 21 * 2. Redistributions in binary form must reproduce the above copyright 22 * notice, this list of conditions and the following disclaimer in the 23 * documentation and/or other materials provided with the distribution. 24 * 3. All advertising materials mentioning features or use of this software 25 * must display the following acknowledgement: 26 * This product includes software developed by Charles D. Cranor, 27 * Washington University, the University of California, Berkeley and 28 * its contributors. 29 * 4. Neither the name of the University nor the names of its contributors 30 * may be used to endorse or promote products derived from this software 31 * without specific prior written permission. 32 * 33 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 34 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 35 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 36 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 37 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 38 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 39 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 40 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 41 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 42 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 43 * SUCH DAMAGE. 44 * 45 * @(#)vnode_pager.c 8.8 (Berkeley) 2/13/94 46 * from: Id: uvm_vnode.c,v 1.1.2.26 1998/02/02 20:38:07 chuck Exp 47 */ 48 49 /* 50 * uvm_vnode.c: the vnode pager. 51 */ 52 53 #include <sys/param.h> 54 #include <sys/systm.h> 55 #include <sys/proc.h> 56 #include <sys/malloc.h> 57 #include <sys/vnode.h> 58 #include <sys/disklabel.h> 59 #include <sys/ioctl.h> 60 #include <sys/fcntl.h> 61 #include <sys/conf.h> 62 #include <sys/rwlock.h> 63 64 #include <miscfs/specfs/specdev.h> 65 66 #include <uvm/uvm.h> 67 #include <uvm/uvm_vnode.h> 68 69 /* 70 * private global data structure 71 * 72 * we keep a list of writeable active vnode-backed VM objects for sync op. 73 * we keep a simpleq of vnodes that are currently being sync'd. 74 */ 75 76 LIST_HEAD(uvn_list_struct, uvm_vnode); 77 struct uvn_list_struct uvn_wlist; /* writeable uvns */ 78 79 SIMPLEQ_HEAD(uvn_sq_struct, uvm_vnode); 80 struct uvn_sq_struct uvn_sync_q; /* sync'ing uvns */ 81 struct rwlock uvn_sync_lock; /* locks sync operation */ 82 83 /* 84 * functions 85 */ 86 87 void uvn_cluster(struct uvm_object *, voff_t, voff_t *, voff_t *); 88 void uvn_detach(struct uvm_object *); 89 boolean_t uvn_flush(struct uvm_object *, voff_t, voff_t, int); 90 int uvn_get(struct uvm_object *, voff_t, vm_page_t *, int *, int, 91 vm_prot_t, int, int); 92 void uvn_init(void); 93 int uvn_io(struct uvm_vnode *, vm_page_t *, int, int, int); 94 int uvn_put(struct uvm_object *, vm_page_t *, int, boolean_t); 95 void uvn_reference(struct uvm_object *); 96 boolean_t uvn_releasepg(struct vm_page *, struct vm_page **); 97 98 /* 99 * master pager structure 100 */ 101 102 struct uvm_pagerops uvm_vnodeops = { 103 uvn_init, 104 uvn_reference, 105 uvn_detach, 106 NULL, /* no specialized fault routine required */ 107 uvn_flush, 108 uvn_get, 109 uvn_put, 110 uvn_cluster, 111 uvm_mk_pcluster, /* use generic version of this: see uvm_pager.c */ 112 uvn_releasepg, 113 }; 114 115 /* 116 * the ops! 117 */ 118 119 /* 120 * uvn_init 121 * 122 * init pager private data structures. 123 */ 124 125 void 126 uvn_init(void) 127 { 128 129 LIST_INIT(&uvn_wlist); 130 /* note: uvn_sync_q init'd in uvm_vnp_sync() */ 131 rw_init(&uvn_sync_lock, "uvnsync"); 132 } 133 134 /* 135 * uvn_attach 136 * 137 * attach a vnode structure to a VM object. if the vnode is already 138 * attached, then just bump the reference count by one and return the 139 * VM object. if not already attached, attach and return the new VM obj. 140 * the "accessprot" tells the max access the attaching thread wants to 141 * our pages. 142 * 143 * => caller must _not_ already be holding the lock on the uvm_object. 144 * => in fact, nothing should be locked so that we can sleep here. 145 * => note that uvm_object is first thing in vnode structure, so their 146 * pointers are equiv. 147 */ 148 149 struct uvm_object * 150 uvn_attach(void *arg, vm_prot_t accessprot) 151 { 152 struct vnode *vp = arg; 153 struct uvm_vnode *uvn = &vp->v_uvm; 154 struct vattr vattr; 155 int oldflags, result; 156 struct partinfo pi; 157 u_quad_t used_vnode_size; 158 UVMHIST_FUNC("uvn_attach"); UVMHIST_CALLED(maphist); 159 160 UVMHIST_LOG(maphist, "(vn=%p)", arg,0,0,0); 161 162 used_vnode_size = (u_quad_t)0; /* XXX gcc -Wuninitialized */ 163 164 /* 165 * first get a lock on the uvn. 166 */ 167 simple_lock(&uvn->u_obj.vmobjlock); 168 while (uvn->u_flags & UVM_VNODE_BLOCKED) { 169 uvn->u_flags |= UVM_VNODE_WANTED; 170 UVMHIST_LOG(maphist, " SLEEPING on blocked vn",0,0,0,0); 171 UVM_UNLOCK_AND_WAIT(uvn, &uvn->u_obj.vmobjlock, FALSE, 172 "uvn_attach", 0); 173 simple_lock(&uvn->u_obj.vmobjlock); 174 UVMHIST_LOG(maphist," WOKE UP",0,0,0,0); 175 } 176 177 /* 178 * if we're mapping a BLK device, make sure it is a disk. 179 */ 180 if (vp->v_type == VBLK && bdevsw[major(vp->v_rdev)].d_type != D_DISK) { 181 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock */ 182 UVMHIST_LOG(maphist,"<- done (VBLK not D_DISK!)", 0,0,0,0); 183 return(NULL); 184 } 185 186 /* 187 * now we have lock and uvn must not be in a blocked state. 188 * first check to see if it is already active, in which case 189 * we can bump the reference count, check to see if we need to 190 * add it to the writeable list, and then return. 191 */ 192 if (uvn->u_flags & UVM_VNODE_VALID) { /* already active? */ 193 194 /* regain VREF if we were persisting */ 195 if (uvn->u_obj.uo_refs == 0) { 196 VREF(vp); 197 UVMHIST_LOG(maphist," VREF (reclaim persisting vnode)", 198 0,0,0,0); 199 } 200 uvn->u_obj.uo_refs++; /* bump uvn ref! */ 201 202 /* check for new writeable uvn */ 203 if ((accessprot & VM_PROT_WRITE) != 0 && 204 (uvn->u_flags & UVM_VNODE_WRITEABLE) == 0) { 205 LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist); 206 /* we are now on wlist! */ 207 uvn->u_flags |= UVM_VNODE_WRITEABLE; 208 } 209 210 /* unlock and return */ 211 simple_unlock(&uvn->u_obj.vmobjlock); 212 UVMHIST_LOG(maphist,"<- done, refcnt=%ld", uvn->u_obj.uo_refs, 213 0, 0, 0); 214 return (&uvn->u_obj); 215 } 216 217 /* 218 * need to call VOP_GETATTR() to get the attributes, but that could 219 * block (due to I/O), so we want to unlock the object before calling. 220 * however, we want to keep anyone else from playing with the object 221 * while it is unlocked. to do this we set UVM_VNODE_ALOCK which 222 * prevents anyone from attaching to the vnode until we are done with 223 * it. 224 */ 225 uvn->u_flags = UVM_VNODE_ALOCK; 226 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock in case we sleep */ 227 /* XXX: curproc? */ 228 229 if (vp->v_type == VBLK) { 230 /* 231 * We could implement this as a specfs getattr call, but: 232 * 233 * (1) VOP_GETATTR() would get the file system 234 * vnode operation, not the specfs operation. 235 * 236 * (2) All we want is the size, anyhow. 237 */ 238 result = (*bdevsw[major(vp->v_rdev)].d_ioctl)(vp->v_rdev, 239 DIOCGPART, (caddr_t)&pi, FREAD, curproc); 240 if (result == 0) { 241 /* XXX should remember blocksize */ 242 used_vnode_size = (u_quad_t)pi.disklab->d_secsize * 243 (u_quad_t)DL_GETPSIZE(pi.part); 244 } 245 } else { 246 result = VOP_GETATTR(vp, &vattr, curproc->p_ucred, curproc); 247 if (result == 0) 248 used_vnode_size = vattr.va_size; 249 } 250 251 /* relock object */ 252 simple_lock(&uvn->u_obj.vmobjlock); 253 254 if (result != 0) { 255 if (uvn->u_flags & UVM_VNODE_WANTED) 256 wakeup(uvn); 257 uvn->u_flags = 0; 258 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock */ 259 UVMHIST_LOG(maphist,"<- done (VOP_GETATTR FAILED!)", 0,0,0,0); 260 return(NULL); 261 } 262 263 /* 264 * make sure that the newsize fits within a vaddr_t 265 * XXX: need to revise addressing data types 266 */ 267 #ifdef DEBUG 268 if (vp->v_type == VBLK) 269 printf("used_vnode_size = %llu\n", (long long)used_vnode_size); 270 #endif 271 272 /* 273 * now set up the uvn. 274 */ 275 uvn->u_obj.pgops = &uvm_vnodeops; 276 TAILQ_INIT(&uvn->u_obj.memq); 277 uvn->u_obj.uo_npages = 0; 278 uvn->u_obj.uo_refs = 1; /* just us... */ 279 oldflags = uvn->u_flags; 280 uvn->u_flags = UVM_VNODE_VALID|UVM_VNODE_CANPERSIST; 281 uvn->u_nio = 0; 282 uvn->u_size = used_vnode_size; 283 284 /* if write access, we need to add it to the wlist */ 285 if (accessprot & VM_PROT_WRITE) { 286 LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist); 287 uvn->u_flags |= UVM_VNODE_WRITEABLE; /* we are on wlist! */ 288 } 289 290 /* 291 * add a reference to the vnode. this reference will stay as long 292 * as there is a valid mapping of the vnode. dropped when the 293 * reference count goes to zero [and we either free or persist]. 294 */ 295 VREF(vp); 296 simple_unlock(&uvn->u_obj.vmobjlock); 297 if (oldflags & UVM_VNODE_WANTED) 298 wakeup(uvn); 299 300 UVMHIST_LOG(maphist,"<- done/VREF, ret %p", &uvn->u_obj,0,0,0); 301 return(&uvn->u_obj); 302 } 303 304 305 /* 306 * uvn_reference 307 * 308 * duplicate a reference to a VM object. Note that the reference 309 * count must already be at least one (the passed in reference) so 310 * there is no chance of the uvn being killed or locked out here. 311 * 312 * => caller must call with object unlocked. 313 * => caller must be using the same accessprot as was used at attach time 314 */ 315 316 317 void 318 uvn_reference(struct uvm_object *uobj) 319 { 320 #ifdef DEBUG 321 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 322 #endif 323 UVMHIST_FUNC("uvn_reference"); UVMHIST_CALLED(maphist); 324 325 simple_lock(&uobj->vmobjlock); 326 #ifdef DEBUG 327 if ((uvn->u_flags & UVM_VNODE_VALID) == 0) { 328 printf("uvn_reference: ref=%d, flags=0x%x\n", uvn->u_flags, 329 uobj->uo_refs); 330 panic("uvn_reference: invalid state"); 331 } 332 #endif 333 uobj->uo_refs++; 334 UVMHIST_LOG(maphist, "<- done (uobj=%p, ref = %ld)", 335 uobj, uobj->uo_refs,0,0); 336 simple_unlock(&uobj->vmobjlock); 337 } 338 339 /* 340 * uvn_detach 341 * 342 * remove a reference to a VM object. 343 * 344 * => caller must call with object unlocked and map locked. 345 * => this starts the detach process, but doesn't have to finish it 346 * (async i/o could still be pending). 347 */ 348 void 349 uvn_detach(struct uvm_object *uobj) 350 { 351 struct uvm_vnode *uvn; 352 struct vnode *vp; 353 int oldflags; 354 UVMHIST_FUNC("uvn_detach"); UVMHIST_CALLED(maphist); 355 356 simple_lock(&uobj->vmobjlock); 357 358 UVMHIST_LOG(maphist," (uobj=%p) ref=%ld", uobj,uobj->uo_refs,0,0); 359 uobj->uo_refs--; /* drop ref! */ 360 if (uobj->uo_refs) { /* still more refs */ 361 simple_unlock(&uobj->vmobjlock); 362 UVMHIST_LOG(maphist, "<- done (rc>0)", 0,0,0,0); 363 return; 364 } 365 366 /* 367 * get other pointers ... 368 */ 369 370 uvn = (struct uvm_vnode *) uobj; 371 vp = (struct vnode *) uobj; 372 373 /* 374 * clear VTEXT flag now that there are no mappings left (VTEXT is used 375 * to keep an active text file from being overwritten). 376 */ 377 vp->v_flag &= ~VTEXT; 378 379 /* 380 * we just dropped the last reference to the uvn. see if we can 381 * let it "stick around". 382 */ 383 384 if (uvn->u_flags & UVM_VNODE_CANPERSIST) { 385 /* won't block */ 386 uvn_flush(uobj, 0, 0, PGO_DEACTIVATE|PGO_ALLPAGES); 387 simple_unlock(&uobj->vmobjlock); 388 vrele(vp); /* drop vnode reference */ 389 UVMHIST_LOG(maphist,"<- done/vrele! (persist)", 0,0,0,0); 390 return; 391 } 392 393 /* 394 * its a goner! 395 */ 396 397 UVMHIST_LOG(maphist," its a goner (flushing)!", 0,0,0,0); 398 399 uvn->u_flags |= UVM_VNODE_DYING; 400 401 /* 402 * even though we may unlock in flush, no one can gain a reference 403 * to us until we clear the "dying" flag [because it blocks 404 * attaches]. we will not do that until after we've disposed of all 405 * the pages with uvn_flush(). note that before the flush the only 406 * pages that could be marked PG_BUSY are ones that are in async 407 * pageout by the daemon. (there can't be any pending "get"'s 408 * because there are no references to the object). 409 */ 410 411 (void) uvn_flush(uobj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES); 412 413 UVMHIST_LOG(maphist," its a goner (done flush)!", 0,0,0,0); 414 415 /* 416 * given the structure of this pager, the above flush request will 417 * create the following state: all the pages that were in the object 418 * have either been free'd or they are marked PG_BUSY|PG_RELEASED. 419 * the PG_BUSY bit was set either by us or the daemon for async I/O. 420 * in either case, if we have pages left we can't kill the object 421 * yet because i/o is pending. in this case we set the "relkill" 422 * flag which will cause pgo_releasepg to kill the object once all 423 * the I/O's are done [pgo_releasepg will be called from the aiodone 424 * routine or from the page daemon]. 425 */ 426 427 if (uobj->uo_npages) { /* I/O pending. iodone will free */ 428 #ifdef DEBUG 429 /* 430 * XXXCDC: very unlikely to happen until we have async i/o 431 * so print a little info message in case it does. 432 */ 433 printf("uvn_detach: vn %p has pages left after flush - " 434 "relkill mode\n", uobj); 435 #endif 436 uvn->u_flags |= UVM_VNODE_RELKILL; 437 simple_unlock(&uobj->vmobjlock); 438 UVMHIST_LOG(maphist,"<- done! (releasepg will kill obj)", 0, 0, 439 0, 0); 440 return; 441 } 442 443 /* 444 * kill object now. note that we can't be on the sync q because 445 * all references are gone. 446 */ 447 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 448 LIST_REMOVE(uvn, u_wlist); 449 } 450 #ifdef DIAGNOSTIC 451 if (!TAILQ_EMPTY(&uobj->memq)) 452 panic("uvn_deref: vnode VM object still has pages afer " 453 "syncio/free flush"); 454 #endif 455 oldflags = uvn->u_flags; 456 uvn->u_flags = 0; 457 simple_unlock(&uobj->vmobjlock); 458 459 /* wake up any sleepers */ 460 if (oldflags & UVM_VNODE_WANTED) 461 wakeup(uvn); 462 463 /* 464 * drop our reference to the vnode. 465 */ 466 vrele(vp); 467 UVMHIST_LOG(maphist,"<- done (vrele) final", 0,0,0,0); 468 469 return; 470 } 471 472 /* 473 * uvm_vnp_terminate: external hook to clear out a vnode's VM 474 * 475 * called in two cases: 476 * [1] when a persisting vnode vm object (i.e. one with a zero reference 477 * count) needs to be freed so that a vnode can be reused. this 478 * happens under "getnewvnode" in vfs_subr.c. if the vnode from 479 * the free list is still attached (i.e. not VBAD) then vgone is 480 * called. as part of the vgone trace this should get called to 481 * free the vm object. this is the common case. 482 * [2] when a filesystem is being unmounted by force (MNT_FORCE, 483 * "umount -f") the vgone() function is called on active vnodes 484 * on the mounted file systems to kill their data (the vnodes become 485 * "dead" ones [see src/sys/miscfs/deadfs/...]). that results in a 486 * call here (even if the uvn is still in use -- i.e. has a non-zero 487 * reference count). this case happens at "umount -f" and during a 488 * "reboot/halt" operation. 489 * 490 * => the caller must XLOCK and VOP_LOCK the vnode before calling us 491 * [protects us from getting a vnode that is already in the DYING 492 * state...] 493 * => unlike uvn_detach, this function must not return until all the 494 * uvn's pages are disposed of. 495 * => in case [2] the uvn is still alive after this call, but all I/O 496 * ops will fail (due to the backing vnode now being "dead"). this 497 * will prob. kill any process using the uvn due to pgo_get failing. 498 */ 499 500 void 501 uvm_vnp_terminate(struct vnode *vp) 502 { 503 struct uvm_vnode *uvn = &vp->v_uvm; 504 int oldflags; 505 UVMHIST_FUNC("uvm_vnp_terminate"); UVMHIST_CALLED(maphist); 506 507 /* 508 * lock object and check if it is valid 509 */ 510 simple_lock(&uvn->u_obj.vmobjlock); 511 UVMHIST_LOG(maphist, " vp=%p, ref=%ld, flag=0x%lx", vp, 512 uvn->u_obj.uo_refs, uvn->u_flags, 0); 513 if ((uvn->u_flags & UVM_VNODE_VALID) == 0) { 514 simple_unlock(&uvn->u_obj.vmobjlock); 515 UVMHIST_LOG(maphist, "<- done (not active)", 0, 0, 0, 0); 516 return; 517 } 518 519 /* 520 * must be a valid uvn that is not already dying (because XLOCK 521 * protects us from that). the uvn can't in the ALOCK state 522 * because it is valid, and uvn's that are in the ALOCK state haven't 523 * been marked valid yet. 524 */ 525 526 #ifdef DEBUG 527 /* 528 * debug check: are we yanking the vnode out from under our uvn? 529 */ 530 if (uvn->u_obj.uo_refs) { 531 printf("uvm_vnp_terminate(%p): terminating active vnode " 532 "(refs=%d)\n", uvn, uvn->u_obj.uo_refs); 533 } 534 #endif 535 536 /* 537 * it is possible that the uvn was detached and is in the relkill 538 * state [i.e. waiting for async i/o to finish so that releasepg can 539 * kill object]. we take over the vnode now and cancel the relkill. 540 * we want to know when the i/o is done so we can recycle right 541 * away. note that a uvn can only be in the RELKILL state if it 542 * has a zero reference count. 543 */ 544 545 if (uvn->u_flags & UVM_VNODE_RELKILL) 546 uvn->u_flags &= ~UVM_VNODE_RELKILL; /* cancel RELKILL */ 547 548 /* 549 * block the uvn by setting the dying flag, and then flush the 550 * pages. (note that flush may unlock object while doing I/O, but 551 * it will re-lock it before it returns control here). 552 * 553 * also, note that we tell I/O that we are already VOP_LOCK'd so 554 * that uvn_io doesn't attempt to VOP_LOCK again. 555 * 556 * XXXCDC: setting VNISLOCKED on an active uvn which is being terminated 557 * due to a forceful unmount might not be a good idea. maybe we 558 * need a way to pass in this info to uvn_flush through a 559 * pager-defined PGO_ constant [currently there are none]. 560 */ 561 uvn->u_flags |= UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED; 562 563 (void) uvn_flush(&uvn->u_obj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES); 564 565 /* 566 * as we just did a flush we expect all the pages to be gone or in 567 * the process of going. sleep to wait for the rest to go [via iosync]. 568 */ 569 570 while (uvn->u_obj.uo_npages) { 571 #ifdef DEBUG 572 struct vm_page *pp; 573 TAILQ_FOREACH(pp, &uvn->u_obj.memq, listq) { 574 if ((pp->pg_flags & PG_BUSY) == 0) 575 panic("uvm_vnp_terminate: detected unbusy pg"); 576 } 577 if (uvn->u_nio == 0) 578 panic("uvm_vnp_terminate: no I/O to wait for?"); 579 printf("uvm_vnp_terminate: waiting for I/O to fin.\n"); 580 /* 581 * XXXCDC: this is unlikely to happen without async i/o so we 582 * put a printf in just to keep an eye on it. 583 */ 584 #endif 585 uvn->u_flags |= UVM_VNODE_IOSYNC; 586 UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock, FALSE, 587 "uvn_term",0); 588 simple_lock(&uvn->u_obj.vmobjlock); 589 } 590 591 /* 592 * done. now we free the uvn if its reference count is zero 593 * (true if we are zapping a persisting uvn). however, if we are 594 * terminating a uvn with active mappings we let it live ... future 595 * calls down to the vnode layer will fail. 596 */ 597 598 oldflags = uvn->u_flags; 599 if (uvn->u_obj.uo_refs) { 600 601 /* 602 * uvn must live on it is dead-vnode state until all references 603 * are gone. restore flags. clear CANPERSIST state. 604 */ 605 606 uvn->u_flags &= ~(UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED| 607 UVM_VNODE_WANTED|UVM_VNODE_CANPERSIST); 608 609 } else { 610 611 /* 612 * free the uvn now. note that the VREF reference is already 613 * gone [it is dropped when we enter the persist state]. 614 */ 615 if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED) 616 panic("uvm_vnp_terminate: io sync wanted bit set"); 617 618 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 619 LIST_REMOVE(uvn, u_wlist); 620 } 621 uvn->u_flags = 0; /* uvn is history, clear all bits */ 622 } 623 624 if (oldflags & UVM_VNODE_WANTED) 625 wakeup(uvn); /* object lock still held */ 626 627 simple_unlock(&uvn->u_obj.vmobjlock); 628 UVMHIST_LOG(maphist, "<- done", 0, 0, 0, 0); 629 630 } 631 632 /* 633 * uvn_releasepg: handled a released page in a uvn 634 * 635 * => "pg" is a PG_BUSY [caller owns it], PG_RELEASED page that we need 636 * to dispose of. 637 * => caller must handled PG_WANTED case 638 * => called with page's object locked, pageq's unlocked 639 * => returns TRUE if page's object is still alive, FALSE if we 640 * killed the page's object. if we return TRUE, then we 641 * return with the object locked. 642 * => if (nextpgp != NULL) => we return pageq.tqe_next here, and return 643 * with the page queues locked [for pagedaemon] 644 * => if (nextpgp == NULL) => we return with page queues unlocked [normal case] 645 * => we kill the uvn if it is not referenced and we are suppose to 646 * kill it ("relkill"). 647 */ 648 649 boolean_t 650 uvn_releasepg(struct vm_page *pg, struct vm_page **nextpgp /* OUT */) 651 { 652 struct uvm_vnode *uvn = (struct uvm_vnode *) pg->uobject; 653 struct vnode *vp = (struct vnode *)uvn; 654 #ifdef DIAGNOSTIC 655 if ((pg->pg_flags & PG_RELEASED) == 0) 656 panic("uvn_releasepg: page not released!"); 657 #endif 658 659 /* 660 * dispose of the page [caller handles PG_WANTED] 661 */ 662 pmap_page_protect(pg, VM_PROT_NONE); 663 uvm_lock_pageq(); 664 if (nextpgp) 665 *nextpgp = TAILQ_NEXT(pg, pageq); /* next page for daemon */ 666 uvm_pagefree(pg); 667 if (!nextpgp) 668 uvm_unlock_pageq(); 669 670 /* 671 * now see if we need to kill the object 672 */ 673 if (uvn->u_flags & UVM_VNODE_RELKILL) { 674 if (uvn->u_obj.uo_refs) 675 panic("uvn_releasepg: kill flag set on referenced " 676 "object!"); 677 if (uvn->u_obj.uo_npages == 0) { 678 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 679 LIST_REMOVE(uvn, u_wlist); 680 } 681 #ifdef DIAGNOSTIC 682 if (!TAILQ_EMPTY(&uvn->u_obj.memq)) 683 panic("uvn_releasepg: pages in object with npages == 0"); 684 #endif 685 if (uvn->u_flags & UVM_VNODE_WANTED) 686 /* still holding object lock */ 687 wakeup(uvn); 688 689 uvn->u_flags = 0; /* DEAD! */ 690 simple_unlock(&uvn->u_obj.vmobjlock); 691 vrele(vp); 692 return (FALSE); 693 } 694 } 695 return (TRUE); 696 } 697 698 /* 699 * NOTE: currently we have to use VOP_READ/VOP_WRITE because they go 700 * through the buffer cache and allow I/O in any size. These VOPs use 701 * synchronous i/o. [vs. VOP_STRATEGY which can be async, but doesn't 702 * go through the buffer cache or allow I/O sizes larger than a 703 * block]. we will eventually want to change this. 704 * 705 * issues to consider: 706 * uvm provides the uvm_aiodesc structure for async i/o management. 707 * there are two tailq's in the uvm. structure... one for pending async 708 * i/o and one for "done" async i/o. to do an async i/o one puts 709 * an aiodesc on the "pending" list (protected by splbio()), starts the 710 * i/o and returns VM_PAGER_PEND. when the i/o is done, we expect 711 * some sort of "i/o done" function to be called (at splbio(), interrupt 712 * time). this function should remove the aiodesc from the pending list 713 * and place it on the "done" list and wakeup the daemon. the daemon 714 * will run at normal spl() and will remove all items from the "done" 715 * list and call the "aiodone" hook for each done request (see uvm_pager.c). 716 * [in the old vm code, this was done by calling the "put" routine with 717 * null arguments which made the code harder to read and understand because 718 * you had one function ("put") doing two things.] 719 * 720 * so the current pager needs: 721 * int uvn_aiodone(struct uvm_aiodesc *) 722 * 723 * => return 0 (aio finished, free it). otherwise requeue for later collection. 724 * => called with pageq's locked by the daemon. 725 * 726 * general outline: 727 * - "try" to lock object. if fail, just return (will try again later) 728 * - drop "u_nio" (this req is done!) 729 * - if (object->iosync && u_naio == 0) { wakeup &uvn->u_naio } 730 * - get "page" structures (atop?). 731 * - handle "wanted" pages 732 * - handle "released" pages [using pgo_releasepg] 733 * >>> pgo_releasepg may kill the object 734 * dont forget to look at "object" wanted flag in all cases. 735 */ 736 737 738 /* 739 * uvn_flush: flush pages out of a uvm object. 740 * 741 * => object should be locked by caller. we may _unlock_ the object 742 * if (and only if) we need to clean a page (PGO_CLEANIT). 743 * we return with the object locked. 744 * => if PGO_CLEANIT is set, we may block (due to I/O). thus, a caller 745 * might want to unlock higher level resources (e.g. vm_map) 746 * before calling flush. 747 * => if PGO_CLEANIT is not set, then we will neither unlock the object 748 * or block. 749 * => if PGO_ALLPAGE is set, then all pages in the object are valid targets 750 * for flushing. 751 * => NOTE: we rely on the fact that the object's memq is a TAILQ and 752 * that new pages are inserted on the tail end of the list. thus, 753 * we can make a complete pass through the object in one go by starting 754 * at the head and working towards the tail (new pages are put in 755 * front of us). 756 * => NOTE: we are allowed to lock the page queues, so the caller 757 * must not be holding the lock on them [e.g. pagedaemon had 758 * better not call us with the queues locked] 759 * => we return TRUE unless we encountered some sort of I/O error 760 * 761 * comment on "cleaning" object and PG_BUSY pages: 762 * this routine is holding the lock on the object. the only time 763 * that it can run into a PG_BUSY page that it does not own is if 764 * some other process has started I/O on the page (e.g. either 765 * a pagein, or a pageout). if the PG_BUSY page is being paged 766 * in, then it can not be dirty (!PG_CLEAN) because no one has 767 * had a chance to modify it yet. if the PG_BUSY page is being 768 * paged out then it means that someone else has already started 769 * cleaning the page for us (how nice!). in this case, if we 770 * have syncio specified, then after we make our pass through the 771 * object we need to wait for the other PG_BUSY pages to clear 772 * off (i.e. we need to do an iosync). also note that once a 773 * page is PG_BUSY it must stay in its object until it is un-busyed. 774 * 775 * note on page traversal: 776 * we can traverse the pages in an object either by going down the 777 * linked list in "uobj->memq", or we can go over the address range 778 * by page doing hash table lookups for each address. depending 779 * on how many pages are in the object it may be cheaper to do one 780 * or the other. we set "by_list" to true if we are using memq. 781 * if the cost of a hash lookup was equal to the cost of the list 782 * traversal we could compare the number of pages in the start->stop 783 * range to the total number of pages in the object. however, it 784 * seems that a hash table lookup is more expensive than the linked 785 * list traversal, so we multiply the number of pages in the 786 * start->stop range by a penalty which we define below. 787 */ 788 789 #define UVN_HASH_PENALTY 4 /* XXX: a guess */ 790 791 boolean_t 792 uvn_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags) 793 { 794 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 795 struct vm_page *pp, *ppnext, *ptmp; 796 struct vm_page *pps[MAXBSIZE >> PAGE_SHIFT], **ppsp; 797 int npages, result, lcv; 798 boolean_t retval, need_iosync, by_list, needs_clean, all; 799 voff_t curoff; 800 u_short pp_version; 801 UVMHIST_FUNC("uvn_flush"); UVMHIST_CALLED(maphist); 802 803 curoff = 0; /* XXX: shut up gcc */ 804 /* 805 * get init vals and determine how we are going to traverse object 806 */ 807 808 need_iosync = FALSE; 809 retval = TRUE; /* return value */ 810 if (flags & PGO_ALLPAGES) { 811 all = TRUE; 812 by_list = TRUE; /* always go by the list */ 813 } else { 814 start = trunc_page(start); 815 stop = round_page(stop); 816 #ifdef DEBUG 817 if (stop > round_page(uvn->u_size)) 818 printf("uvn_flush: strange, got an out of range " 819 "flush (fixed)\n"); 820 #endif 821 all = FALSE; 822 by_list = (uobj->uo_npages <= 823 ((stop - start) >> PAGE_SHIFT) * UVN_HASH_PENALTY); 824 } 825 826 UVMHIST_LOG(maphist, 827 " flush start=0x%lx, stop=0x%lx, by_list=%ld, flags=0x%lx", 828 (u_long)start, (u_long)stop, by_list, flags); 829 830 /* 831 * PG_CLEANCHK: this bit is used by the pgo_mk_pcluster function as 832 * a _hint_ as to how up to date the PG_CLEAN bit is. if the hint 833 * is wrong it will only prevent us from clustering... it won't break 834 * anything. we clear all PG_CLEANCHK bits here, and pgo_mk_pcluster 835 * will set them as it syncs PG_CLEAN. This is only an issue if we 836 * are looking at non-inactive pages (because inactive page's PG_CLEAN 837 * bit is always up to date since there are no mappings). 838 * [borrowed PG_CLEANCHK idea from FreeBSD VM] 839 */ 840 841 if ((flags & PGO_CLEANIT) != 0 && 842 uobj->pgops->pgo_mk_pcluster != NULL) { 843 if (by_list) { 844 TAILQ_FOREACH(pp, &uobj->memq, listq) { 845 if (!all && 846 (pp->offset < start || pp->offset >= stop)) 847 continue; 848 atomic_clearbits_int(&pp->pg_flags, 849 PG_CLEANCHK); 850 } 851 852 } else { /* by hash */ 853 for (curoff = start ; curoff < stop; 854 curoff += PAGE_SIZE) { 855 pp = uvm_pagelookup(uobj, curoff); 856 if (pp) 857 atomic_clearbits_int(&pp->pg_flags, 858 PG_CLEANCHK); 859 } 860 } 861 } 862 863 /* 864 * now do it. note: we must update ppnext in body of loop or we 865 * will get stuck. we need to use ppnext because we may free "pp" 866 * before doing the next loop. 867 */ 868 869 if (by_list) { 870 pp = TAILQ_FIRST(&uobj->memq); 871 } else { 872 curoff = start; 873 pp = uvm_pagelookup(uobj, curoff); 874 } 875 876 ppnext = NULL; /* XXX: shut up gcc */ 877 ppsp = NULL; /* XXX: shut up gcc */ 878 uvm_lock_pageq(); /* page queues locked */ 879 880 /* locked: both page queues and uobj */ 881 for ( ; (by_list && pp != NULL) || 882 (!by_list && curoff < stop) ; pp = ppnext) { 883 884 if (by_list) { 885 886 /* 887 * range check 888 */ 889 890 if (!all && 891 (pp->offset < start || pp->offset >= stop)) { 892 ppnext = TAILQ_NEXT(pp, listq); 893 continue; 894 } 895 896 } else { 897 898 /* 899 * null check 900 */ 901 902 curoff += PAGE_SIZE; 903 if (pp == NULL) { 904 if (curoff < stop) 905 ppnext = uvm_pagelookup(uobj, curoff); 906 continue; 907 } 908 909 } 910 911 /* 912 * handle case where we do not need to clean page (either 913 * because we are not clean or because page is not dirty or 914 * is busy): 915 * 916 * NOTE: we are allowed to deactivate a non-wired active 917 * PG_BUSY page, but once a PG_BUSY page is on the inactive 918 * queue it must stay put until it is !PG_BUSY (so as not to 919 * confuse pagedaemon). 920 */ 921 922 if ((flags & PGO_CLEANIT) == 0 || (pp->pg_flags & PG_BUSY) != 0) { 923 needs_clean = FALSE; 924 if ((pp->pg_flags & PG_BUSY) != 0 && 925 (flags & (PGO_CLEANIT|PGO_SYNCIO)) == 926 (PGO_CLEANIT|PGO_SYNCIO)) 927 need_iosync = TRUE; 928 } else { 929 /* 930 * freeing: nuke all mappings so we can sync 931 * PG_CLEAN bit with no race 932 */ 933 if ((pp->pg_flags & PG_CLEAN) != 0 && 934 (flags & PGO_FREE) != 0 && 935 (pp->pg_flags & PQ_ACTIVE) != 0) 936 pmap_page_protect(pp, VM_PROT_NONE); 937 if ((pp->pg_flags & PG_CLEAN) != 0 && 938 pmap_is_modified(pp)) 939 atomic_clearbits_int(&pp->pg_flags, PG_CLEAN); 940 atomic_setbits_int(&pp->pg_flags, PG_CLEANCHK); 941 942 needs_clean = ((pp->pg_flags & PG_CLEAN) == 0); 943 } 944 945 /* 946 * if we don't need a clean... load ppnext and dispose of pp 947 */ 948 if (!needs_clean) { 949 /* load ppnext */ 950 if (by_list) 951 ppnext = TAILQ_NEXT(pp, listq); 952 else { 953 if (curoff < stop) 954 ppnext = uvm_pagelookup(uobj, curoff); 955 } 956 957 /* now dispose of pp */ 958 if (flags & PGO_DEACTIVATE) { 959 if ((pp->pg_flags & PQ_INACTIVE) == 0 && 960 pp->wire_count == 0) { 961 pmap_page_protect(pp, VM_PROT_NONE); 962 uvm_pagedeactivate(pp); 963 } 964 965 } else if (flags & PGO_FREE) { 966 if (pp->pg_flags & PG_BUSY) { 967 /* release busy pages */ 968 atomic_setbits_int(&pp->pg_flags, 969 PG_RELEASED); 970 } else { 971 pmap_page_protect(pp, VM_PROT_NONE); 972 /* removed page from object */ 973 uvm_pagefree(pp); 974 } 975 } 976 /* ppnext is valid so we can continue... */ 977 continue; 978 } 979 980 /* 981 * pp points to a page in the locked object that we are 982 * working on. if it is !PG_CLEAN,!PG_BUSY and we asked 983 * for cleaning (PGO_CLEANIT). we clean it now. 984 * 985 * let uvm_pager_put attempted a clustered page out. 986 * note: locked: uobj and page queues. 987 */ 988 989 atomic_setbits_int(&pp->pg_flags, PG_BUSY); 990 UVM_PAGE_OWN(pp, "uvn_flush"); 991 pmap_page_protect(pp, VM_PROT_READ); 992 pp_version = pp->pg_version; 993 ReTry: 994 ppsp = pps; 995 npages = sizeof(pps) / sizeof(struct vm_page *); 996 997 /* locked: page queues, uobj */ 998 result = uvm_pager_put(uobj, pp, &ppsp, &npages, 999 flags | PGO_DOACTCLUST, start, stop); 1000 /* unlocked: page queues, uobj */ 1001 1002 /* 1003 * at this point nothing is locked. if we did an async I/O 1004 * it is remotely possible for the async i/o to complete and 1005 * the page "pp" be freed or what not before we get a chance 1006 * to relock the object. in order to detect this, we have 1007 * saved the version number of the page in "pp_version". 1008 */ 1009 1010 /* relock! */ 1011 simple_lock(&uobj->vmobjlock); 1012 uvm_lock_pageq(); 1013 1014 /* 1015 * VM_PAGER_AGAIN: given the structure of this pager, this 1016 * can only happen when we are doing async I/O and can't 1017 * map the pages into kernel memory (pager_map) due to lack 1018 * of vm space. if this happens we drop back to sync I/O. 1019 */ 1020 1021 if (result == VM_PAGER_AGAIN) { 1022 /* 1023 * it is unlikely, but page could have been released 1024 * while we had the object lock dropped. we ignore 1025 * this now and retry the I/O. we will detect and 1026 * handle the released page after the syncio I/O 1027 * completes. 1028 */ 1029 #ifdef DIAGNOSTIC 1030 if (flags & PGO_SYNCIO) 1031 panic("uvn_flush: PGO_SYNCIO return 'try again' error (impossible)"); 1032 #endif 1033 flags |= PGO_SYNCIO; 1034 goto ReTry; 1035 } 1036 1037 /* 1038 * the cleaning operation is now done. finish up. note that 1039 * on error (!OK, !PEND) uvm_pager_put drops the cluster for us. 1040 * if success (OK, PEND) then uvm_pager_put returns the cluster 1041 * to us in ppsp/npages. 1042 */ 1043 1044 /* 1045 * for pending async i/o if we are not deactivating/freeing 1046 * we can move on to the next page. 1047 */ 1048 1049 if (result == VM_PAGER_PEND) { 1050 1051 if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) { 1052 /* 1053 * no per-page ops: refresh ppnext and continue 1054 */ 1055 if (by_list) { 1056 if (pp->pg_version == pp_version) 1057 ppnext = TAILQ_NEXT(pp, listq); 1058 else 1059 /* reset */ 1060 ppnext = TAILQ_FIRST(&uobj->memq); 1061 } else { 1062 if (curoff < stop) 1063 ppnext = uvm_pagelookup(uobj, 1064 curoff); 1065 } 1066 continue; 1067 } 1068 1069 /* need to do anything here? */ 1070 } 1071 1072 /* 1073 * need to look at each page of the I/O operation. we defer 1074 * processing "pp" until the last trip through this "for" loop 1075 * so that we can load "ppnext" for the main loop after we 1076 * play with the cluster pages [thus the "npages + 1" in the 1077 * loop below]. 1078 */ 1079 1080 for (lcv = 0 ; lcv < npages + 1 ; lcv++) { 1081 1082 /* 1083 * handle ppnext for outside loop, and saving pp 1084 * until the end. 1085 */ 1086 if (lcv < npages) { 1087 if (ppsp[lcv] == pp) 1088 continue; /* skip pp until the end */ 1089 ptmp = ppsp[lcv]; 1090 } else { 1091 ptmp = pp; 1092 1093 /* set up next page for outer loop */ 1094 if (by_list) { 1095 if (pp->pg_version == pp_version) 1096 ppnext = TAILQ_NEXT(pp, listq); 1097 else 1098 /* reset */ 1099 ppnext = TAILQ_FIRST(&uobj->memq); 1100 } else { 1101 if (curoff < stop) 1102 ppnext = uvm_pagelookup(uobj, curoff); 1103 } 1104 } 1105 1106 /* 1107 * verify the page didn't get moved while obj was 1108 * unlocked 1109 */ 1110 if (result == VM_PAGER_PEND && ptmp->uobject != uobj) 1111 continue; 1112 1113 /* 1114 * unbusy the page if I/O is done. note that for 1115 * pending I/O it is possible that the I/O op 1116 * finished before we relocked the object (in 1117 * which case the page is no longer busy). 1118 */ 1119 1120 if (result != VM_PAGER_PEND) { 1121 if (ptmp->pg_flags & PG_WANTED) 1122 /* still holding object lock */ 1123 wakeup(ptmp); 1124 1125 atomic_clearbits_int(&ptmp->pg_flags, 1126 PG_WANTED|PG_BUSY); 1127 UVM_PAGE_OWN(ptmp, NULL); 1128 if (ptmp->pg_flags & PG_RELEASED) { 1129 1130 /* pgo_releasepg wants this */ 1131 uvm_unlock_pageq(); 1132 if (!uvn_releasepg(ptmp, NULL)) 1133 return (TRUE); 1134 1135 uvm_lock_pageq(); /* relock */ 1136 continue; /* next page */ 1137 1138 } else { 1139 atomic_setbits_int(&ptmp->pg_flags, 1140 PG_CLEAN|PG_CLEANCHK); 1141 if ((flags & PGO_FREE) == 0) 1142 pmap_clear_modify(ptmp); 1143 } 1144 } 1145 1146 /* 1147 * dispose of page 1148 */ 1149 1150 if (flags & PGO_DEACTIVATE) { 1151 if ((pp->pg_flags & PQ_INACTIVE) == 0 && 1152 pp->wire_count == 0) { 1153 pmap_page_protect(ptmp, VM_PROT_NONE); 1154 uvm_pagedeactivate(ptmp); 1155 } 1156 1157 } else if (flags & PGO_FREE) { 1158 if (result == VM_PAGER_PEND) { 1159 if ((ptmp->pg_flags & PG_BUSY) != 0) 1160 /* signal for i/o done */ 1161 atomic_setbits_int( 1162 &ptmp->pg_flags, 1163 PG_RELEASED); 1164 } else { 1165 if (result != VM_PAGER_OK) { 1166 printf("uvn_flush: obj=%p, " 1167 "offset=0x%llx. error " 1168 "during pageout.\n", 1169 pp->uobject, 1170 (long long)pp->offset); 1171 printf("uvn_flush: WARNING: " 1172 "changes to page may be " 1173 "lost!\n"); 1174 retval = FALSE; 1175 } 1176 pmap_page_protect(ptmp, VM_PROT_NONE); 1177 uvm_pagefree(ptmp); 1178 } 1179 } 1180 1181 } /* end of "lcv" for loop */ 1182 1183 } /* end of "pp" for loop */ 1184 1185 /* 1186 * done with pagequeues: unlock 1187 */ 1188 uvm_unlock_pageq(); 1189 1190 /* 1191 * now wait for all I/O if required. 1192 */ 1193 if (need_iosync) { 1194 1195 UVMHIST_LOG(maphist," <<DOING IOSYNC>>",0,0,0,0); 1196 while (uvn->u_nio != 0) { 1197 uvn->u_flags |= UVM_VNODE_IOSYNC; 1198 UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock, 1199 FALSE, "uvn_flush",0); 1200 simple_lock(&uvn->u_obj.vmobjlock); 1201 } 1202 if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED) 1203 wakeup(&uvn->u_flags); 1204 uvn->u_flags &= ~(UVM_VNODE_IOSYNC|UVM_VNODE_IOSYNCWANTED); 1205 } 1206 1207 /* return, with object locked! */ 1208 UVMHIST_LOG(maphist,"<- done (retval=0x%lx)",retval,0,0,0); 1209 return(retval); 1210 } 1211 1212 /* 1213 * uvn_cluster 1214 * 1215 * we are about to do I/O in an object at offset. this function is called 1216 * to establish a range of offsets around "offset" in which we can cluster 1217 * I/O. 1218 * 1219 * - currently doesn't matter if obj locked or not. 1220 */ 1221 1222 void 1223 uvn_cluster(struct uvm_object *uobj, voff_t offset, voff_t *loffset, 1224 voff_t *hoffset) 1225 { 1226 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 1227 *loffset = offset; 1228 1229 if (*loffset >= uvn->u_size) 1230 panic("uvn_cluster: offset out of range"); 1231 1232 /* 1233 * XXX: old pager claims we could use VOP_BMAP to get maxcontig value. 1234 */ 1235 *hoffset = *loffset + MAXBSIZE; 1236 if (*hoffset > round_page(uvn->u_size)) /* past end? */ 1237 *hoffset = round_page(uvn->u_size); 1238 1239 return; 1240 } 1241 1242 /* 1243 * uvn_put: flush page data to backing store. 1244 * 1245 * => prefer map unlocked (not required) 1246 * => object must be locked! we will _unlock_ it before starting I/O. 1247 * => flags: PGO_SYNCIO -- use sync. I/O 1248 * => note: caller must set PG_CLEAN and pmap_clear_modify (if needed) 1249 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync. 1250 * [thus we never do async i/o! see iodone comment] 1251 */ 1252 1253 int 1254 uvn_put(struct uvm_object *uobj, struct vm_page **pps, int npages, int flags) 1255 { 1256 int retval; 1257 1258 /* note: object locked */ 1259 retval = uvn_io((struct uvm_vnode*)uobj, pps, npages, flags, UIO_WRITE); 1260 /* note: object unlocked */ 1261 1262 return(retval); 1263 } 1264 1265 1266 /* 1267 * uvn_get: get pages (synchronously) from backing store 1268 * 1269 * => prefer map unlocked (not required) 1270 * => object must be locked! we will _unlock_ it before starting any I/O. 1271 * => flags: PGO_ALLPAGES: get all of the pages 1272 * PGO_LOCKED: fault data structures are locked 1273 * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx] 1274 * => NOTE: caller must check for released pages!! 1275 */ 1276 1277 int 1278 uvn_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps, 1279 int *npagesp, int centeridx, vm_prot_t access_type, int advice, int flags) 1280 { 1281 voff_t current_offset; 1282 struct vm_page *ptmp; 1283 int lcv, result, gotpages; 1284 boolean_t done; 1285 UVMHIST_FUNC("uvn_get"); UVMHIST_CALLED(maphist); 1286 UVMHIST_LOG(maphist, "flags=%ld", flags,0,0,0); 1287 1288 /* 1289 * step 1: handled the case where fault data structures are locked. 1290 */ 1291 1292 if (flags & PGO_LOCKED) { 1293 1294 /* 1295 * gotpages is the current number of pages we've gotten (which 1296 * we pass back up to caller via *npagesp. 1297 */ 1298 1299 gotpages = 0; 1300 1301 /* 1302 * step 1a: get pages that are already resident. only do this 1303 * if the data structures are locked (i.e. the first time 1304 * through). 1305 */ 1306 1307 done = TRUE; /* be optimistic */ 1308 1309 for (lcv = 0, current_offset = offset ; lcv < *npagesp ; 1310 lcv++, current_offset += PAGE_SIZE) { 1311 1312 /* do we care about this page? if not, skip it */ 1313 if (pps[lcv] == PGO_DONTCARE) 1314 continue; 1315 1316 /* lookup page */ 1317 ptmp = uvm_pagelookup(uobj, current_offset); 1318 1319 /* to be useful must get a non-busy, non-released pg */ 1320 if (ptmp == NULL || 1321 (ptmp->pg_flags & (PG_BUSY|PG_RELEASED)) != 0) { 1322 if (lcv == centeridx || (flags & PGO_ALLPAGES) 1323 != 0) 1324 done = FALSE; /* need to do a wait or I/O! */ 1325 continue; 1326 } 1327 1328 /* 1329 * useful page: busy/lock it and plug it in our 1330 * result array 1331 */ 1332 atomic_setbits_int(&ptmp->pg_flags, PG_BUSY); 1333 UVM_PAGE_OWN(ptmp, "uvn_get1"); 1334 pps[lcv] = ptmp; 1335 gotpages++; 1336 1337 } /* "for" lcv loop */ 1338 1339 /* 1340 * XXX: given the "advice", should we consider async read-ahead? 1341 * XXX: fault current does deactive of pages behind us. is 1342 * this good (other callers might now). 1343 */ 1344 /* 1345 * XXX: read-ahead currently handled by buffer cache (bread) 1346 * level. 1347 * XXX: no async i/o available. 1348 * XXX: so we don't do anything now. 1349 */ 1350 1351 /* 1352 * step 1c: now we've either done everything needed or we to 1353 * unlock and do some waiting or I/O. 1354 */ 1355 1356 *npagesp = gotpages; /* let caller know */ 1357 if (done) 1358 return(VM_PAGER_OK); /* bingo! */ 1359 else 1360 /* EEK! Need to unlock and I/O */ 1361 return(VM_PAGER_UNLOCK); 1362 } 1363 1364 /* 1365 * step 2: get non-resident or busy pages. 1366 * object is locked. data structures are unlocked. 1367 * 1368 * XXX: because we can't do async I/O at this level we get things 1369 * page at a time (otherwise we'd chunk). the VOP_READ() will do 1370 * async-read-ahead for us at a lower level. 1371 */ 1372 1373 for (lcv = 0, current_offset = offset; 1374 lcv < *npagesp ; lcv++, current_offset += PAGE_SIZE) { 1375 1376 /* skip over pages we've already gotten or don't want */ 1377 /* skip over pages we don't _have_ to get */ 1378 if (pps[lcv] != NULL || (lcv != centeridx && 1379 (flags & PGO_ALLPAGES) == 0)) 1380 continue; 1381 1382 /* 1383 * we have yet to locate the current page (pps[lcv]). we first 1384 * look for a page that is already at the current offset. if 1385 * we fine a page, we check to see if it is busy or released. 1386 * if that is the case, then we sleep on the page until it is 1387 * no longer busy or released and repeat the lookup. if the 1388 * page we found is neither busy nor released, then we busy it 1389 * (so we own it) and plug it into pps[lcv]. this breaks the 1390 * following while loop and indicates we are ready to move on 1391 * to the next page in the "lcv" loop above. 1392 * 1393 * if we exit the while loop with pps[lcv] still set to NULL, 1394 * then it means that we allocated a new busy/fake/clean page 1395 * ptmp in the object and we need to do I/O to fill in the data. 1396 */ 1397 1398 while (pps[lcv] == NULL) { /* top of "pps" while loop */ 1399 1400 /* look for a current page */ 1401 ptmp = uvm_pagelookup(uobj, current_offset); 1402 1403 /* nope? allocate one now (if we can) */ 1404 if (ptmp == NULL) { 1405 1406 ptmp = uvm_pagealloc(uobj, current_offset, 1407 NULL, 0); 1408 1409 /* out of RAM? */ 1410 if (ptmp == NULL) { 1411 simple_unlock(&uobj->vmobjlock); 1412 uvm_wait("uvn_getpage"); 1413 simple_lock(&uobj->vmobjlock); 1414 1415 /* goto top of pps while loop */ 1416 continue; 1417 } 1418 1419 /* 1420 * got new page ready for I/O. break pps 1421 * while loop. pps[lcv] is still NULL. 1422 */ 1423 break; 1424 } 1425 1426 /* page is there, see if we need to wait on it */ 1427 if ((ptmp->pg_flags & (PG_BUSY|PG_RELEASED)) != 0) { 1428 atomic_setbits_int(&ptmp->pg_flags, PG_WANTED); 1429 UVM_UNLOCK_AND_WAIT(ptmp, 1430 &uobj->vmobjlock, FALSE, "uvn_get",0); 1431 simple_lock(&uobj->vmobjlock); 1432 continue; /* goto top of pps while loop */ 1433 } 1434 1435 /* 1436 * if we get here then the page has become resident 1437 * and unbusy between steps 1 and 2. we busy it 1438 * now (so we own it) and set pps[lcv] (so that we 1439 * exit the while loop). 1440 */ 1441 atomic_setbits_int(&ptmp->pg_flags, PG_BUSY); 1442 UVM_PAGE_OWN(ptmp, "uvn_get2"); 1443 pps[lcv] = ptmp; 1444 } 1445 1446 /* 1447 * if we own the a valid page at the correct offset, pps[lcv] 1448 * will point to it. nothing more to do except go to the 1449 * next page. 1450 */ 1451 1452 if (pps[lcv]) 1453 continue; /* next lcv */ 1454 1455 /* 1456 * we have a "fake/busy/clean" page that we just allocated. do 1457 * I/O to fill it with valid data. note that object must be 1458 * locked going into uvn_io, but will be unlocked afterwards. 1459 */ 1460 1461 result = uvn_io((struct uvm_vnode *) uobj, &ptmp, 1, 1462 PGO_SYNCIO, UIO_READ); 1463 1464 /* 1465 * I/O done. object is unlocked (by uvn_io). because we used 1466 * syncio the result can not be PEND or AGAIN. we must relock 1467 * and check for errors. 1468 */ 1469 1470 /* lock object. check for errors. */ 1471 simple_lock(&uobj->vmobjlock); 1472 if (result != VM_PAGER_OK) { 1473 if (ptmp->pg_flags & PG_WANTED) 1474 /* object lock still held */ 1475 wakeup(ptmp); 1476 1477 atomic_clearbits_int(&ptmp->pg_flags, 1478 PG_WANTED|PG_BUSY); 1479 UVM_PAGE_OWN(ptmp, NULL); 1480 uvm_lock_pageq(); 1481 uvm_pagefree(ptmp); 1482 uvm_unlock_pageq(); 1483 simple_unlock(&uobj->vmobjlock); 1484 return(result); 1485 } 1486 1487 /* 1488 * we got the page! clear the fake flag (indicates valid 1489 * data now in page) and plug into our result array. note 1490 * that page is still busy. 1491 * 1492 * it is the callers job to: 1493 * => check if the page is released 1494 * => unbusy the page 1495 * => activate the page 1496 */ 1497 1498 /* data is valid ... */ 1499 atomic_clearbits_int(&ptmp->pg_flags, PG_FAKE); 1500 pmap_clear_modify(ptmp); /* ... and clean */ 1501 pps[lcv] = ptmp; 1502 1503 } /* lcv loop */ 1504 1505 /* 1506 * finally, unlock object and return. 1507 */ 1508 1509 simple_unlock(&uobj->vmobjlock); 1510 return (VM_PAGER_OK); 1511 } 1512 1513 /* 1514 * uvn_io: do I/O to a vnode 1515 * 1516 * => prefer map unlocked (not required) 1517 * => object must be locked! we will _unlock_ it before starting I/O. 1518 * => flags: PGO_SYNCIO -- use sync. I/O 1519 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync. 1520 * [thus we never do async i/o! see iodone comment] 1521 */ 1522 1523 int 1524 uvn_io(struct uvm_vnode *uvn, vm_page_t *pps, int npages, int flags, int rw) 1525 { 1526 struct vnode *vn; 1527 struct uio uio; 1528 struct iovec iov; 1529 vaddr_t kva; 1530 off_t file_offset; 1531 int waitf, result, mapinflags; 1532 size_t got, wanted; 1533 UVMHIST_FUNC("uvn_io"); UVMHIST_CALLED(maphist); 1534 1535 UVMHIST_LOG(maphist, "rw=%ld", rw,0,0,0); 1536 1537 /* 1538 * init values 1539 */ 1540 1541 waitf = (flags & PGO_SYNCIO) ? M_WAITOK : M_NOWAIT; 1542 vn = (struct vnode *) uvn; 1543 file_offset = pps[0]->offset; 1544 1545 /* 1546 * check for sync'ing I/O. 1547 */ 1548 1549 while (uvn->u_flags & UVM_VNODE_IOSYNC) { 1550 if (waitf == M_NOWAIT) { 1551 simple_unlock(&uvn->u_obj.vmobjlock); 1552 UVMHIST_LOG(maphist,"<- try again (iosync)",0,0,0,0); 1553 return(VM_PAGER_AGAIN); 1554 } 1555 uvn->u_flags |= UVM_VNODE_IOSYNCWANTED; 1556 UVM_UNLOCK_AND_WAIT(&uvn->u_flags, &uvn->u_obj.vmobjlock, 1557 FALSE, "uvn_iosync",0); 1558 simple_lock(&uvn->u_obj.vmobjlock); 1559 } 1560 1561 /* 1562 * check size 1563 */ 1564 1565 if (file_offset >= uvn->u_size) { 1566 simple_unlock(&uvn->u_obj.vmobjlock); 1567 UVMHIST_LOG(maphist,"<- BAD (size check)",0,0,0,0); 1568 return(VM_PAGER_BAD); 1569 } 1570 1571 /* 1572 * first try and map the pages in (without waiting) 1573 */ 1574 1575 mapinflags = (rw == UIO_READ) ? 1576 UVMPAGER_MAPIN_READ : UVMPAGER_MAPIN_WRITE; 1577 1578 kva = uvm_pagermapin(pps, npages, mapinflags); 1579 if (kva == 0 && waitf == M_NOWAIT) { 1580 simple_unlock(&uvn->u_obj.vmobjlock); 1581 UVMHIST_LOG(maphist,"<- mapin failed (try again)",0,0,0,0); 1582 return(VM_PAGER_AGAIN); 1583 } 1584 1585 /* 1586 * ok, now bump u_nio up. at this point we are done with uvn 1587 * and can unlock it. if we still don't have a kva, try again 1588 * (this time with sleep ok). 1589 */ 1590 1591 uvn->u_nio++; /* we have an I/O in progress! */ 1592 simple_unlock(&uvn->u_obj.vmobjlock); 1593 /* NOTE: object now unlocked */ 1594 if (kva == 0) 1595 kva = uvm_pagermapin(pps, npages, 1596 mapinflags | UVMPAGER_MAPIN_WAITOK); 1597 1598 /* 1599 * ok, mapped in. our pages are PG_BUSY so they are not going to 1600 * get touched (so we can look at "offset" without having to lock 1601 * the object). set up for I/O. 1602 */ 1603 1604 /* 1605 * fill out uio/iov 1606 */ 1607 1608 iov.iov_base = (caddr_t) kva; 1609 wanted = npages << PAGE_SHIFT; 1610 if (file_offset + wanted > uvn->u_size) 1611 wanted = uvn->u_size - file_offset; /* XXX: needed? */ 1612 iov.iov_len = wanted; 1613 uio.uio_iov = &iov; 1614 uio.uio_iovcnt = 1; 1615 uio.uio_offset = file_offset; 1616 uio.uio_segflg = UIO_SYSSPACE; 1617 uio.uio_rw = rw; 1618 uio.uio_resid = wanted; 1619 uio.uio_procp = curproc; 1620 1621 /* 1622 * do the I/O! (XXX: curproc?) 1623 */ 1624 1625 UVMHIST_LOG(maphist, "calling VOP",0,0,0,0); 1626 1627 /* 1628 * This process may already have this vnode locked, if we faulted in 1629 * copyin() or copyout() on a region backed by this vnode 1630 * while doing I/O to the vnode. If this is the case, don't 1631 * panic.. instead, return the error to the user. 1632 * 1633 * XXX this is a stopgap to prevent a panic. 1634 * Ideally, this kind of operation *should* work. 1635 */ 1636 result = 0; 1637 if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0) 1638 result = vn_lock(vn, LK_EXCLUSIVE | LK_RECURSEFAIL, curproc); 1639 1640 if (result == 0) { 1641 /* NOTE: vnode now locked! */ 1642 1643 if (rw == UIO_READ) 1644 result = VOP_READ(vn, &uio, 0, curproc->p_ucred); 1645 else 1646 result = VOP_WRITE(vn, &uio, 0, curproc->p_ucred); 1647 1648 if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0) 1649 VOP_UNLOCK(vn, 0, curproc); 1650 } 1651 1652 /* NOTE: vnode now unlocked (unless vnislocked) */ 1653 1654 UVMHIST_LOG(maphist, "done calling VOP",0,0,0,0); 1655 1656 /* 1657 * result == unix style errno (0 == OK!) 1658 * 1659 * zero out rest of buffer (if needed) 1660 */ 1661 1662 if (result == 0) { 1663 got = wanted - uio.uio_resid; 1664 1665 if (wanted && got == 0) { 1666 result = EIO; /* XXX: error? */ 1667 } else if (got < PAGE_SIZE * npages && rw == UIO_READ) { 1668 memset((void *) (kva + got), 0, 1669 (npages << PAGE_SHIFT) - got); 1670 } 1671 } 1672 1673 /* 1674 * now remove pager mapping 1675 */ 1676 uvm_pagermapout(kva, npages); 1677 1678 /* 1679 * now clean up the object (i.e. drop I/O count) 1680 */ 1681 1682 simple_lock(&uvn->u_obj.vmobjlock); 1683 /* NOTE: object now locked! */ 1684 1685 uvn->u_nio--; /* I/O DONE! */ 1686 if ((uvn->u_flags & UVM_VNODE_IOSYNC) != 0 && uvn->u_nio == 0) { 1687 wakeup(&uvn->u_nio); 1688 } 1689 simple_unlock(&uvn->u_obj.vmobjlock); 1690 /* NOTE: object now unlocked! */ 1691 1692 /* 1693 * done! 1694 */ 1695 1696 UVMHIST_LOG(maphist, "<- done (result %ld)", result,0,0,0); 1697 if (result == 0) 1698 return(VM_PAGER_OK); 1699 else 1700 return(VM_PAGER_ERROR); 1701 } 1702 1703 /* 1704 * uvm_vnp_uncache: disable "persisting" in a vnode... when last reference 1705 * is gone we will kill the object (flushing dirty pages back to the vnode 1706 * if needed). 1707 * 1708 * => returns TRUE if there was no uvm_object attached or if there was 1709 * one and we killed it [i.e. if there is no active uvn] 1710 * => called with the vnode VOP_LOCK'd [we will unlock it for I/O, if 1711 * needed] 1712 * 1713 * => XXX: given that we now kill uvn's when a vnode is recycled (without 1714 * having to hold a reference on the vnode) and given a working 1715 * uvm_vnp_sync(), how does that effect the need for this function? 1716 * [XXXCDC: seems like it can die?] 1717 * 1718 * => XXX: this function should DIE once we merge the VM and buffer 1719 * cache. 1720 * 1721 * research shows that this is called in the following places: 1722 * ext2fs_truncate, ffs_truncate, detrunc[msdosfs]: called when vnode 1723 * changes sizes 1724 * ext2fs_write, WRITE [ufs_readwrite], msdosfs_write: called when we 1725 * are written to 1726 * ex2fs_chmod, ufs_chmod: called if VTEXT vnode and the sticky bit 1727 * is off 1728 * ffs_realloccg: when we can't extend the current block and have 1729 * to allocate a new one we call this [XXX: why?] 1730 * nfsrv_rename, rename_files: called when the target filename is there 1731 * and we want to remove it 1732 * nfsrv_remove, sys_unlink: called on file we are removing 1733 * nfsrv_access: if VTEXT and we want WRITE access and we don't uncache 1734 * then return "text busy" 1735 * nfs_open: seems to uncache any file opened with nfs 1736 * vn_writechk: if VTEXT vnode and can't uncache return "text busy" 1737 */ 1738 1739 boolean_t 1740 uvm_vnp_uncache(struct vnode *vp) 1741 { 1742 struct uvm_vnode *uvn = &vp->v_uvm; 1743 1744 /* 1745 * lock uvn part of the vnode and check to see if we need to do anything 1746 */ 1747 1748 simple_lock(&uvn->u_obj.vmobjlock); 1749 if ((uvn->u_flags & UVM_VNODE_VALID) == 0 || 1750 (uvn->u_flags & UVM_VNODE_BLOCKED) != 0) { 1751 simple_unlock(&uvn->u_obj.vmobjlock); 1752 return(TRUE); 1753 } 1754 1755 /* 1756 * we have a valid, non-blocked uvn. clear persist flag. 1757 * if uvn is currently active we can return now. 1758 */ 1759 1760 uvn->u_flags &= ~UVM_VNODE_CANPERSIST; 1761 if (uvn->u_obj.uo_refs) { 1762 simple_unlock(&uvn->u_obj.vmobjlock); 1763 return(FALSE); 1764 } 1765 1766 /* 1767 * uvn is currently persisting! we have to gain a reference to 1768 * it so that we can call uvn_detach to kill the uvn. 1769 */ 1770 1771 VREF(vp); /* seems ok, even with VOP_LOCK */ 1772 uvn->u_obj.uo_refs++; /* value is now 1 */ 1773 simple_unlock(&uvn->u_obj.vmobjlock); 1774 1775 #ifdef VFSDEBUG 1776 /* 1777 * carry over sanity check from old vnode pager: the vnode should 1778 * be VOP_LOCK'd, and we confirm it here. 1779 */ 1780 if ((vp->v_flag & VLOCKSWORK) && !VOP_ISLOCKED(vp)) 1781 panic("uvm_vnp_uncache: vnode not locked!"); 1782 #endif /* VFSDEBUG */ 1783 1784 /* 1785 * now drop our reference to the vnode. if we have the sole 1786 * reference to the vnode then this will cause it to die [as we 1787 * just cleared the persist flag]. we have to unlock the vnode 1788 * while we are doing this as it may trigger I/O. 1789 * 1790 * XXX: it might be possible for uvn to get reclaimed while we are 1791 * unlocked causing us to return TRUE when we should not. we ignore 1792 * this as a false-positive return value doesn't hurt us. 1793 */ 1794 VOP_UNLOCK(vp, 0, curproc); 1795 uvn_detach(&uvn->u_obj); 1796 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, curproc); 1797 1798 /* 1799 * and return... 1800 */ 1801 1802 return(TRUE); 1803 } 1804 1805 /* 1806 * uvm_vnp_setsize: grow or shrink a vnode uvn 1807 * 1808 * grow => just update size value 1809 * shrink => toss un-needed pages 1810 * 1811 * => we assume that the caller has a reference of some sort to the 1812 * vnode in question so that it will not be yanked out from under 1813 * us. 1814 * 1815 * called from: 1816 * => truncate fns (ext2fs_truncate, ffs_truncate, detrunc[msdos]) 1817 * => "write" fns (ext2fs_write, WRITE [ufs/ufs], msdosfs_write, nfs_write) 1818 * => ffs_balloc [XXX: why? doesn't WRITE handle?] 1819 * => NFS: nfs_loadattrcache, nfs_getattrcache, nfs_setattr 1820 * => union fs: union_newsize 1821 */ 1822 1823 void 1824 uvm_vnp_setsize(struct vnode *vp, voff_t newsize) 1825 { 1826 struct uvm_vnode *uvn = &vp->v_uvm; 1827 1828 /* 1829 * lock uvn and check for valid object, and if valid: do it! 1830 */ 1831 simple_lock(&uvn->u_obj.vmobjlock); 1832 if (uvn->u_flags & UVM_VNODE_VALID) { 1833 1834 /* 1835 * now check if the size has changed: if we shrink we had better 1836 * toss some pages... 1837 */ 1838 1839 if (uvn->u_size > newsize) { 1840 (void)uvn_flush(&uvn->u_obj, newsize, 1841 uvn->u_size, PGO_FREE); 1842 } 1843 uvn->u_size = newsize; 1844 } 1845 simple_unlock(&uvn->u_obj.vmobjlock); 1846 1847 /* 1848 * done 1849 */ 1850 return; 1851 } 1852 1853 /* 1854 * uvm_vnp_sync: flush all dirty VM pages back to their backing vnodes. 1855 * 1856 * => called from sys_sync with no VM structures locked 1857 * => only one process can do a sync at a time (because the uvn 1858 * structure only has one queue for sync'ing). we ensure this 1859 * by holding the uvn_sync_lock while the sync is in progress. 1860 * other processes attempting a sync will sleep on this lock 1861 * until we are done. 1862 */ 1863 void 1864 uvm_vnp_sync(struct mount *mp) 1865 { 1866 struct uvm_vnode *uvn; 1867 struct vnode *vp; 1868 1869 /* 1870 * step 1: ensure we are only ones using the uvn_sync_q by locking 1871 * our lock... 1872 */ 1873 rw_enter_write(&uvn_sync_lock); 1874 1875 /* 1876 * step 2: build up a simpleq of uvns of interest based on the 1877 * write list. we gain a reference to uvns of interest. 1878 */ 1879 SIMPLEQ_INIT(&uvn_sync_q); 1880 LIST_FOREACH(uvn, &uvn_wlist, u_wlist) { 1881 1882 vp = (struct vnode *)uvn; 1883 if (mp && vp->v_mount != mp) 1884 continue; 1885 1886 /* 1887 * If the vnode is "blocked" it means it must be dying, which 1888 * in turn means its in the process of being flushed out so 1889 * we can safely skip it. 1890 * 1891 * note that uvn must already be valid because we found it on 1892 * the wlist (this also means it can't be ALOCK'd). 1893 */ 1894 if ((uvn->u_flags & UVM_VNODE_BLOCKED) != 0) 1895 continue; 1896 1897 1898 /* 1899 * gain reference. watch out for persisting uvns (need to 1900 * regain vnode REF). 1901 */ 1902 if (uvn->u_obj.uo_refs == 0) 1903 VREF(vp); 1904 uvn->u_obj.uo_refs++; 1905 1906 SIMPLEQ_INSERT_HEAD(&uvn_sync_q, uvn, u_syncq); 1907 } 1908 1909 /* step 3: we now have a list of uvn's that may need cleaning. */ 1910 SIMPLEQ_FOREACH(uvn, &uvn_sync_q, u_syncq) { 1911 #ifdef DEBUG 1912 if (uvn->u_flags & UVM_VNODE_DYING) { 1913 printf("uvm_vnp_sync: dying vnode on sync list\n"); 1914 } 1915 #endif 1916 uvn_flush(&uvn->u_obj, 0, 0, PGO_CLEANIT|PGO_ALLPAGES|PGO_DOACTCLUST); 1917 1918 /* 1919 * if we have the only reference and we just cleaned the uvn, 1920 * then we can pull it out of the UVM_VNODE_WRITEABLE state 1921 * thus allowing us to avoid thinking about flushing it again 1922 * on later sync ops. 1923 */ 1924 if (uvn->u_obj.uo_refs == 1 && 1925 (uvn->u_flags & UVM_VNODE_WRITEABLE)) { 1926 LIST_REMOVE(uvn, u_wlist); 1927 uvn->u_flags &= ~UVM_VNODE_WRITEABLE; 1928 } 1929 1930 /* now drop our reference to the uvn */ 1931 uvn_detach(&uvn->u_obj); 1932 } 1933 1934 rw_exit_write(&uvn_sync_lock); 1935 } 1936