1 /* $OpenBSD: uvm_vnode.c,v 1.51 2007/09/17 20:29:55 thib Exp $ */ 2 /* $NetBSD: uvm_vnode.c,v 1.36 2000/11/24 20:34:01 chs Exp $ */ 3 4 /* 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * Copyright (c) 1991, 1993 7 * The Regents of the University of California. 8 * Copyright (c) 1990 University of Utah. 9 * 10 * All rights reserved. 11 * 12 * This code is derived from software contributed to Berkeley by 13 * the Systems Programming Group of the University of Utah Computer 14 * Science Department. 15 * 16 * Redistribution and use in source and binary forms, with or without 17 * modification, are permitted provided that the following conditions 18 * are met: 19 * 1. Redistributions of source code must retain the above copyright 20 * notice, this list of conditions and the following disclaimer. 21 * 2. Redistributions in binary form must reproduce the above copyright 22 * notice, this list of conditions and the following disclaimer in the 23 * documentation and/or other materials provided with the distribution. 24 * 3. All advertising materials mentioning features or use of this software 25 * must display the following acknowledgement: 26 * This product includes software developed by Charles D. Cranor, 27 * Washington University, the University of California, Berkeley and 28 * its contributors. 29 * 4. Neither the name of the University nor the names of its contributors 30 * may be used to endorse or promote products derived from this software 31 * without specific prior written permission. 32 * 33 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 34 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 35 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 36 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 37 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 38 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 39 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 40 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 41 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 42 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 43 * SUCH DAMAGE. 44 * 45 * @(#)vnode_pager.c 8.8 (Berkeley) 2/13/94 46 * from: Id: uvm_vnode.c,v 1.1.2.26 1998/02/02 20:38:07 chuck Exp 47 */ 48 49 /* 50 * uvm_vnode.c: the vnode pager. 51 */ 52 53 #include <sys/param.h> 54 #include <sys/systm.h> 55 #include <sys/proc.h> 56 #include <sys/malloc.h> 57 #include <sys/vnode.h> 58 #include <sys/disklabel.h> 59 #include <sys/ioctl.h> 60 #include <sys/fcntl.h> 61 #include <sys/conf.h> 62 #include <sys/rwlock.h> 63 64 #include <miscfs/specfs/specdev.h> 65 66 #include <uvm/uvm.h> 67 #include <uvm/uvm_vnode.h> 68 69 /* 70 * private global data structure 71 * 72 * we keep a list of writeable active vnode-backed VM objects for sync op. 73 * we keep a simpleq of vnodes that are currently being sync'd. 74 */ 75 76 LIST_HEAD(uvn_list_struct, uvm_vnode); 77 struct uvn_list_struct uvn_wlist; /* writeable uvns */ 78 79 SIMPLEQ_HEAD(uvn_sq_struct, uvm_vnode); 80 struct uvn_sq_struct uvn_sync_q; /* sync'ing uvns */ 81 struct rwlock uvn_sync_lock; /* locks sync operation */ 82 83 /* 84 * functions 85 */ 86 87 void uvn_cluster(struct uvm_object *, voff_t, 88 voff_t *, voff_t *); 89 void uvn_detach(struct uvm_object *); 90 boolean_t uvn_flush(struct uvm_object *, voff_t, 91 voff_t, int); 92 int uvn_get(struct uvm_object *, voff_t, 93 vm_page_t *, int *, int, 94 vm_prot_t, int, int); 95 void uvn_init(void); 96 int uvn_io(struct uvm_vnode *, vm_page_t *, 97 int, int, int); 98 int uvn_put(struct uvm_object *, vm_page_t *, 99 int, boolean_t); 100 void uvn_reference(struct uvm_object *); 101 boolean_t uvn_releasepg(struct vm_page *, 102 struct vm_page **); 103 104 /* 105 * master pager structure 106 */ 107 108 struct uvm_pagerops uvm_vnodeops = { 109 uvn_init, 110 uvn_reference, 111 uvn_detach, 112 NULL, /* no specialized fault routine required */ 113 uvn_flush, 114 uvn_get, 115 uvn_put, 116 uvn_cluster, 117 uvm_mk_pcluster, /* use generic version of this: see uvm_pager.c */ 118 uvn_releasepg, 119 }; 120 121 /* 122 * the ops! 123 */ 124 125 /* 126 * uvn_init 127 * 128 * init pager private data structures. 129 */ 130 131 void 132 uvn_init(void) 133 { 134 135 LIST_INIT(&uvn_wlist); 136 /* note: uvn_sync_q init'd in uvm_vnp_sync() */ 137 rw_init(&uvn_sync_lock, "uvnsync"); 138 } 139 140 /* 141 * uvn_attach 142 * 143 * attach a vnode structure to a VM object. if the vnode is already 144 * attached, then just bump the reference count by one and return the 145 * VM object. if not already attached, attach and return the new VM obj. 146 * the "accessprot" tells the max access the attaching thread wants to 147 * our pages. 148 * 149 * => caller must _not_ already be holding the lock on the uvm_object. 150 * => in fact, nothing should be locked so that we can sleep here. 151 * => note that uvm_object is first thing in vnode structure, so their 152 * pointers are equiv. 153 */ 154 155 struct uvm_object * 156 uvn_attach(arg, accessprot) 157 void *arg; 158 vm_prot_t accessprot; 159 { 160 struct vnode *vp = arg; 161 struct uvm_vnode *uvn = &vp->v_uvm; 162 struct vattr vattr; 163 int oldflags, result; 164 struct partinfo pi; 165 u_quad_t used_vnode_size; 166 UVMHIST_FUNC("uvn_attach"); UVMHIST_CALLED(maphist); 167 168 UVMHIST_LOG(maphist, "(vn=%p)", arg,0,0,0); 169 170 used_vnode_size = (u_quad_t)0; /* XXX gcc -Wuninitialized */ 171 172 /* 173 * first get a lock on the uvn. 174 */ 175 simple_lock(&uvn->u_obj.vmobjlock); 176 while (uvn->u_flags & UVM_VNODE_BLOCKED) { 177 printf("uvn_attach: blocked at %p flags 0x%x\n", 178 uvn, uvn->u_flags); 179 uvn->u_flags |= UVM_VNODE_WANTED; 180 UVMHIST_LOG(maphist, " SLEEPING on blocked vn",0,0,0,0); 181 UVM_UNLOCK_AND_WAIT(uvn, &uvn->u_obj.vmobjlock, FALSE, 182 "uvn_attach", 0); 183 simple_lock(&uvn->u_obj.vmobjlock); 184 UVMHIST_LOG(maphist," WOKE UP",0,0,0,0); 185 } 186 187 /* 188 * if we're mapping a BLK device, make sure it is a disk. 189 */ 190 if (vp->v_type == VBLK && bdevsw[major(vp->v_rdev)].d_type != D_DISK) { 191 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock */ 192 UVMHIST_LOG(maphist,"<- done (VBLK not D_DISK!)", 0,0,0,0); 193 return(NULL); 194 } 195 196 /* 197 * now we have lock and uvn must not be in a blocked state. 198 * first check to see if it is already active, in which case 199 * we can bump the reference count, check to see if we need to 200 * add it to the writeable list, and then return. 201 */ 202 if (uvn->u_flags & UVM_VNODE_VALID) { /* already active? */ 203 204 /* regain VREF if we were persisting */ 205 if (uvn->u_obj.uo_refs == 0) { 206 VREF(vp); 207 UVMHIST_LOG(maphist," VREF (reclaim persisting vnode)", 208 0,0,0,0); 209 } 210 uvn->u_obj.uo_refs++; /* bump uvn ref! */ 211 212 /* check for new writeable uvn */ 213 if ((accessprot & VM_PROT_WRITE) != 0 && 214 (uvn->u_flags & UVM_VNODE_WRITEABLE) == 0) { 215 LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist); 216 /* we are now on wlist! */ 217 uvn->u_flags |= UVM_VNODE_WRITEABLE; 218 } 219 220 /* unlock and return */ 221 simple_unlock(&uvn->u_obj.vmobjlock); 222 UVMHIST_LOG(maphist,"<- done, refcnt=%ld", uvn->u_obj.uo_refs, 223 0, 0, 0); 224 return (&uvn->u_obj); 225 } 226 227 /* 228 * need to call VOP_GETATTR() to get the attributes, but that could 229 * block (due to I/O), so we want to unlock the object before calling. 230 * however, we want to keep anyone else from playing with the object 231 * while it is unlocked. to do this we set UVM_VNODE_ALOCK which 232 * prevents anyone from attaching to the vnode until we are done with 233 * it. 234 */ 235 uvn->u_flags = UVM_VNODE_ALOCK; 236 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock in case we sleep */ 237 /* XXX: curproc? */ 238 239 if (vp->v_type == VBLK) { 240 /* 241 * We could implement this as a specfs getattr call, but: 242 * 243 * (1) VOP_GETATTR() would get the file system 244 * vnode operation, not the specfs operation. 245 * 246 * (2) All we want is the size, anyhow. 247 */ 248 result = (*bdevsw[major(vp->v_rdev)].d_ioctl)(vp->v_rdev, 249 DIOCGPART, (caddr_t)&pi, FREAD, curproc); 250 if (result == 0) { 251 /* XXX should remember blocksize */ 252 used_vnode_size = (u_quad_t)pi.disklab->d_secsize * 253 (u_quad_t)DL_GETPSIZE(pi.part); 254 } 255 } else { 256 result = VOP_GETATTR(vp, &vattr, curproc->p_ucred, curproc); 257 if (result == 0) 258 used_vnode_size = vattr.va_size; 259 } 260 261 /* relock object */ 262 simple_lock(&uvn->u_obj.vmobjlock); 263 264 if (result != 0) { 265 if (uvn->u_flags & UVM_VNODE_WANTED) 266 wakeup(uvn); 267 uvn->u_flags = 0; 268 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock */ 269 UVMHIST_LOG(maphist,"<- done (VOP_GETATTR FAILED!)", 0,0,0,0); 270 return(NULL); 271 } 272 273 /* 274 * make sure that the newsize fits within a vaddr_t 275 * XXX: need to revise addressing data types 276 */ 277 #ifdef DEBUG 278 if (vp->v_type == VBLK) 279 printf("used_vnode_size = %llu\n", (long long)used_vnode_size); 280 #endif 281 282 /* 283 * now set up the uvn. 284 */ 285 uvn->u_obj.pgops = &uvm_vnodeops; 286 TAILQ_INIT(&uvn->u_obj.memq); 287 uvn->u_obj.uo_npages = 0; 288 uvn->u_obj.uo_refs = 1; /* just us... */ 289 oldflags = uvn->u_flags; 290 uvn->u_flags = UVM_VNODE_VALID|UVM_VNODE_CANPERSIST; 291 uvn->u_nio = 0; 292 uvn->u_size = used_vnode_size; 293 294 /* if write access, we need to add it to the wlist */ 295 if (accessprot & VM_PROT_WRITE) { 296 LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist); 297 uvn->u_flags |= UVM_VNODE_WRITEABLE; /* we are on wlist! */ 298 } 299 300 /* 301 * add a reference to the vnode. this reference will stay as long 302 * as there is a valid mapping of the vnode. dropped when the 303 * reference count goes to zero [and we either free or persist]. 304 */ 305 VREF(vp); 306 simple_unlock(&uvn->u_obj.vmobjlock); 307 if (oldflags & UVM_VNODE_WANTED) 308 wakeup(uvn); 309 310 UVMHIST_LOG(maphist,"<- done/VREF, ret %p", &uvn->u_obj,0,0,0); 311 return(&uvn->u_obj); 312 } 313 314 315 /* 316 * uvn_reference 317 * 318 * duplicate a reference to a VM object. Note that the reference 319 * count must already be at least one (the passed in reference) so 320 * there is no chance of the uvn being killed or locked out here. 321 * 322 * => caller must call with object unlocked. 323 * => caller must be using the same accessprot as was used at attach time 324 */ 325 326 327 void 328 uvn_reference(uobj) 329 struct uvm_object *uobj; 330 { 331 #ifdef DEBUG 332 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 333 #endif 334 UVMHIST_FUNC("uvn_reference"); UVMHIST_CALLED(maphist); 335 336 simple_lock(&uobj->vmobjlock); 337 #ifdef DEBUG 338 if ((uvn->u_flags & UVM_VNODE_VALID) == 0) { 339 printf("uvn_reference: ref=%d, flags=0x%x\n", uvn->u_flags, 340 uobj->uo_refs); 341 panic("uvn_reference: invalid state"); 342 } 343 #endif 344 uobj->uo_refs++; 345 UVMHIST_LOG(maphist, "<- done (uobj=%p, ref = %ld)", 346 uobj, uobj->uo_refs,0,0); 347 simple_unlock(&uobj->vmobjlock); 348 } 349 350 /* 351 * uvn_detach 352 * 353 * remove a reference to a VM object. 354 * 355 * => caller must call with object unlocked and map locked. 356 * => this starts the detach process, but doesn't have to finish it 357 * (async i/o could still be pending). 358 */ 359 void 360 uvn_detach(uobj) 361 struct uvm_object *uobj; 362 { 363 struct uvm_vnode *uvn; 364 struct vnode *vp; 365 int oldflags; 366 UVMHIST_FUNC("uvn_detach"); UVMHIST_CALLED(maphist); 367 368 simple_lock(&uobj->vmobjlock); 369 370 UVMHIST_LOG(maphist," (uobj=%p) ref=%ld", uobj,uobj->uo_refs,0,0); 371 uobj->uo_refs--; /* drop ref! */ 372 if (uobj->uo_refs) { /* still more refs */ 373 simple_unlock(&uobj->vmobjlock); 374 UVMHIST_LOG(maphist, "<- done (rc>0)", 0,0,0,0); 375 return; 376 } 377 378 /* 379 * get other pointers ... 380 */ 381 382 uvn = (struct uvm_vnode *) uobj; 383 vp = (struct vnode *) uobj; 384 385 /* 386 * clear VTEXT flag now that there are no mappings left (VTEXT is used 387 * to keep an active text file from being overwritten). 388 */ 389 vp->v_flag &= ~VTEXT; 390 391 /* 392 * we just dropped the last reference to the uvn. see if we can 393 * let it "stick around". 394 */ 395 396 if (uvn->u_flags & UVM_VNODE_CANPERSIST) { 397 /* won't block */ 398 uvn_flush(uobj, 0, 0, PGO_DEACTIVATE|PGO_ALLPAGES); 399 simple_unlock(&uobj->vmobjlock); 400 vrele(vp); /* drop vnode reference */ 401 UVMHIST_LOG(maphist,"<- done/vrele! (persist)", 0,0,0,0); 402 return; 403 } 404 405 /* 406 * its a goner! 407 */ 408 409 UVMHIST_LOG(maphist," its a goner (flushing)!", 0,0,0,0); 410 411 uvn->u_flags |= UVM_VNODE_DYING; 412 413 /* 414 * even though we may unlock in flush, no one can gain a reference 415 * to us until we clear the "dying" flag [because it blocks 416 * attaches]. we will not do that until after we've disposed of all 417 * the pages with uvn_flush(). note that before the flush the only 418 * pages that could be marked PG_BUSY are ones that are in async 419 * pageout by the daemon. (there can't be any pending "get"'s 420 * because there are no references to the object). 421 */ 422 423 (void) uvn_flush(uobj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES); 424 425 UVMHIST_LOG(maphist," its a goner (done flush)!", 0,0,0,0); 426 427 /* 428 * given the structure of this pager, the above flush request will 429 * create the following state: all the pages that were in the object 430 * have either been free'd or they are marked PG_BUSY|PG_RELEASED. 431 * the PG_BUSY bit was set either by us or the daemon for async I/O. 432 * in either case, if we have pages left we can't kill the object 433 * yet because i/o is pending. in this case we set the "relkill" 434 * flag which will cause pgo_releasepg to kill the object once all 435 * the I/O's are done [pgo_releasepg will be called from the aiodone 436 * routine or from the page daemon]. 437 */ 438 439 if (uobj->uo_npages) { /* I/O pending. iodone will free */ 440 #ifdef DEBUG 441 /* 442 * XXXCDC: very unlikely to happen until we have async i/o 443 * so print a little info message in case it does. 444 */ 445 printf("uvn_detach: vn %p has pages left after flush - " 446 "relkill mode\n", uobj); 447 #endif 448 uvn->u_flags |= UVM_VNODE_RELKILL; 449 simple_unlock(&uobj->vmobjlock); 450 UVMHIST_LOG(maphist,"<- done! (releasepg will kill obj)", 0, 0, 451 0, 0); 452 return; 453 } 454 455 /* 456 * kill object now. note that we can't be on the sync q because 457 * all references are gone. 458 */ 459 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 460 LIST_REMOVE(uvn, u_wlist); 461 } 462 #ifdef DIAGNOSTIC 463 if (!TAILQ_EMPTY(&uobj->memq)) 464 panic("uvn_deref: vnode VM object still has pages afer " 465 "syncio/free flush"); 466 #endif 467 oldflags = uvn->u_flags; 468 uvn->u_flags = 0; 469 simple_unlock(&uobj->vmobjlock); 470 471 /* wake up any sleepers */ 472 if (oldflags & UVM_VNODE_WANTED) 473 wakeup(uvn); 474 475 /* 476 * drop our reference to the vnode. 477 */ 478 vrele(vp); 479 UVMHIST_LOG(maphist,"<- done (vrele) final", 0,0,0,0); 480 481 return; 482 } 483 484 /* 485 * uvm_vnp_terminate: external hook to clear out a vnode's VM 486 * 487 * called in two cases: 488 * [1] when a persisting vnode vm object (i.e. one with a zero reference 489 * count) needs to be freed so that a vnode can be reused. this 490 * happens under "getnewvnode" in vfs_subr.c. if the vnode from 491 * the free list is still attached (i.e. not VBAD) then vgone is 492 * called. as part of the vgone trace this should get called to 493 * free the vm object. this is the common case. 494 * [2] when a filesystem is being unmounted by force (MNT_FORCE, 495 * "umount -f") the vgone() function is called on active vnodes 496 * on the mounted file systems to kill their data (the vnodes become 497 * "dead" ones [see src/sys/miscfs/deadfs/...]). that results in a 498 * call here (even if the uvn is still in use -- i.e. has a non-zero 499 * reference count). this case happens at "umount -f" and during a 500 * "reboot/halt" operation. 501 * 502 * => the caller must XLOCK and VOP_LOCK the vnode before calling us 503 * [protects us from getting a vnode that is already in the DYING 504 * state...] 505 * => unlike uvn_detach, this function must not return until all the 506 * uvn's pages are disposed of. 507 * => in case [2] the uvn is still alive after this call, but all I/O 508 * ops will fail (due to the backing vnode now being "dead"). this 509 * will prob. kill any process using the uvn due to pgo_get failing. 510 */ 511 512 void 513 uvm_vnp_terminate(vp) 514 struct vnode *vp; 515 { 516 struct uvm_vnode *uvn = &vp->v_uvm; 517 int oldflags; 518 UVMHIST_FUNC("uvm_vnp_terminate"); UVMHIST_CALLED(maphist); 519 520 /* 521 * lock object and check if it is valid 522 */ 523 simple_lock(&uvn->u_obj.vmobjlock); 524 UVMHIST_LOG(maphist, " vp=%p, ref=%ld, flag=0x%lx", vp, 525 uvn->u_obj.uo_refs, uvn->u_flags, 0); 526 if ((uvn->u_flags & UVM_VNODE_VALID) == 0) { 527 simple_unlock(&uvn->u_obj.vmobjlock); 528 UVMHIST_LOG(maphist, "<- done (not active)", 0, 0, 0, 0); 529 return; 530 } 531 532 /* 533 * must be a valid uvn that is not already dying (because XLOCK 534 * protects us from that). the uvn can't in the ALOCK state 535 * because it is valid, and uvn's that are in the ALOCK state haven't 536 * been marked valid yet. 537 */ 538 539 #ifdef DEBUG 540 /* 541 * debug check: are we yanking the vnode out from under our uvn? 542 */ 543 if (uvn->u_obj.uo_refs) { 544 printf("uvm_vnp_terminate(%p): terminating active vnode " 545 "(refs=%d)\n", uvn, uvn->u_obj.uo_refs); 546 } 547 #endif 548 549 /* 550 * it is possible that the uvn was detached and is in the relkill 551 * state [i.e. waiting for async i/o to finish so that releasepg can 552 * kill object]. we take over the vnode now and cancel the relkill. 553 * we want to know when the i/o is done so we can recycle right 554 * away. note that a uvn can only be in the RELKILL state if it 555 * has a zero reference count. 556 */ 557 558 if (uvn->u_flags & UVM_VNODE_RELKILL) 559 uvn->u_flags &= ~UVM_VNODE_RELKILL; /* cancel RELKILL */ 560 561 /* 562 * block the uvn by setting the dying flag, and then flush the 563 * pages. (note that flush may unlock object while doing I/O, but 564 * it will re-lock it before it returns control here). 565 * 566 * also, note that we tell I/O that we are already VOP_LOCK'd so 567 * that uvn_io doesn't attempt to VOP_LOCK again. 568 * 569 * XXXCDC: setting VNISLOCKED on an active uvn which is being terminated 570 * due to a forceful unmount might not be a good idea. maybe we 571 * need a way to pass in this info to uvn_flush through a 572 * pager-defined PGO_ constant [currently there are none]. 573 */ 574 uvn->u_flags |= UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED; 575 576 (void) uvn_flush(&uvn->u_obj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES); 577 578 /* 579 * as we just did a flush we expect all the pages to be gone or in 580 * the process of going. sleep to wait for the rest to go [via iosync]. 581 */ 582 583 while (uvn->u_obj.uo_npages) { 584 #ifdef DEBUG 585 struct vm_page *pp; 586 TAILQ_FOREACH(pp, &uvn->u_obj.memq, listq) { 587 if ((pp->pg_flags & PG_BUSY) == 0) 588 panic("uvm_vnp_terminate: detected unbusy pg"); 589 } 590 if (uvn->u_nio == 0) 591 panic("uvm_vnp_terminate: no I/O to wait for?"); 592 printf("uvm_vnp_terminate: waiting for I/O to fin.\n"); 593 /* 594 * XXXCDC: this is unlikely to happen without async i/o so we 595 * put a printf in just to keep an eye on it. 596 */ 597 #endif 598 uvn->u_flags |= UVM_VNODE_IOSYNC; 599 UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock, FALSE, 600 "uvn_term",0); 601 simple_lock(&uvn->u_obj.vmobjlock); 602 } 603 604 /* 605 * done. now we free the uvn if its reference count is zero 606 * (true if we are zapping a persisting uvn). however, if we are 607 * terminating a uvn with active mappings we let it live ... future 608 * calls down to the vnode layer will fail. 609 */ 610 611 oldflags = uvn->u_flags; 612 if (uvn->u_obj.uo_refs) { 613 614 /* 615 * uvn must live on it is dead-vnode state until all references 616 * are gone. restore flags. clear CANPERSIST state. 617 */ 618 619 uvn->u_flags &= ~(UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED| 620 UVM_VNODE_WANTED|UVM_VNODE_CANPERSIST); 621 622 } else { 623 624 /* 625 * free the uvn now. note that the VREF reference is already 626 * gone [it is dropped when we enter the persist state]. 627 */ 628 if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED) 629 panic("uvm_vnp_terminate: io sync wanted bit set"); 630 631 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 632 LIST_REMOVE(uvn, u_wlist); 633 } 634 uvn->u_flags = 0; /* uvn is history, clear all bits */ 635 } 636 637 if (oldflags & UVM_VNODE_WANTED) 638 wakeup(uvn); /* object lock still held */ 639 640 simple_unlock(&uvn->u_obj.vmobjlock); 641 UVMHIST_LOG(maphist, "<- done", 0, 0, 0, 0); 642 643 } 644 645 /* 646 * uvn_releasepg: handled a released page in a uvn 647 * 648 * => "pg" is a PG_BUSY [caller owns it], PG_RELEASED page that we need 649 * to dispose of. 650 * => caller must handled PG_WANTED case 651 * => called with page's object locked, pageq's unlocked 652 * => returns TRUE if page's object is still alive, FALSE if we 653 * killed the page's object. if we return TRUE, then we 654 * return with the object locked. 655 * => if (nextpgp != NULL) => we return pageq.tqe_next here, and return 656 * with the page queues locked [for pagedaemon] 657 * => if (nextpgp == NULL) => we return with page queues unlocked [normal case] 658 * => we kill the uvn if it is not referenced and we are suppose to 659 * kill it ("relkill"). 660 */ 661 662 boolean_t 663 uvn_releasepg(pg, nextpgp) 664 struct vm_page *pg; 665 struct vm_page **nextpgp; /* OUT */ 666 { 667 struct uvm_vnode *uvn = (struct uvm_vnode *) pg->uobject; 668 #ifdef DIAGNOSTIC 669 if ((pg->pg_flags & PG_RELEASED) == 0) 670 panic("uvn_releasepg: page not released!"); 671 #endif 672 673 /* 674 * dispose of the page [caller handles PG_WANTED] 675 */ 676 pmap_page_protect(pg, VM_PROT_NONE); 677 uvm_lock_pageq(); 678 if (nextpgp) 679 *nextpgp = TAILQ_NEXT(pg, pageq); /* next page for daemon */ 680 uvm_pagefree(pg); 681 if (!nextpgp) 682 uvm_unlock_pageq(); 683 684 /* 685 * now see if we need to kill the object 686 */ 687 if (uvn->u_flags & UVM_VNODE_RELKILL) { 688 if (uvn->u_obj.uo_refs) 689 panic("uvn_releasepg: kill flag set on referenced " 690 "object!"); 691 if (uvn->u_obj.uo_npages == 0) { 692 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 693 LIST_REMOVE(uvn, u_wlist); 694 } 695 #ifdef DIAGNOSTIC 696 if (!TAILQ_EMPTY(&uvn->u_obj.memq)) 697 panic("uvn_releasepg: pages in object with npages == 0"); 698 #endif 699 if (uvn->u_flags & UVM_VNODE_WANTED) 700 /* still holding object lock */ 701 wakeup(uvn); 702 703 uvn->u_flags = 0; /* DEAD! */ 704 simple_unlock(&uvn->u_obj.vmobjlock); 705 return (FALSE); 706 } 707 } 708 return (TRUE); 709 } 710 711 /* 712 * NOTE: currently we have to use VOP_READ/VOP_WRITE because they go 713 * through the buffer cache and allow I/O in any size. These VOPs use 714 * synchronous i/o. [vs. VOP_STRATEGY which can be async, but doesn't 715 * go through the buffer cache or allow I/O sizes larger than a 716 * block]. we will eventually want to change this. 717 * 718 * issues to consider: 719 * uvm provides the uvm_aiodesc structure for async i/o management. 720 * there are two tailq's in the uvm. structure... one for pending async 721 * i/o and one for "done" async i/o. to do an async i/o one puts 722 * an aiodesc on the "pending" list (protected by splbio()), starts the 723 * i/o and returns VM_PAGER_PEND. when the i/o is done, we expect 724 * some sort of "i/o done" function to be called (at splbio(), interrupt 725 * time). this function should remove the aiodesc from the pending list 726 * and place it on the "done" list and wakeup the daemon. the daemon 727 * will run at normal spl() and will remove all items from the "done" 728 * list and call the "aiodone" hook for each done request (see uvm_pager.c). 729 * [in the old vm code, this was done by calling the "put" routine with 730 * null arguments which made the code harder to read and understand because 731 * you had one function ("put") doing two things.] 732 * 733 * so the current pager needs: 734 * int uvn_aiodone(struct uvm_aiodesc *) 735 * 736 * => return 0 (aio finished, free it). otherwise requeue for later collection. 737 * => called with pageq's locked by the daemon. 738 * 739 * general outline: 740 * - "try" to lock object. if fail, just return (will try again later) 741 * - drop "u_nio" (this req is done!) 742 * - if (object->iosync && u_naio == 0) { wakeup &uvn->u_naio } 743 * - get "page" structures (atop?). 744 * - handle "wanted" pages 745 * - handle "released" pages [using pgo_releasepg] 746 * >>> pgo_releasepg may kill the object 747 * dont forget to look at "object" wanted flag in all cases. 748 */ 749 750 751 /* 752 * uvn_flush: flush pages out of a uvm object. 753 * 754 * => object should be locked by caller. we may _unlock_ the object 755 * if (and only if) we need to clean a page (PGO_CLEANIT). 756 * we return with the object locked. 757 * => if PGO_CLEANIT is set, we may block (due to I/O). thus, a caller 758 * might want to unlock higher level resources (e.g. vm_map) 759 * before calling flush. 760 * => if PGO_CLEANIT is not set, then we will neither unlock the object 761 * or block. 762 * => if PGO_ALLPAGE is set, then all pages in the object are valid targets 763 * for flushing. 764 * => NOTE: we rely on the fact that the object's memq is a TAILQ and 765 * that new pages are inserted on the tail end of the list. thus, 766 * we can make a complete pass through the object in one go by starting 767 * at the head and working towards the tail (new pages are put in 768 * front of us). 769 * => NOTE: we are allowed to lock the page queues, so the caller 770 * must not be holding the lock on them [e.g. pagedaemon had 771 * better not call us with the queues locked] 772 * => we return TRUE unless we encountered some sort of I/O error 773 * 774 * comment on "cleaning" object and PG_BUSY pages: 775 * this routine is holding the lock on the object. the only time 776 * that it can run into a PG_BUSY page that it does not own is if 777 * some other process has started I/O on the page (e.g. either 778 * a pagein, or a pageout). if the PG_BUSY page is being paged 779 * in, then it can not be dirty (!PG_CLEAN) because no one has 780 * had a chance to modify it yet. if the PG_BUSY page is being 781 * paged out then it means that someone else has already started 782 * cleaning the page for us (how nice!). in this case, if we 783 * have syncio specified, then after we make our pass through the 784 * object we need to wait for the other PG_BUSY pages to clear 785 * off (i.e. we need to do an iosync). also note that once a 786 * page is PG_BUSY it must stay in its object until it is un-busyed. 787 * 788 * note on page traversal: 789 * we can traverse the pages in an object either by going down the 790 * linked list in "uobj->memq", or we can go over the address range 791 * by page doing hash table lookups for each address. depending 792 * on how many pages are in the object it may be cheaper to do one 793 * or the other. we set "by_list" to true if we are using memq. 794 * if the cost of a hash lookup was equal to the cost of the list 795 * traversal we could compare the number of pages in the start->stop 796 * range to the total number of pages in the object. however, it 797 * seems that a hash table lookup is more expensive than the linked 798 * list traversal, so we multiply the number of pages in the 799 * start->stop range by a penalty which we define below. 800 */ 801 802 #define UVN_HASH_PENALTY 4 /* XXX: a guess */ 803 804 boolean_t 805 uvn_flush(uobj, start, stop, flags) 806 struct uvm_object *uobj; 807 voff_t start, stop; 808 int flags; 809 { 810 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 811 struct vm_page *pp, *ppnext, *ptmp; 812 struct vm_page *pps[MAXBSIZE >> PAGE_SHIFT], **ppsp; 813 int npages, result, lcv; 814 boolean_t retval, need_iosync, by_list, needs_clean, all; 815 voff_t curoff; 816 u_short pp_version; 817 UVMHIST_FUNC("uvn_flush"); UVMHIST_CALLED(maphist); 818 819 curoff = 0; /* XXX: shut up gcc */ 820 /* 821 * get init vals and determine how we are going to traverse object 822 */ 823 824 need_iosync = FALSE; 825 retval = TRUE; /* return value */ 826 if (flags & PGO_ALLPAGES) { 827 all = TRUE; 828 by_list = TRUE; /* always go by the list */ 829 } else { 830 start = trunc_page(start); 831 stop = round_page(stop); 832 #ifdef DEBUG 833 if (stop > round_page(uvn->u_size)) 834 printf("uvn_flush: strange, got an out of range " 835 "flush (fixed)\n"); 836 #endif 837 all = FALSE; 838 by_list = (uobj->uo_npages <= 839 ((stop - start) >> PAGE_SHIFT) * UVN_HASH_PENALTY); 840 } 841 842 UVMHIST_LOG(maphist, 843 " flush start=0x%lx, stop=0x%lx, by_list=%ld, flags=0x%lx", 844 (u_long)start, (u_long)stop, by_list, flags); 845 846 /* 847 * PG_CLEANCHK: this bit is used by the pgo_mk_pcluster function as 848 * a _hint_ as to how up to date the PG_CLEAN bit is. if the hint 849 * is wrong it will only prevent us from clustering... it won't break 850 * anything. we clear all PG_CLEANCHK bits here, and pgo_mk_pcluster 851 * will set them as it syncs PG_CLEAN. This is only an issue if we 852 * are looking at non-inactive pages (because inactive page's PG_CLEAN 853 * bit is always up to date since there are no mappings). 854 * [borrowed PG_CLEANCHK idea from FreeBSD VM] 855 */ 856 857 if ((flags & PGO_CLEANIT) != 0 && 858 uobj->pgops->pgo_mk_pcluster != NULL) { 859 if (by_list) { 860 TAILQ_FOREACH(pp, &uobj->memq, listq) { 861 if (!all && 862 (pp->offset < start || pp->offset >= stop)) 863 continue; 864 atomic_clearbits_int(&pp->pg_flags, 865 PG_CLEANCHK); 866 } 867 868 } else { /* by hash */ 869 for (curoff = start ; curoff < stop; 870 curoff += PAGE_SIZE) { 871 pp = uvm_pagelookup(uobj, curoff); 872 if (pp) 873 atomic_clearbits_int(&pp->pg_flags, 874 PG_CLEANCHK); 875 } 876 } 877 } 878 879 /* 880 * now do it. note: we must update ppnext in body of loop or we 881 * will get stuck. we need to use ppnext because we may free "pp" 882 * before doing the next loop. 883 */ 884 885 if (by_list) { 886 pp = TAILQ_FIRST(&uobj->memq); 887 } else { 888 curoff = start; 889 pp = uvm_pagelookup(uobj, curoff); 890 } 891 892 ppnext = NULL; /* XXX: shut up gcc */ 893 ppsp = NULL; /* XXX: shut up gcc */ 894 uvm_lock_pageq(); /* page queues locked */ 895 896 /* locked: both page queues and uobj */ 897 for ( ; (by_list && pp != NULL) || 898 (!by_list && curoff < stop) ; pp = ppnext) { 899 900 if (by_list) { 901 902 /* 903 * range check 904 */ 905 906 if (!all && 907 (pp->offset < start || pp->offset >= stop)) { 908 ppnext = TAILQ_NEXT(pp, listq); 909 continue; 910 } 911 912 } else { 913 914 /* 915 * null check 916 */ 917 918 curoff += PAGE_SIZE; 919 if (pp == NULL) { 920 if (curoff < stop) 921 ppnext = uvm_pagelookup(uobj, curoff); 922 continue; 923 } 924 925 } 926 927 /* 928 * handle case where we do not need to clean page (either 929 * because we are not clean or because page is not dirty or 930 * is busy): 931 * 932 * NOTE: we are allowed to deactivate a non-wired active 933 * PG_BUSY page, but once a PG_BUSY page is on the inactive 934 * queue it must stay put until it is !PG_BUSY (so as not to 935 * confuse pagedaemon). 936 */ 937 938 if ((flags & PGO_CLEANIT) == 0 || (pp->pg_flags & PG_BUSY) != 0) { 939 needs_clean = FALSE; 940 if ((pp->pg_flags & PG_BUSY) != 0 && 941 (flags & (PGO_CLEANIT|PGO_SYNCIO)) == 942 (PGO_CLEANIT|PGO_SYNCIO)) 943 need_iosync = TRUE; 944 } else { 945 /* 946 * freeing: nuke all mappings so we can sync 947 * PG_CLEAN bit with no race 948 */ 949 if ((pp->pg_flags & PG_CLEAN) != 0 && 950 (flags & PGO_FREE) != 0 && 951 (pp->pg_flags & PQ_ACTIVE) != 0) 952 pmap_page_protect(pp, VM_PROT_NONE); 953 if ((pp->pg_flags & PG_CLEAN) != 0 && 954 pmap_is_modified(pp)) 955 atomic_clearbits_int(&pp->pg_flags, PG_CLEAN); 956 atomic_setbits_int(&pp->pg_flags, PG_CLEANCHK); 957 958 needs_clean = ((pp->pg_flags & PG_CLEAN) == 0); 959 } 960 961 /* 962 * if we don't need a clean... load ppnext and dispose of pp 963 */ 964 if (!needs_clean) { 965 /* load ppnext */ 966 if (by_list) 967 ppnext = TAILQ_NEXT(pp, listq); 968 else { 969 if (curoff < stop) 970 ppnext = uvm_pagelookup(uobj, curoff); 971 } 972 973 /* now dispose of pp */ 974 if (flags & PGO_DEACTIVATE) { 975 if ((pp->pg_flags & PQ_INACTIVE) == 0 && 976 pp->wire_count == 0) { 977 pmap_page_protect(pp, VM_PROT_NONE); 978 uvm_pagedeactivate(pp); 979 } 980 981 } else if (flags & PGO_FREE) { 982 if (pp->pg_flags & PG_BUSY) { 983 /* release busy pages */ 984 atomic_setbits_int(&pp->pg_flags, 985 PG_RELEASED); 986 } else { 987 pmap_page_protect(pp, VM_PROT_NONE); 988 /* removed page from object */ 989 uvm_pagefree(pp); 990 } 991 } 992 /* ppnext is valid so we can continue... */ 993 continue; 994 } 995 996 /* 997 * pp points to a page in the locked object that we are 998 * working on. if it is !PG_CLEAN,!PG_BUSY and we asked 999 * for cleaning (PGO_CLEANIT). we clean it now. 1000 * 1001 * let uvm_pager_put attempted a clustered page out. 1002 * note: locked: uobj and page queues. 1003 */ 1004 1005 atomic_setbits_int(&pp->pg_flags, PG_BUSY); 1006 UVM_PAGE_OWN(pp, "uvn_flush"); 1007 pmap_page_protect(pp, VM_PROT_READ); 1008 pp_version = pp->pg_version; 1009 ReTry: 1010 ppsp = pps; 1011 npages = sizeof(pps) / sizeof(struct vm_page *); 1012 1013 /* locked: page queues, uobj */ 1014 result = uvm_pager_put(uobj, pp, &ppsp, &npages, 1015 flags | PGO_DOACTCLUST, start, stop); 1016 /* unlocked: page queues, uobj */ 1017 1018 /* 1019 * at this point nothing is locked. if we did an async I/O 1020 * it is remotely possible for the async i/o to complete and 1021 * the page "pp" be freed or what not before we get a chance 1022 * to relock the object. in order to detect this, we have 1023 * saved the version number of the page in "pp_version". 1024 */ 1025 1026 /* relock! */ 1027 simple_lock(&uobj->vmobjlock); 1028 uvm_lock_pageq(); 1029 1030 /* 1031 * VM_PAGER_AGAIN: given the structure of this pager, this 1032 * can only happen when we are doing async I/O and can't 1033 * map the pages into kernel memory (pager_map) due to lack 1034 * of vm space. if this happens we drop back to sync I/O. 1035 */ 1036 1037 if (result == VM_PAGER_AGAIN) { 1038 /* 1039 * it is unlikely, but page could have been released 1040 * while we had the object lock dropped. we ignore 1041 * this now and retry the I/O. we will detect and 1042 * handle the released page after the syncio I/O 1043 * completes. 1044 */ 1045 #ifdef DIAGNOSTIC 1046 if (flags & PGO_SYNCIO) 1047 panic("uvn_flush: PGO_SYNCIO return 'try again' error (impossible)"); 1048 #endif 1049 flags |= PGO_SYNCIO; 1050 goto ReTry; 1051 } 1052 1053 /* 1054 * the cleaning operation is now done. finish up. note that 1055 * on error (!OK, !PEND) uvm_pager_put drops the cluster for us. 1056 * if success (OK, PEND) then uvm_pager_put returns the cluster 1057 * to us in ppsp/npages. 1058 */ 1059 1060 /* 1061 * for pending async i/o if we are not deactivating/freeing 1062 * we can move on to the next page. 1063 */ 1064 1065 if (result == VM_PAGER_PEND) { 1066 1067 if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) { 1068 /* 1069 * no per-page ops: refresh ppnext and continue 1070 */ 1071 if (by_list) { 1072 if (pp->pg_version == pp_version) 1073 ppnext = TAILQ_NEXT(pp, listq); 1074 else 1075 /* reset */ 1076 ppnext = TAILQ_FIRST(&uobj->memq); 1077 } else { 1078 if (curoff < stop) 1079 ppnext = uvm_pagelookup(uobj, 1080 curoff); 1081 } 1082 continue; 1083 } 1084 1085 /* need to do anything here? */ 1086 } 1087 1088 /* 1089 * need to look at each page of the I/O operation. we defer 1090 * processing "pp" until the last trip through this "for" loop 1091 * so that we can load "ppnext" for the main loop after we 1092 * play with the cluster pages [thus the "npages + 1" in the 1093 * loop below]. 1094 */ 1095 1096 for (lcv = 0 ; lcv < npages + 1 ; lcv++) { 1097 1098 /* 1099 * handle ppnext for outside loop, and saving pp 1100 * until the end. 1101 */ 1102 if (lcv < npages) { 1103 if (ppsp[lcv] == pp) 1104 continue; /* skip pp until the end */ 1105 ptmp = ppsp[lcv]; 1106 } else { 1107 ptmp = pp; 1108 1109 /* set up next page for outer loop */ 1110 if (by_list) { 1111 if (pp->pg_version == pp_version) 1112 ppnext = TAILQ_NEXT(pp, listq); 1113 else 1114 /* reset */ 1115 ppnext = TAILQ_FIRST(&uobj->memq); 1116 } else { 1117 if (curoff < stop) 1118 ppnext = uvm_pagelookup(uobj, curoff); 1119 } 1120 } 1121 1122 /* 1123 * verify the page didn't get moved while obj was 1124 * unlocked 1125 */ 1126 if (result == VM_PAGER_PEND && ptmp->uobject != uobj) 1127 continue; 1128 1129 /* 1130 * unbusy the page if I/O is done. note that for 1131 * pending I/O it is possible that the I/O op 1132 * finished before we relocked the object (in 1133 * which case the page is no longer busy). 1134 */ 1135 1136 if (result != VM_PAGER_PEND) { 1137 if (ptmp->pg_flags & PG_WANTED) 1138 /* still holding object lock */ 1139 wakeup(ptmp); 1140 1141 atomic_clearbits_int(&ptmp->pg_flags, 1142 PG_WANTED|PG_BUSY); 1143 UVM_PAGE_OWN(ptmp, NULL); 1144 if (ptmp->pg_flags & PG_RELEASED) { 1145 1146 /* pgo_releasepg wants this */ 1147 uvm_unlock_pageq(); 1148 if (!uvn_releasepg(ptmp, NULL)) 1149 return (TRUE); 1150 1151 uvm_lock_pageq(); /* relock */ 1152 continue; /* next page */ 1153 1154 } else { 1155 atomic_setbits_int(&ptmp->pg_flags, 1156 PG_CLEAN|PG_CLEANCHK); 1157 if ((flags & PGO_FREE) == 0) 1158 pmap_clear_modify(ptmp); 1159 } 1160 } 1161 1162 /* 1163 * dispose of page 1164 */ 1165 1166 if (flags & PGO_DEACTIVATE) { 1167 if ((pp->pg_flags & PQ_INACTIVE) == 0 && 1168 pp->wire_count == 0) { 1169 pmap_page_protect(ptmp, VM_PROT_NONE); 1170 uvm_pagedeactivate(ptmp); 1171 } 1172 1173 } else if (flags & PGO_FREE) { 1174 if (result == VM_PAGER_PEND) { 1175 if ((ptmp->pg_flags & PG_BUSY) != 0) 1176 /* signal for i/o done */ 1177 atomic_setbits_int( 1178 &ptmp->pg_flags, 1179 PG_RELEASED); 1180 } else { 1181 if (result != VM_PAGER_OK) { 1182 printf("uvn_flush: obj=%p, " 1183 "offset=0x%llx. error " 1184 "during pageout.\n", 1185 pp->uobject, 1186 (long long)pp->offset); 1187 printf("uvn_flush: WARNING: " 1188 "changes to page may be " 1189 "lost!\n"); 1190 retval = FALSE; 1191 } 1192 pmap_page_protect(ptmp, VM_PROT_NONE); 1193 uvm_pagefree(ptmp); 1194 } 1195 } 1196 1197 } /* end of "lcv" for loop */ 1198 1199 } /* end of "pp" for loop */ 1200 1201 /* 1202 * done with pagequeues: unlock 1203 */ 1204 uvm_unlock_pageq(); 1205 1206 /* 1207 * now wait for all I/O if required. 1208 */ 1209 if (need_iosync) { 1210 1211 UVMHIST_LOG(maphist," <<DOING IOSYNC>>",0,0,0,0); 1212 while (uvn->u_nio != 0) { 1213 uvn->u_flags |= UVM_VNODE_IOSYNC; 1214 UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock, 1215 FALSE, "uvn_flush",0); 1216 simple_lock(&uvn->u_obj.vmobjlock); 1217 } 1218 if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED) 1219 wakeup(&uvn->u_flags); 1220 uvn->u_flags &= ~(UVM_VNODE_IOSYNC|UVM_VNODE_IOSYNCWANTED); 1221 } 1222 1223 /* return, with object locked! */ 1224 UVMHIST_LOG(maphist,"<- done (retval=0x%lx)",retval,0,0,0); 1225 return(retval); 1226 } 1227 1228 /* 1229 * uvn_cluster 1230 * 1231 * we are about to do I/O in an object at offset. this function is called 1232 * to establish a range of offsets around "offset" in which we can cluster 1233 * I/O. 1234 * 1235 * - currently doesn't matter if obj locked or not. 1236 */ 1237 1238 void 1239 uvn_cluster(uobj, offset, loffset, hoffset) 1240 struct uvm_object *uobj; 1241 voff_t offset; 1242 voff_t *loffset, *hoffset; /* OUT */ 1243 { 1244 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 1245 *loffset = offset; 1246 1247 if (*loffset >= uvn->u_size) 1248 panic("uvn_cluster: offset out of range"); 1249 1250 /* 1251 * XXX: old pager claims we could use VOP_BMAP to get maxcontig value. 1252 */ 1253 *hoffset = *loffset + MAXBSIZE; 1254 if (*hoffset > round_page(uvn->u_size)) /* past end? */ 1255 *hoffset = round_page(uvn->u_size); 1256 1257 return; 1258 } 1259 1260 /* 1261 * uvn_put: flush page data to backing store. 1262 * 1263 * => prefer map unlocked (not required) 1264 * => object must be locked! we will _unlock_ it before starting I/O. 1265 * => flags: PGO_SYNCIO -- use sync. I/O 1266 * => note: caller must set PG_CLEAN and pmap_clear_modify (if needed) 1267 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync. 1268 * [thus we never do async i/o! see iodone comment] 1269 */ 1270 1271 int 1272 uvn_put(uobj, pps, npages, flags) 1273 struct uvm_object *uobj; 1274 struct vm_page **pps; 1275 int npages, flags; 1276 { 1277 int retval; 1278 1279 /* note: object locked */ 1280 retval = uvn_io((struct uvm_vnode*)uobj, pps, npages, flags, UIO_WRITE); 1281 /* note: object unlocked */ 1282 1283 return(retval); 1284 } 1285 1286 1287 /* 1288 * uvn_get: get pages (synchronously) from backing store 1289 * 1290 * => prefer map unlocked (not required) 1291 * => object must be locked! we will _unlock_ it before starting any I/O. 1292 * => flags: PGO_ALLPAGES: get all of the pages 1293 * PGO_LOCKED: fault data structures are locked 1294 * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx] 1295 * => NOTE: caller must check for released pages!! 1296 */ 1297 1298 int 1299 uvn_get(uobj, offset, pps, npagesp, centeridx, access_type, advice, flags) 1300 struct uvm_object *uobj; 1301 voff_t offset; 1302 struct vm_page **pps; /* IN/OUT */ 1303 int *npagesp; /* IN (OUT if PGO_LOCKED) */ 1304 int centeridx, advice, flags; 1305 vm_prot_t access_type; 1306 { 1307 voff_t current_offset; 1308 struct vm_page *ptmp; 1309 int lcv, result, gotpages; 1310 boolean_t done; 1311 UVMHIST_FUNC("uvn_get"); UVMHIST_CALLED(maphist); 1312 UVMHIST_LOG(maphist, "flags=%ld", flags,0,0,0); 1313 1314 /* 1315 * step 1: handled the case where fault data structures are locked. 1316 */ 1317 1318 if (flags & PGO_LOCKED) { 1319 1320 /* 1321 * gotpages is the current number of pages we've gotten (which 1322 * we pass back up to caller via *npagesp. 1323 */ 1324 1325 gotpages = 0; 1326 1327 /* 1328 * step 1a: get pages that are already resident. only do this 1329 * if the data structures are locked (i.e. the first time 1330 * through). 1331 */ 1332 1333 done = TRUE; /* be optimistic */ 1334 1335 for (lcv = 0, current_offset = offset ; lcv < *npagesp ; 1336 lcv++, current_offset += PAGE_SIZE) { 1337 1338 /* do we care about this page? if not, skip it */ 1339 if (pps[lcv] == PGO_DONTCARE) 1340 continue; 1341 1342 /* lookup page */ 1343 ptmp = uvm_pagelookup(uobj, current_offset); 1344 1345 /* to be useful must get a non-busy, non-released pg */ 1346 if (ptmp == NULL || 1347 (ptmp->pg_flags & (PG_BUSY|PG_RELEASED)) != 0) { 1348 if (lcv == centeridx || (flags & PGO_ALLPAGES) 1349 != 0) 1350 done = FALSE; /* need to do a wait or I/O! */ 1351 continue; 1352 } 1353 1354 /* 1355 * useful page: busy/lock it and plug it in our 1356 * result array 1357 */ 1358 atomic_setbits_int(&ptmp->pg_flags, PG_BUSY); 1359 UVM_PAGE_OWN(ptmp, "uvn_get1"); 1360 pps[lcv] = ptmp; 1361 gotpages++; 1362 1363 } /* "for" lcv loop */ 1364 1365 /* 1366 * XXX: given the "advice", should we consider async read-ahead? 1367 * XXX: fault current does deactive of pages behind us. is 1368 * this good (other callers might now). 1369 */ 1370 /* 1371 * XXX: read-ahead currently handled by buffer cache (bread) 1372 * level. 1373 * XXX: no async i/o available. 1374 * XXX: so we don't do anything now. 1375 */ 1376 1377 /* 1378 * step 1c: now we've either done everything needed or we to 1379 * unlock and do some waiting or I/O. 1380 */ 1381 1382 *npagesp = gotpages; /* let caller know */ 1383 if (done) 1384 return(VM_PAGER_OK); /* bingo! */ 1385 else 1386 /* EEK! Need to unlock and I/O */ 1387 return(VM_PAGER_UNLOCK); 1388 } 1389 1390 /* 1391 * step 2: get non-resident or busy pages. 1392 * object is locked. data structures are unlocked. 1393 * 1394 * XXX: because we can't do async I/O at this level we get things 1395 * page at a time (otherwise we'd chunk). the VOP_READ() will do 1396 * async-read-ahead for us at a lower level. 1397 */ 1398 1399 for (lcv = 0, current_offset = offset; 1400 lcv < *npagesp ; lcv++, current_offset += PAGE_SIZE) { 1401 1402 /* skip over pages we've already gotten or don't want */ 1403 /* skip over pages we don't _have_ to get */ 1404 if (pps[lcv] != NULL || (lcv != centeridx && 1405 (flags & PGO_ALLPAGES) == 0)) 1406 continue; 1407 1408 /* 1409 * we have yet to locate the current page (pps[lcv]). we first 1410 * look for a page that is already at the current offset. if 1411 * we fine a page, we check to see if it is busy or released. 1412 * if that is the case, then we sleep on the page until it is 1413 * no longer busy or released and repeat the lookup. if the 1414 * page we found is neither busy nor released, then we busy it 1415 * (so we own it) and plug it into pps[lcv]. this breaks the 1416 * following while loop and indicates we are ready to move on 1417 * to the next page in the "lcv" loop above. 1418 * 1419 * if we exit the while loop with pps[lcv] still set to NULL, 1420 * then it means that we allocated a new busy/fake/clean page 1421 * ptmp in the object and we need to do I/O to fill in the data. 1422 */ 1423 1424 while (pps[lcv] == NULL) { /* top of "pps" while loop */ 1425 1426 /* look for a current page */ 1427 ptmp = uvm_pagelookup(uobj, current_offset); 1428 1429 /* nope? allocate one now (if we can) */ 1430 if (ptmp == NULL) { 1431 1432 ptmp = uvm_pagealloc(uobj, current_offset, 1433 NULL, 0); 1434 1435 /* out of RAM? */ 1436 if (ptmp == NULL) { 1437 simple_unlock(&uobj->vmobjlock); 1438 uvm_wait("uvn_getpage"); 1439 simple_lock(&uobj->vmobjlock); 1440 1441 /* goto top of pps while loop */ 1442 continue; 1443 } 1444 1445 /* 1446 * got new page ready for I/O. break pps 1447 * while loop. pps[lcv] is still NULL. 1448 */ 1449 break; 1450 } 1451 1452 /* page is there, see if we need to wait on it */ 1453 if ((ptmp->pg_flags & (PG_BUSY|PG_RELEASED)) != 0) { 1454 atomic_setbits_int(&ptmp->pg_flags, PG_WANTED); 1455 UVM_UNLOCK_AND_WAIT(ptmp, 1456 &uobj->vmobjlock, FALSE, "uvn_get",0); 1457 simple_lock(&uobj->vmobjlock); 1458 continue; /* goto top of pps while loop */ 1459 } 1460 1461 /* 1462 * if we get here then the page has become resident 1463 * and unbusy between steps 1 and 2. we busy it 1464 * now (so we own it) and set pps[lcv] (so that we 1465 * exit the while loop). 1466 */ 1467 atomic_setbits_int(&ptmp->pg_flags, PG_BUSY); 1468 UVM_PAGE_OWN(ptmp, "uvn_get2"); 1469 pps[lcv] = ptmp; 1470 } 1471 1472 /* 1473 * if we own the a valid page at the correct offset, pps[lcv] 1474 * will point to it. nothing more to do except go to the 1475 * next page. 1476 */ 1477 1478 if (pps[lcv]) 1479 continue; /* next lcv */ 1480 1481 /* 1482 * we have a "fake/busy/clean" page that we just allocated. do 1483 * I/O to fill it with valid data. note that object must be 1484 * locked going into uvn_io, but will be unlocked afterwards. 1485 */ 1486 1487 result = uvn_io((struct uvm_vnode *) uobj, &ptmp, 1, 1488 PGO_SYNCIO, UIO_READ); 1489 1490 /* 1491 * I/O done. object is unlocked (by uvn_io). because we used 1492 * syncio the result can not be PEND or AGAIN. we must relock 1493 * and check for errors. 1494 */ 1495 1496 /* lock object. check for errors. */ 1497 simple_lock(&uobj->vmobjlock); 1498 if (result != VM_PAGER_OK) { 1499 if (ptmp->pg_flags & PG_WANTED) 1500 /* object lock still held */ 1501 wakeup(ptmp); 1502 1503 atomic_clearbits_int(&ptmp->pg_flags, 1504 PG_WANTED|PG_BUSY); 1505 UVM_PAGE_OWN(ptmp, NULL); 1506 uvm_lock_pageq(); 1507 uvm_pagefree(ptmp); 1508 uvm_unlock_pageq(); 1509 simple_unlock(&uobj->vmobjlock); 1510 return(result); 1511 } 1512 1513 /* 1514 * we got the page! clear the fake flag (indicates valid 1515 * data now in page) and plug into our result array. note 1516 * that page is still busy. 1517 * 1518 * it is the callers job to: 1519 * => check if the page is released 1520 * => unbusy the page 1521 * => activate the page 1522 */ 1523 1524 /* data is valid ... */ 1525 atomic_clearbits_int(&ptmp->pg_flags, PG_FAKE); 1526 pmap_clear_modify(ptmp); /* ... and clean */ 1527 pps[lcv] = ptmp; 1528 1529 } /* lcv loop */ 1530 1531 /* 1532 * finally, unlock object and return. 1533 */ 1534 1535 simple_unlock(&uobj->vmobjlock); 1536 return (VM_PAGER_OK); 1537 } 1538 1539 /* 1540 * uvn_io: do I/O to a vnode 1541 * 1542 * => prefer map unlocked (not required) 1543 * => object must be locked! we will _unlock_ it before starting I/O. 1544 * => flags: PGO_SYNCIO -- use sync. I/O 1545 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync. 1546 * [thus we never do async i/o! see iodone comment] 1547 */ 1548 1549 int 1550 uvn_io(uvn, pps, npages, flags, rw) 1551 struct uvm_vnode *uvn; 1552 vm_page_t *pps; 1553 int npages, flags, rw; 1554 { 1555 struct vnode *vn; 1556 struct uio uio; 1557 struct iovec iov; 1558 vaddr_t kva; 1559 off_t file_offset; 1560 int waitf, result, mapinflags; 1561 size_t got, wanted; 1562 UVMHIST_FUNC("uvn_io"); UVMHIST_CALLED(maphist); 1563 1564 UVMHIST_LOG(maphist, "rw=%ld", rw,0,0,0); 1565 1566 /* 1567 * init values 1568 */ 1569 1570 waitf = (flags & PGO_SYNCIO) ? M_WAITOK : M_NOWAIT; 1571 vn = (struct vnode *) uvn; 1572 file_offset = pps[0]->offset; 1573 1574 /* 1575 * check for sync'ing I/O. 1576 */ 1577 1578 while (uvn->u_flags & UVM_VNODE_IOSYNC) { 1579 if (waitf == M_NOWAIT) { 1580 simple_unlock(&uvn->u_obj.vmobjlock); 1581 UVMHIST_LOG(maphist,"<- try again (iosync)",0,0,0,0); 1582 return(VM_PAGER_AGAIN); 1583 } 1584 uvn->u_flags |= UVM_VNODE_IOSYNCWANTED; 1585 UVM_UNLOCK_AND_WAIT(&uvn->u_flags, &uvn->u_obj.vmobjlock, 1586 FALSE, "uvn_iosync",0); 1587 simple_lock(&uvn->u_obj.vmobjlock); 1588 } 1589 1590 /* 1591 * check size 1592 */ 1593 1594 if (file_offset >= uvn->u_size) { 1595 simple_unlock(&uvn->u_obj.vmobjlock); 1596 UVMHIST_LOG(maphist,"<- BAD (size check)",0,0,0,0); 1597 return(VM_PAGER_BAD); 1598 } 1599 1600 /* 1601 * first try and map the pages in (without waiting) 1602 */ 1603 1604 mapinflags = (rw == UIO_READ) ? 1605 UVMPAGER_MAPIN_READ : UVMPAGER_MAPIN_WRITE; 1606 1607 kva = uvm_pagermapin(pps, npages, mapinflags); 1608 if (kva == 0 && waitf == M_NOWAIT) { 1609 simple_unlock(&uvn->u_obj.vmobjlock); 1610 UVMHIST_LOG(maphist,"<- mapin failed (try again)",0,0,0,0); 1611 return(VM_PAGER_AGAIN); 1612 } 1613 1614 /* 1615 * ok, now bump u_nio up. at this point we are done with uvn 1616 * and can unlock it. if we still don't have a kva, try again 1617 * (this time with sleep ok). 1618 */ 1619 1620 uvn->u_nio++; /* we have an I/O in progress! */ 1621 simple_unlock(&uvn->u_obj.vmobjlock); 1622 /* NOTE: object now unlocked */ 1623 if (kva == 0) 1624 kva = uvm_pagermapin(pps, npages, 1625 mapinflags | UVMPAGER_MAPIN_WAITOK); 1626 1627 /* 1628 * ok, mapped in. our pages are PG_BUSY so they are not going to 1629 * get touched (so we can look at "offset" without having to lock 1630 * the object). set up for I/O. 1631 */ 1632 1633 /* 1634 * fill out uio/iov 1635 */ 1636 1637 iov.iov_base = (caddr_t) kva; 1638 wanted = npages << PAGE_SHIFT; 1639 if (file_offset + wanted > uvn->u_size) 1640 wanted = uvn->u_size - file_offset; /* XXX: needed? */ 1641 iov.iov_len = wanted; 1642 uio.uio_iov = &iov; 1643 uio.uio_iovcnt = 1; 1644 uio.uio_offset = file_offset; 1645 uio.uio_segflg = UIO_SYSSPACE; 1646 uio.uio_rw = rw; 1647 uio.uio_resid = wanted; 1648 uio.uio_procp = curproc; 1649 1650 /* 1651 * do the I/O! (XXX: curproc?) 1652 */ 1653 1654 UVMHIST_LOG(maphist, "calling VOP",0,0,0,0); 1655 1656 /* 1657 * This process may already have this vnode locked, if we faulted in 1658 * copyin() or copyout() on a region backed by this vnode 1659 * while doing I/O to the vnode. If this is the case, don't 1660 * panic.. instead, return the error to the user. 1661 * 1662 * XXX this is a stopgap to prevent a panic. 1663 * Ideally, this kind of operation *should* work. 1664 */ 1665 result = 0; 1666 if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0) 1667 result = vn_lock(vn, LK_EXCLUSIVE | LK_RECURSEFAIL, curproc); 1668 1669 if (result == 0) { 1670 /* NOTE: vnode now locked! */ 1671 1672 if (rw == UIO_READ) 1673 result = VOP_READ(vn, &uio, 0, curproc->p_ucred); 1674 else 1675 result = VOP_WRITE(vn, &uio, 0, curproc->p_ucred); 1676 1677 if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0) 1678 VOP_UNLOCK(vn, 0, curproc); 1679 } 1680 1681 /* NOTE: vnode now unlocked (unless vnislocked) */ 1682 1683 UVMHIST_LOG(maphist, "done calling VOP",0,0,0,0); 1684 1685 /* 1686 * result == unix style errno (0 == OK!) 1687 * 1688 * zero out rest of buffer (if needed) 1689 */ 1690 1691 if (result == 0) { 1692 got = wanted - uio.uio_resid; 1693 1694 if (wanted && got == 0) { 1695 result = EIO; /* XXX: error? */ 1696 } else if (got < PAGE_SIZE * npages && rw == UIO_READ) { 1697 memset((void *) (kva + got), 0, 1698 (npages << PAGE_SHIFT) - got); 1699 } 1700 } 1701 1702 /* 1703 * now remove pager mapping 1704 */ 1705 uvm_pagermapout(kva, npages); 1706 1707 /* 1708 * now clean up the object (i.e. drop I/O count) 1709 */ 1710 1711 simple_lock(&uvn->u_obj.vmobjlock); 1712 /* NOTE: object now locked! */ 1713 1714 uvn->u_nio--; /* I/O DONE! */ 1715 if ((uvn->u_flags & UVM_VNODE_IOSYNC) != 0 && uvn->u_nio == 0) { 1716 wakeup(&uvn->u_nio); 1717 } 1718 simple_unlock(&uvn->u_obj.vmobjlock); 1719 /* NOTE: object now unlocked! */ 1720 1721 /* 1722 * done! 1723 */ 1724 1725 UVMHIST_LOG(maphist, "<- done (result %ld)", result,0,0,0); 1726 if (result == 0) 1727 return(VM_PAGER_OK); 1728 else 1729 return(VM_PAGER_ERROR); 1730 } 1731 1732 /* 1733 * uvm_vnp_uncache: disable "persisting" in a vnode... when last reference 1734 * is gone we will kill the object (flushing dirty pages back to the vnode 1735 * if needed). 1736 * 1737 * => returns TRUE if there was no uvm_object attached or if there was 1738 * one and we killed it [i.e. if there is no active uvn] 1739 * => called with the vnode VOP_LOCK'd [we will unlock it for I/O, if 1740 * needed] 1741 * 1742 * => XXX: given that we now kill uvn's when a vnode is recycled (without 1743 * having to hold a reference on the vnode) and given a working 1744 * uvm_vnp_sync(), how does that effect the need for this function? 1745 * [XXXCDC: seems like it can die?] 1746 * 1747 * => XXX: this function should DIE once we merge the VM and buffer 1748 * cache. 1749 * 1750 * research shows that this is called in the following places: 1751 * ext2fs_truncate, ffs_truncate, detrunc[msdosfs]: called when vnode 1752 * changes sizes 1753 * ext2fs_write, WRITE [ufs_readwrite], msdosfs_write: called when we 1754 * are written to 1755 * ex2fs_chmod, ufs_chmod: called if VTEXT vnode and the sticky bit 1756 * is off 1757 * ffs_realloccg: when we can't extend the current block and have 1758 * to allocate a new one we call this [XXX: why?] 1759 * nfsrv_rename, rename_files: called when the target filename is there 1760 * and we want to remove it 1761 * nfsrv_remove, sys_unlink: called on file we are removing 1762 * nfsrv_access: if VTEXT and we want WRITE access and we don't uncache 1763 * then return "text busy" 1764 * nfs_open: seems to uncache any file opened with nfs 1765 * vn_writechk: if VTEXT vnode and can't uncache return "text busy" 1766 */ 1767 1768 boolean_t 1769 uvm_vnp_uncache(struct vnode *vp) 1770 { 1771 struct uvm_vnode *uvn = &vp->v_uvm; 1772 1773 /* 1774 * lock uvn part of the vnode and check to see if we need to do anything 1775 */ 1776 1777 simple_lock(&uvn->u_obj.vmobjlock); 1778 if ((uvn->u_flags & UVM_VNODE_VALID) == 0 || 1779 (uvn->u_flags & UVM_VNODE_BLOCKED) != 0) { 1780 simple_unlock(&uvn->u_obj.vmobjlock); 1781 return(TRUE); 1782 } 1783 1784 /* 1785 * we have a valid, non-blocked uvn. clear persist flag. 1786 * if uvn is currently active we can return now. 1787 */ 1788 1789 uvn->u_flags &= ~UVM_VNODE_CANPERSIST; 1790 if (uvn->u_obj.uo_refs) { 1791 simple_unlock(&uvn->u_obj.vmobjlock); 1792 return(FALSE); 1793 } 1794 1795 /* 1796 * uvn is currently persisting! we have to gain a reference to 1797 * it so that we can call uvn_detach to kill the uvn. 1798 */ 1799 1800 VREF(vp); /* seems ok, even with VOP_LOCK */ 1801 uvn->u_obj.uo_refs++; /* value is now 1 */ 1802 simple_unlock(&uvn->u_obj.vmobjlock); 1803 1804 #ifdef VFSDEBUG 1805 /* 1806 * carry over sanity check from old vnode pager: the vnode should 1807 * be VOP_LOCK'd, and we confirm it here. 1808 */ 1809 if ((vp->v_flag & VLOCKSWORK) && !VOP_ISLOCKED(vp)) 1810 panic("uvm_vnp_uncache: vnode not locked!"); 1811 #endif /* VFSDEBUG */ 1812 1813 /* 1814 * now drop our reference to the vnode. if we have the sole 1815 * reference to the vnode then this will cause it to die [as we 1816 * just cleared the persist flag]. we have to unlock the vnode 1817 * while we are doing this as it may trigger I/O. 1818 * 1819 * XXX: it might be possible for uvn to get reclaimed while we are 1820 * unlocked causing us to return TRUE when we should not. we ignore 1821 * this as a false-positive return value doesn't hurt us. 1822 */ 1823 VOP_UNLOCK(vp, 0, curproc); 1824 uvn_detach(&uvn->u_obj); 1825 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, curproc); 1826 1827 /* 1828 * and return... 1829 */ 1830 1831 return(TRUE); 1832 } 1833 1834 /* 1835 * uvm_vnp_setsize: grow or shrink a vnode uvn 1836 * 1837 * grow => just update size value 1838 * shrink => toss un-needed pages 1839 * 1840 * => we assume that the caller has a reference of some sort to the 1841 * vnode in question so that it will not be yanked out from under 1842 * us. 1843 * 1844 * called from: 1845 * => truncate fns (ext2fs_truncate, ffs_truncate, detrunc[msdos]) 1846 * => "write" fns (ext2fs_write, WRITE [ufs/ufs], msdosfs_write, nfs_write) 1847 * => ffs_balloc [XXX: why? doesn't WRITE handle?] 1848 * => NFS: nfs_loadattrcache, nfs_getattrcache, nfs_setattr 1849 * => union fs: union_newsize 1850 */ 1851 1852 void 1853 uvm_vnp_setsize(vp, newsize) 1854 struct vnode *vp; 1855 voff_t newsize; 1856 { 1857 struct uvm_vnode *uvn = &vp->v_uvm; 1858 1859 /* 1860 * lock uvn and check for valid object, and if valid: do it! 1861 */ 1862 simple_lock(&uvn->u_obj.vmobjlock); 1863 if (uvn->u_flags & UVM_VNODE_VALID) { 1864 1865 /* 1866 * now check if the size has changed: if we shrink we had better 1867 * toss some pages... 1868 */ 1869 1870 if (uvn->u_size > newsize) { 1871 (void)uvn_flush(&uvn->u_obj, newsize, 1872 uvn->u_size, PGO_FREE); 1873 } 1874 uvn->u_size = newsize; 1875 } 1876 simple_unlock(&uvn->u_obj.vmobjlock); 1877 1878 /* 1879 * done 1880 */ 1881 return; 1882 } 1883 1884 /* 1885 * uvm_vnp_sync: flush all dirty VM pages back to their backing vnodes. 1886 * 1887 * => called from sys_sync with no VM structures locked 1888 * => only one process can do a sync at a time (because the uvn 1889 * structure only has one queue for sync'ing). we ensure this 1890 * by holding the uvn_sync_lock while the sync is in progress. 1891 * other processes attempting a sync will sleep on this lock 1892 * until we are done. 1893 */ 1894 void 1895 uvm_vnp_sync(struct mount *mp) 1896 { 1897 struct uvm_vnode *uvn; 1898 struct vnode *vp; 1899 1900 /* 1901 * step 1: ensure we are only ones using the uvn_sync_q by locking 1902 * our lock... 1903 */ 1904 rw_enter_write(&uvn_sync_lock); 1905 1906 /* 1907 * step 2: build up a simpleq of uvns of interest based on the 1908 * write list. we gain a reference to uvns of interest. 1909 */ 1910 SIMPLEQ_INIT(&uvn_sync_q); 1911 LIST_FOREACH(uvn, &uvn_wlist, u_wlist) { 1912 1913 vp = (struct vnode *)uvn; 1914 if (mp && vp->v_mount != mp) 1915 continue; 1916 1917 /* 1918 * If the vnode is "blocked" it means it must be dying, which 1919 * in turn means its in the process of being flushed out so 1920 * we can safely skip it. 1921 * 1922 * note that uvn must already be valid because we found it on 1923 * the wlist (this also means it can't be ALOCK'd). 1924 */ 1925 if ((uvn->u_flags & UVM_VNODE_BLOCKED) != 0) 1926 continue; 1927 1928 1929 /* 1930 * gain reference. watch out for persisting uvns (need to 1931 * regain vnode REF). 1932 */ 1933 if (uvn->u_obj.uo_refs == 0) 1934 VREF(vp); 1935 uvn->u_obj.uo_refs++; 1936 1937 SIMPLEQ_INSERT_HEAD(&uvn_sync_q, uvn, u_syncq); 1938 } 1939 1940 /* step 3: we now have a list of uvn's that may need cleaning. */ 1941 SIMPLEQ_FOREACH(uvn, &uvn_sync_q, u_syncq) { 1942 #ifdef DEBUG 1943 if (uvn->u_flags & UVM_VNODE_DYING) { 1944 printf("uvm_vnp_sync: dying vnode on sync list\n"); 1945 } 1946 #endif 1947 uvn_flush(&uvn->u_obj, 0, 0, PGO_CLEANIT|PGO_ALLPAGES|PGO_DOACTCLUST); 1948 1949 /* 1950 * if we have the only reference and we just cleaned the uvn, 1951 * then we can pull it out of the UVM_VNODE_WRITEABLE state 1952 * thus allowing us to avoid thinking about flushing it again 1953 * on later sync ops. 1954 */ 1955 if (uvn->u_obj.uo_refs == 1 && 1956 (uvn->u_flags & UVM_VNODE_WRITEABLE)) { 1957 LIST_REMOVE(uvn, u_wlist); 1958 uvn->u_flags &= ~UVM_VNODE_WRITEABLE; 1959 } 1960 1961 /* now drop our reference to the uvn */ 1962 uvn_detach(&uvn->u_obj); 1963 } 1964 1965 rw_exit_write(&uvn_sync_lock); 1966 } 1967