1 /* $NetBSD: uvm_vnode.c,v 1.9 1998/03/11 01:37:40 chuck Exp $ */ 2 3 /* 4 * XXXCDC: "ROUGH DRAFT" QUALITY UVM PRE-RELEASE FILE! 5 * >>>USE AT YOUR OWN RISK, WORK IS NOT FINISHED<<< 6 */ 7 /* 8 * Copyright (c) 1997 Charles D. Cranor and Washington University. 9 * Copyright (c) 1991, 1993 10 * The Regents of the University of California. 11 * Copyright (c) 1990 University of Utah. 12 * 13 * All rights reserved. 14 * 15 * This code is derived from software contributed to Berkeley by 16 * the Systems Programming Group of the University of Utah Computer 17 * Science Department. 18 * 19 * Redistribution and use in source and binary forms, with or without 20 * modification, are permitted provided that the following conditions 21 * are met: 22 * 1. Redistributions of source code must retain the above copyright 23 * notice, this list of conditions and the following disclaimer. 24 * 2. Redistributions in binary form must reproduce the above copyright 25 * notice, this list of conditions and the following disclaimer in the 26 * documentation and/or other materials provided with the distribution. 27 * 3. All advertising materials mentioning features or use of this software 28 * must display the following acknowledgement: 29 * This product includes software developed by Charles D. Cranor, 30 * Washington University, the University of California, Berkeley and 31 * its contributors. 32 * 4. Neither the name of the University nor the names of its contributors 33 * may be used to endorse or promote products derived from this software 34 * without specific prior written permission. 35 * 36 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 37 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 38 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 39 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 40 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 41 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 42 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 43 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 44 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 45 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 46 * SUCH DAMAGE. 47 * 48 * @(#)vnode_pager.c 8.8 (Berkeley) 2/13/94 49 * from: Id: uvm_vnode.c,v 1.1.2.26 1998/02/02 20:38:07 chuck Exp 50 */ 51 52 #include "fs_nfs.h" 53 #include "opt_uvmhist.h" 54 55 /* 56 * uvm_vnode.c: the vnode pager. 57 */ 58 59 #include <sys/param.h> 60 #include <sys/systm.h> 61 #include <sys/mount.h> 62 #include <sys/proc.h> 63 #include <sys/malloc.h> 64 #include <sys/vnode.h> 65 66 #include <vm/vm.h> 67 #include <vm/vm_page.h> 68 #include <vm/vm_kern.h> 69 70 #include <sys/syscallargs.h> 71 72 #include <uvm/uvm.h> 73 #include <uvm/uvm_vnode.h> 74 75 /* 76 * private global data structure 77 * 78 * we keep a list of writeable active vnode-backed VM objects for sync op. 79 * we keep a simpleq of vnodes that are currently being sync'd. 80 */ 81 82 LIST_HEAD(uvn_list_struct, uvm_vnode); 83 static struct uvn_list_struct uvn_wlist; /* writeable uvns */ 84 static simple_lock_data_t uvn_wl_lock; /* locks uvn_wlist */ 85 86 SIMPLEQ_HEAD(uvn_sq_struct, uvm_vnode); 87 static struct uvn_sq_struct uvn_sync_q; /* sync'ing uvns */ 88 lock_data_t uvn_sync_lock; /* locks sync operation */ 89 90 /* 91 * functions 92 */ 93 94 static int uvn_asyncget __P((struct uvm_object *, vm_offset_t, 95 int)); 96 struct uvm_object *uvn_attach __P((void *, vm_prot_t)); 97 static void uvn_cluster __P((struct uvm_object *, vm_offset_t, 98 vm_offset_t *, vm_offset_t *)); 99 static void uvn_detach __P((struct uvm_object *)); 100 static boolean_t uvn_flush __P((struct uvm_object *, vm_offset_t, 101 vm_offset_t, int)); 102 static int uvn_get __P((struct uvm_object *, vm_offset_t, 103 vm_page_t *, int *, int, 104 vm_prot_t, int, int)); 105 static void uvn_init __P((void)); 106 static int uvn_io __P((struct uvm_vnode *, vm_page_t *, 107 int, int, int)); 108 static int uvn_put __P((struct uvm_object *, vm_page_t *, 109 int, boolean_t)); 110 static void uvn_reference __P((struct uvm_object *)); 111 static boolean_t uvn_releasepg __P((struct vm_page *, 112 struct vm_page **)); 113 114 /* 115 * master pager structure 116 */ 117 118 struct uvm_pagerops uvm_vnodeops = { 119 uvn_init, 120 uvn_attach, 121 uvn_reference, 122 uvn_detach, 123 NULL, /* no specialized fault routine required */ 124 uvn_flush, 125 uvn_get, 126 uvn_asyncget, 127 uvn_put, 128 uvn_cluster, 129 uvm_mk_pcluster, /* use generic version of this: see uvm_pager.c */ 130 uvm_shareprot, /* !NULL: allow us in share maps */ 131 NULL, /* AIO-DONE function (not until we have asyncio) */ 132 uvn_releasepg, 133 }; 134 135 /* 136 * the ops! 137 */ 138 139 /* 140 * uvn_init 141 * 142 * init pager private data structures. 143 */ 144 145 static void 146 uvn_init() 147 { 148 149 LIST_INIT(&uvn_wlist); 150 simple_lock_init(&uvn_wl_lock); 151 /* note: uvn_sync_q init'd in uvm_vnp_sync() */ 152 lockinit(&uvn_sync_lock, PVM, "uvnsync", 0, 0); 153 } 154 155 /* 156 * uvn_attach 157 * 158 * attach a vnode structure to a VM object. if the vnode is already 159 * attached, then just bump the reference count by one and return the 160 * VM object. if not already attached, attach and return the new VM obj. 161 * the "accessprot" tells the max access the attaching thread wants to 162 * our pages. 163 * 164 * => caller must _not_ already be holding the lock on the uvm_object. 165 * => in fact, nothing should be locked so that we can sleep here. 166 * => note that uvm_object is first thing in vnode structure, so their 167 * pointers are equiv. 168 */ 169 170 struct uvm_object * 171 uvn_attach(arg, accessprot) 172 void *arg; 173 vm_prot_t accessprot; 174 { 175 struct vnode *vp = arg; 176 struct uvm_vnode *uvn = &vp->v_uvm; 177 struct vattr vattr; 178 int oldflags, result; 179 u_quad_t used_vnode_size; 180 UVMHIST_FUNC("uvn_attach"); UVMHIST_CALLED(maphist); 181 182 UVMHIST_LOG(maphist, "(vn=0x%x)", arg,0,0,0); 183 184 /* 185 * first get a lock on the uvn. 186 */ 187 simple_lock(&uvn->u_obj.vmobjlock); 188 while (uvn->u_flags & UVM_VNODE_BLOCKED) { 189 uvn->u_flags |= UVM_VNODE_WANTED; 190 UVMHIST_LOG(maphist, " SLEEPING on blocked vn",0,0,0,0); 191 UVM_UNLOCK_AND_WAIT(uvn, &uvn->u_obj.vmobjlock, FALSE, 192 "uvn_attach", 0); 193 simple_lock(&uvn->u_obj.vmobjlock); 194 UVMHIST_LOG(maphist," WOKE UP",0,0,0,0); 195 } 196 197 /* 198 * now we have lock and uvn must not be in a blocked state. 199 * first check to see if it is already active, in which case 200 * we can bump the reference count, check to see if we need to 201 * add it to the writeable list, and then return. 202 */ 203 if (uvn->u_flags & UVM_VNODE_VALID) { /* already active? */ 204 205 /* regain VREF if we were persisting */ 206 if (uvn->u_obj.uo_refs == 0) { 207 VREF(vp); 208 UVMHIST_LOG(maphist," VREF (reclaim persisting vnode)", 209 0,0,0,0); 210 } 211 uvn->u_obj.uo_refs++; /* bump uvn ref! */ 212 213 /* check for new writeable uvn */ 214 if ((accessprot & VM_PROT_WRITE) != 0 && 215 (uvn->u_flags & UVM_VNODE_WRITEABLE) == 0) { 216 simple_lock(&uvn_wl_lock); 217 LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist); 218 simple_unlock(&uvn_wl_lock); 219 /* we are now on wlist! */ 220 uvn->u_flags |= UVM_VNODE_WRITEABLE; 221 } 222 223 /* unlock and return */ 224 simple_unlock(&uvn->u_obj.vmobjlock); 225 UVMHIST_LOG(maphist,"<- done, refcnt=%d", uvn->u_obj.uo_refs, 226 0, 0, 0); 227 return (&uvn->u_obj); 228 } 229 230 /* 231 * need to call VOP_GETATTR() to get the attributes, but that could 232 * block (due to I/O), so we want to unlock the object before calling. 233 * however, we want to keep anyone else from playing with the object 234 * while it is unlocked. to do this we set UVM_VNODE_ALOCK which 235 * prevents anyone from attaching to the vnode until we are done with 236 * it. 237 */ 238 uvn->u_flags = UVM_VNODE_ALOCK; 239 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock in case we sleep */ 240 /* XXX: curproc? */ 241 result = VOP_GETATTR(vp, &vattr, curproc->p_ucred, curproc); 242 243 /* 244 * make sure that the newsize fits within a vm_offset_t 245 * XXX: need to revise addressing data types 246 */ 247 used_vnode_size = vattr.va_size; 248 if (used_vnode_size > (vm_offset_t) -PAGE_SIZE) { 249 #ifdef DEBUG 250 printf("uvn_attach: vn %p size truncated %qx->%x\n", vp, 251 used_vnode_size, -PAGE_SIZE); 252 #endif 253 used_vnode_size = (vm_offset_t) -PAGE_SIZE; 254 } 255 256 /* relock object */ 257 simple_lock(&uvn->u_obj.vmobjlock); 258 259 if (result != 0) { 260 if (uvn->u_flags & UVM_VNODE_WANTED) 261 wakeup(uvn); 262 uvn->u_flags = 0; 263 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock */ 264 UVMHIST_LOG(maphist,"<- done (VOP_GETATTR FAILED!)", 0,0,0,0); 265 return(NULL); 266 } 267 268 /* 269 * now set up the uvn. 270 */ 271 uvn->u_obj.pgops = &uvm_vnodeops; 272 TAILQ_INIT(&uvn->u_obj.memq); 273 uvn->u_obj.uo_npages = 0; 274 uvn->u_obj.uo_refs = 1; /* just us... */ 275 oldflags = uvn->u_flags; 276 uvn->u_flags = UVM_VNODE_VALID|UVM_VNODE_CANPERSIST; 277 uvn->u_nio = 0; 278 uvn->u_size = used_vnode_size; 279 280 /* if write access, we need to add it to the wlist */ 281 if (accessprot & VM_PROT_WRITE) { 282 simple_lock(&uvn_wl_lock); 283 LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist); 284 simple_unlock(&uvn_wl_lock); 285 uvn->u_flags |= UVM_VNODE_WRITEABLE; /* we are on wlist! */ 286 } 287 288 /* 289 * add a reference to the vnode. this reference will stay as long 290 * as there is a valid mapping of the vnode. dropped when the 291 * reference count goes to zero [and we either free or persist]. 292 */ 293 VREF(vp); 294 simple_unlock(&uvn->u_obj.vmobjlock); 295 if (oldflags & UVM_VNODE_WANTED) 296 wakeup(uvn); 297 298 UVMHIST_LOG(maphist,"<- done/VREF, ret 0x%x", &uvn->u_obj,0,0,0); 299 return(&uvn->u_obj); 300 } 301 302 303 /* 304 * uvn_reference 305 * 306 * duplicate a reference to a VM object. Note that the reference 307 * count must already be at least one (the passed in reference) so 308 * there is no chance of the uvn being killed or locked out here. 309 * 310 * => caller must call with object unlocked. 311 * => caller must be using the same accessprot as was used at attach time 312 */ 313 314 315 static void 316 uvn_reference(uobj) 317 struct uvm_object *uobj; 318 { 319 #ifdef DIAGNOSTIC 320 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 321 #endif 322 UVMHIST_FUNC("uvn_reference"); UVMHIST_CALLED(maphist); 323 324 simple_lock(&uobj->vmobjlock); 325 #ifdef DIAGNOSTIC 326 if ((uvn->u_flags & UVM_VNODE_VALID) == 0) { 327 printf("uvn_reference: ref=%d, flags=0x%x\n", uvn->u_flags, 328 uobj->uo_refs); 329 panic("uvn_reference: invalid state"); 330 } 331 #endif 332 uobj->uo_refs++; 333 UVMHIST_LOG(maphist, "<- done (uobj=0x%x, ref = %d)", 334 uobj, uobj->uo_refs,0,0); 335 simple_unlock(&uobj->vmobjlock); 336 } 337 338 /* 339 * uvn_detach 340 * 341 * remove a reference to a VM object. 342 * 343 * => caller must call with object unlocked and map locked. 344 * => this starts the detach process, but doesn't have to finish it 345 * (async i/o could still be pending). 346 */ 347 static void 348 uvn_detach(uobj) 349 struct uvm_object *uobj; 350 { 351 struct uvm_vnode *uvn; 352 struct vnode *vp; 353 int oldflags; 354 UVMHIST_FUNC("uvn_detach"); UVMHIST_CALLED(maphist); 355 356 simple_lock(&uobj->vmobjlock); 357 358 UVMHIST_LOG(maphist," (uobj=0x%x) ref=%d", uobj,uobj->uo_refs,0,0); 359 uobj->uo_refs--; /* drop ref! */ 360 if (uobj->uo_refs) { /* still more refs */ 361 simple_unlock(&uobj->vmobjlock); 362 UVMHIST_LOG(maphist, "<- done (rc>0)", 0,0,0,0); 363 return; 364 } 365 366 /* 367 * get other pointers ... 368 */ 369 370 uvn = (struct uvm_vnode *) uobj; 371 vp = (struct vnode *) uobj; 372 373 /* 374 * clear VTEXT flag now that there are no mappings left (VTEXT is used 375 * to keep an active text file from being overwritten). 376 */ 377 vp->v_flag &= ~VTEXT; 378 379 /* 380 * we just dropped the last reference to the uvn. see if we can 381 * let it "stick around". 382 */ 383 384 if (uvn->u_flags & UVM_VNODE_CANPERSIST) { 385 /* won't block */ 386 uvn_flush(uobj, 0, 0, PGO_DEACTIVATE|PGO_ALLPAGES); 387 vrele(vp); /* drop vnode reference */ 388 simple_unlock(&uobj->vmobjlock); 389 UVMHIST_LOG(maphist,"<- done/vrele! (persist)", 0,0,0,0); 390 return; 391 } 392 393 /* 394 * its a goner! 395 */ 396 397 UVMHIST_LOG(maphist," its a goner (flushing)!", 0,0,0,0); 398 399 uvn->u_flags |= UVM_VNODE_DYING; 400 401 /* 402 * even though we may unlock in flush, no one can gain a reference 403 * to us until we clear the "dying" flag [because it blocks 404 * attaches]. we will not do that until after we've disposed of all 405 * the pages with uvn_flush(). note that before the flush the only 406 * pages that could be marked PG_BUSY are ones that are in async 407 * pageout by the daemon. (there can't be any pending "get"'s 408 * because there are no references to the object). 409 */ 410 411 (void) uvn_flush(uobj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES); 412 413 UVMHIST_LOG(maphist," its a goner (done flush)!", 0,0,0,0); 414 415 /* 416 * given the structure of this pager, the above flush request will 417 * create the following state: all the pages that were in the object 418 * have either been free'd or they are marked PG_BUSY|PG_RELEASED. 419 * the PG_BUSY bit was set either by us or the daemon for async I/O. 420 * in either case, if we have pages left we can't kill the object 421 * yet because i/o is pending. in this case we set the "relkill" 422 * flag which will cause pgo_releasepg to kill the object once all 423 * the I/O's are done [pgo_releasepg will be called from the aiodone 424 * routine or from the page daemon]. 425 */ 426 427 if (uobj->uo_npages) { /* I/O pending. iodone will free */ 428 #ifdef DIAGNOSTIC 429 /* 430 * XXXCDC: very unlikely to happen until we have async i/o 431 * so print a little info message in case it does. 432 */ 433 printf("uvn_detach: vn %p has pages left after flush - " 434 "relkill mode\n", uobj); 435 #endif 436 uvn->u_flags |= UVM_VNODE_RELKILL; 437 simple_unlock(&uobj->vmobjlock); 438 UVMHIST_LOG(maphist,"<- done! (releasepg will kill obj)", 0, 0, 439 0, 0); 440 return; 441 } 442 443 /* 444 * kill object now. note that we can't be on the sync q because 445 * all references are gone. 446 */ 447 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 448 simple_lock(&uvn_wl_lock); /* protect uvn_wlist */ 449 LIST_REMOVE(uvn, u_wlist); 450 simple_unlock(&uvn_wl_lock); 451 } 452 #ifdef DIAGNOSTIC 453 if (uobj->memq.tqh_first != NULL) 454 panic("uvn_deref: vnode VM object still has pages afer " 455 "syncio/free flush"); 456 #endif 457 oldflags = uvn->u_flags; 458 uvn->u_flags = 0; 459 simple_unlock(&uobj->vmobjlock); 460 461 /* wake up any sleepers */ 462 if (oldflags & UVM_VNODE_WANTED) 463 wakeup(uvn); 464 465 /* 466 * drop our reference to the vnode. 467 */ 468 vrele(vp); 469 UVMHIST_LOG(maphist,"<- done (vrele) final", 0,0,0,0); 470 471 return; 472 } 473 474 /* 475 * uvm_vnp_terminate: external hook to clear out a vnode's VM 476 * 477 * called in two cases: 478 * [1] when a persisting vnode vm object (i.e. one with a zero reference 479 * count) needs to be freed so that a vnode can be reused. this 480 * happens under "getnewvnode" in vfs_subr.c. if the vnode from 481 * the free list is still attached (i.e. not VBAD) then vgone is 482 * called. as part of the vgone trace this should get called to 483 * free the vm object. this is the common case. 484 * [2] when a filesystem is being unmounted by force (MNT_FORCE, 485 * "umount -f") the vgone() function is called on active vnodes 486 * on the mounted file systems to kill their data (the vnodes become 487 * "dead" ones [see src/sys/miscfs/deadfs/...]). that results in a 488 * call here (even if the uvn is still in use -- i.e. has a non-zero 489 * reference count). this case happens at "umount -f" and during a 490 * "reboot/halt" operation. 491 * 492 * => the caller must XLOCK and VOP_LOCK the vnode before calling us 493 * [protects us from getting a vnode that is already in the DYING 494 * state...] 495 * => unlike uvn_detach, this function must not return until all the 496 * uvn's pages are disposed of. 497 * => in case [2] the uvn is still alive after this call, but all I/O 498 * ops will fail (due to the backing vnode now being "dead"). this 499 * will prob. kill any process using the uvn due to pgo_get failing. 500 */ 501 502 void 503 uvm_vnp_terminate(vp) 504 struct vnode *vp; 505 { 506 struct uvm_vnode *uvn = &vp->v_uvm; 507 int oldflags; 508 UVMHIST_FUNC("uvm_vnp_terminate"); UVMHIST_CALLED(maphist); 509 510 /* 511 * lock object and check if it is valid 512 */ 513 simple_lock(&uvn->u_obj.vmobjlock); 514 UVMHIST_LOG(maphist, " vp=0x%x, ref=%d, flag=0x%x", vp, 515 uvn->u_obj.uo_refs, uvn->u_flags, 0); 516 if ((uvn->u_flags & UVM_VNODE_VALID) == 0) { 517 simple_unlock(&uvn->u_obj.vmobjlock); 518 UVMHIST_LOG(maphist, "<- done (not active)", 0, 0, 0, 0); 519 return; 520 } 521 522 /* 523 * must be a valid uvn that is not already dying (because XLOCK 524 * protects us from that). the uvn can't in the the ALOCK state 525 * because it is valid, and uvn's that are in the ALOCK state haven't 526 * been marked valid yet. 527 */ 528 529 #ifdef DEBUG 530 /* 531 * debug check: are we yanking the vnode out from under our uvn? 532 */ 533 if (uvn->u_obj.uo_refs) { 534 printf("uvm_vnp_terminate(%p): terminating active vnode " 535 "(refs=%d)\n", uvn, uvn->u_obj.uo_refs); 536 } 537 #endif 538 539 /* 540 * it is possible that the uvn was detached and is in the relkill 541 * state [i.e. waiting for async i/o to finish so that releasepg can 542 * kill object]. we take over the vnode now and cancel the relkill. 543 * we want to know when the i/o is done so we can recycle right 544 * away. note that a uvn can only be in the RELKILL state if it 545 * has a zero reference count. 546 */ 547 548 if (uvn->u_flags & UVM_VNODE_RELKILL) 549 uvn->u_flags &= ~UVM_VNODE_RELKILL; /* cancel RELKILL */ 550 551 /* 552 * block the uvn by setting the dying flag, and then flush the 553 * pages. (note that flush may unlock object while doing I/O, but 554 * it will re-lock it before it returns control here). 555 * 556 * also, note that we tell I/O that we are already VOP_LOCK'd so 557 * that uvn_io doesn't attempt to VOP_LOCK again. 558 * 559 * XXXCDC: setting VNISLOCKED on an active uvn which is being terminated 560 * due to a forceful unmount might not be a good idea. maybe we 561 * need a way to pass in this info to uvn_flush through a 562 * pager-defined PGO_ constant [currently there are none]. 563 */ 564 uvn->u_flags |= UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED; 565 566 (void) uvn_flush(&uvn->u_obj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES); 567 568 /* 569 * as we just did a flush we expect all the pages to be gone or in 570 * the process of going. sleep to wait for the rest to go [via iosync]. 571 */ 572 573 while (uvn->u_obj.uo_npages) { 574 #ifdef DIAGNOSTIC 575 struct vm_page *pp; 576 for (pp = uvn->u_obj.memq.tqh_first ; pp != NULL ; 577 pp = pp->listq.tqe_next) { 578 if ((pp->flags & PG_BUSY) == 0) 579 panic("uvm_vnp_terminate: detected unbusy pg"); 580 } 581 if (uvn->u_nio == 0) 582 panic("uvm_vnp_terminate: no I/O to wait for?"); 583 printf("uvm_vnp_terminate: waiting for I/O to fin.\n"); 584 /* 585 * XXXCDC: this is unlikely to happen without async i/o so we 586 * put a printf in just to keep an eye on it. 587 */ 588 #endif 589 uvn->u_flags |= UVM_VNODE_IOSYNC; 590 UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock, FALSE, 591 "uvn_term",0); 592 simple_lock(&uvn->u_obj.vmobjlock); 593 } 594 595 /* 596 * done. now we free the uvn if its reference count is zero 597 * (true if we are zapping a persisting uvn). however, if we are 598 * terminating a uvn with active mappings we let it live ... future 599 * calls down to the vnode layer will fail. 600 */ 601 602 oldflags = uvn->u_flags; 603 if (uvn->u_obj.uo_refs) { 604 605 /* 606 * uvn must live on it is dead-vnode state until all references 607 * are gone. restore flags. clear CANPERSIST state. 608 */ 609 610 uvn->u_flags &= ~(UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED| 611 UVM_VNODE_WANTED|UVM_VNODE_CANPERSIST); 612 613 } else { 614 615 /* 616 * free the uvn now. note that the VREF reference is already 617 * gone [it is dropped when we enter the persist state]. 618 */ 619 if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED) 620 panic("uvm_vnp_terminate: io sync wanted bit set"); 621 622 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 623 simple_lock(&uvn_wl_lock); 624 LIST_REMOVE(uvn, u_wlist); 625 simple_unlock(&uvn_wl_lock); 626 } 627 uvn->u_flags = 0; /* uvn is history, clear all bits */ 628 } 629 630 if (oldflags & UVM_VNODE_WANTED) 631 wakeup(uvn); /* object lock still held */ 632 633 simple_unlock(&uvn->u_obj.vmobjlock); 634 UVMHIST_LOG(maphist, "<- done", 0, 0, 0, 0); 635 636 } 637 638 /* 639 * uvn_releasepg: handled a released page in a uvn 640 * 641 * => "pg" is a PG_BUSY [caller owns it], PG_RELEASED page that we need 642 * to dispose of. 643 * => caller must handled PG_WANTED case 644 * => called with page's object locked, pageq's unlocked 645 * => returns TRUE if page's object is still alive, FALSE if we 646 * killed the page's object. if we return TRUE, then we 647 * return with the object locked. 648 * => if (nextpgp != NULL) => we return pageq.tqe_next here, and return 649 * with the page queues locked [for pagedaemon] 650 * => if (nextpgp == NULL) => we return with page queues unlocked [normal case] 651 * => we kill the uvn if it is not referenced and we are suppose to 652 * kill it ("relkill"). 653 */ 654 655 boolean_t 656 uvn_releasepg(pg, nextpgp) 657 struct vm_page *pg; 658 struct vm_page **nextpgp; /* OUT */ 659 { 660 struct uvm_vnode *uvn = (struct uvm_vnode *) pg->uobject; 661 #ifdef DIAGNOSTIC 662 if ((pg->flags & PG_RELEASED) == 0) 663 panic("uvn_releasepg: page not released!"); 664 #endif 665 666 /* 667 * dispose of the page [caller handles PG_WANTED] 668 */ 669 pmap_page_protect(PMAP_PGARG(pg), VM_PROT_NONE); 670 uvm_lock_pageq(); 671 if (nextpgp) 672 *nextpgp = pg->pageq.tqe_next; /* next page for daemon */ 673 uvm_pagefree(pg); 674 if (!nextpgp) 675 uvm_unlock_pageq(); 676 677 /* 678 * now see if we need to kill the object 679 */ 680 if (uvn->u_flags & UVM_VNODE_RELKILL) { 681 if (uvn->u_obj.uo_refs) 682 panic("uvn_releasepg: kill flag set on referenced " 683 "object!"); 684 if (uvn->u_obj.uo_npages == 0) { 685 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 686 simple_lock(&uvn_wl_lock); 687 LIST_REMOVE(uvn, u_wlist); 688 simple_unlock(&uvn_wl_lock); 689 } 690 #ifdef DIAGNOSTIC 691 if (uvn->u_obj.memq.tqh_first) 692 panic("uvn_releasepg: pages in object with npages == 0"); 693 #endif 694 if (uvn->u_flags & UVM_VNODE_WANTED) 695 /* still holding object lock */ 696 wakeup(uvn); 697 698 uvn->u_flags = 0; /* DEAD! */ 699 simple_unlock(&uvn->u_obj.vmobjlock); 700 return (FALSE); 701 } 702 } 703 return (TRUE); 704 } 705 706 /* 707 * NOTE: currently we have to use VOP_READ/VOP_WRITE because they go 708 * through the buffer cache and allow I/O in any size. These VOPs use 709 * synchronous i/o. [vs. VOP_STRATEGY which can be async, but doesn't 710 * go through the buffer cache or allow I/O sizes larger than a 711 * block]. we will eventually want to change this. 712 * 713 * issues to consider: 714 * uvm provides the uvm_aiodesc structure for async i/o management. 715 * there are two tailq's in the uvm. structure... one for pending async 716 * i/o and one for "done" async i/o. to do an async i/o one puts 717 * an aiodesc on the "pending" list (protected by splbio()), starts the 718 * i/o and returns VM_PAGER_PEND. when the i/o is done, we expect 719 * some sort of "i/o done" function to be called (at splbio(), interrupt 720 * time). this function should remove the aiodesc from the pending list 721 * and place it on the "done" list and wakeup the daemon. the daemon 722 * will run at normal spl() and will remove all items from the "done" 723 * list and call the "aiodone" hook for each done request (see uvm_pager.c). 724 * [in the old vm code, this was done by calling the "put" routine with 725 * null arguments which made the code harder to read and understand because 726 * you had one function ("put") doing two things.] 727 * 728 * so the current pager needs: 729 * int uvn_aiodone(struct uvm_aiodesc *) 730 * 731 * => return KERN_SUCCESS (aio finished, free it). otherwise requeue for 732 * later collection. 733 * => called with pageq's locked by the daemon. 734 * 735 * general outline: 736 * - "try" to lock object. if fail, just return (will try again later) 737 * - drop "u_nio" (this req is done!) 738 * - if (object->iosync && u_naio == 0) { wakeup &uvn->u_naio } 739 * - get "page" structures (atop?). 740 * - handle "wanted" pages 741 * - handle "released" pages [using pgo_releasepg] 742 * >>> pgo_releasepg may kill the object 743 * dont forget to look at "object" wanted flag in all cases. 744 */ 745 746 747 /* 748 * uvn_flush: flush pages out of a uvm object. 749 * 750 * => object should be locked by caller. we may _unlock_ the object 751 * if (and only if) we need to clean a page (PGO_CLEANIT). 752 * we return with the object locked. 753 * => if PGO_CLEANIT is set, we may block (due to I/O). thus, a caller 754 * might want to unlock higher level resources (e.g. vm_map) 755 * before calling flush. 756 * => if PGO_CLEANIT is not set, then we will neither unlock the object 757 * or block. 758 * => if PGO_ALLPAGE is set, then all pages in the object are valid targets 759 * for flushing. 760 * => NOTE: we rely on the fact that the object's memq is a TAILQ and 761 * that new pages are inserted on the tail end of the list. thus, 762 * we can make a complete pass through the object in one go by starting 763 * at the head and working towards the tail (new pages are put in 764 * front of us). 765 * => NOTE: we are allowed to lock the page queues, so the caller 766 * must not be holding the lock on them [e.g. pagedaemon had 767 * better not call us with the queues locked] 768 * => we return TRUE unless we encountered some sort of I/O error 769 * 770 * comment on "cleaning" object and PG_BUSY pages: 771 * this routine is holding the lock on the object. the only time 772 * that it can run into a PG_BUSY page that it does not own is if 773 * some other process has started I/O on the page (e.g. either 774 * a pagein, or a pageout). if the PG_BUSY page is being paged 775 * in, then it can not be dirty (!PG_CLEAN) because no one has 776 * had a chance to modify it yet. if the PG_BUSY page is being 777 * paged out then it means that someone else has already started 778 * cleaning the page for us (how nice!). in this case, if we 779 * have syncio specified, then after we make our pass through the 780 * object we need to wait for the other PG_BUSY pages to clear 781 * off (i.e. we need to do an iosync). also note that once a 782 * page is PG_BUSY it must stay in its object until it is un-busyed. 783 * 784 * note on page traversal: 785 * we can traverse the pages in an object either by going down the 786 * linked list in "uobj->memq", or we can go over the address range 787 * by page doing hash table lookups for each address. depending 788 * on how many pages are in the object it may be cheaper to do one 789 * or the other. we set "by_list" to true if we are using memq. 790 * if the cost of a hash lookup was equal to the cost of the list 791 * traversal we could compare the number of pages in the start->stop 792 * range to the total number of pages in the object. however, it 793 * seems that a hash table lookup is more expensive than the linked 794 * list traversal, so we multiply the number of pages in the 795 * start->stop range by a penalty which we define below. 796 */ 797 798 #define UVN_HASH_PENALTY 4 /* XXX: a guess */ 799 800 static boolean_t 801 uvn_flush(uobj, start, stop, flags) 802 struct uvm_object *uobj; 803 vm_offset_t start, stop; 804 int flags; 805 { 806 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 807 struct vm_page *pp, *ppnext, *ptmp; 808 struct vm_page *pps[MAXBSIZE/PAGE_SIZE], **ppsp; 809 int npages, result, lcv; 810 boolean_t retval, need_iosync, by_list, needs_clean; 811 vm_offset_t curoff; 812 u_short pp_version; 813 UVMHIST_FUNC("uvn_flush"); UVMHIST_CALLED(maphist); 814 815 curoff = 0; /* XXX: shut up gcc */ 816 /* 817 * get init vals and determine how we are going to traverse object 818 */ 819 820 need_iosync = FALSE; 821 retval = TRUE; /* return value */ 822 if (flags & PGO_ALLPAGES) { 823 start = 0; 824 stop = round_page(uvn->u_size); 825 by_list = TRUE; /* always go by the list */ 826 } else { 827 start = trunc_page(start); 828 stop = round_page(stop); 829 if (stop > round_page(uvn->u_size)) 830 printf("uvn_flush: strange, got an out of range " 831 "flush (fixed)\n"); 832 833 by_list = (uobj->uo_npages <= 834 ((stop - start) / PAGE_SIZE) * UVN_HASH_PENALTY); 835 } 836 837 UVMHIST_LOG(maphist, 838 " flush start=0x%x, stop=0x%x, by_list=%d, flags=0x%x", 839 start, stop, by_list, flags); 840 841 /* 842 * PG_CLEANCHK: this bit is used by the pgo_mk_pcluster function as 843 * a _hint_ as to how up to date the PG_CLEAN bit is. if the hint 844 * is wrong it will only prevent us from clustering... it won't break 845 * anything. we clear all PG_CLEANCHK bits here, and pgo_mk_pcluster 846 * will set them as it syncs PG_CLEAN. This is only an issue if we 847 * are looking at non-inactive pages (because inactive page's PG_CLEAN 848 * bit is always up to date since there are no mappings). 849 * [borrowed PG_CLEANCHK idea from FreeBSD VM] 850 */ 851 852 if ((flags & PGO_CLEANIT) != 0 && 853 uobj->pgops->pgo_mk_pcluster != NULL) { 854 if (by_list) { 855 for (pp = uobj->memq.tqh_first ; pp != NULL ; 856 pp = pp->listq.tqe_next) { 857 if (pp->offset < start || pp->offset >= stop) 858 continue; 859 pp->flags &= ~PG_CLEANCHK; 860 } 861 862 } else { /* by hash */ 863 for (curoff = start ; curoff < stop; 864 curoff += PAGE_SIZE) { 865 pp = uvm_pagelookup(uobj, curoff); 866 if (pp) 867 pp->flags &= ~PG_CLEANCHK; 868 } 869 } 870 } 871 872 /* 873 * now do it. note: we must update ppnext in body of loop or we 874 * will get stuck. we need to use ppnext because we may free "pp" 875 * before doing the next loop. 876 */ 877 878 if (by_list) { 879 pp = uobj->memq.tqh_first; 880 } else { 881 curoff = start; 882 pp = uvm_pagelookup(uobj, curoff); 883 } 884 885 ppnext = NULL; /* XXX: shut up gcc */ 886 ppsp = NULL; /* XXX: shut up gcc */ 887 uvm_lock_pageq(); /* page queues locked */ 888 889 /* locked: both page queues and uobj */ 890 for ( ; (by_list && pp != NULL) || 891 (!by_list && curoff < stop) ; pp = ppnext) { 892 893 if (by_list) { 894 895 /* 896 * range check 897 */ 898 899 if (pp->offset < start || pp->offset >= stop) { 900 ppnext = pp->listq.tqe_next; 901 continue; 902 } 903 904 } else { 905 906 /* 907 * null check 908 */ 909 910 curoff += PAGE_SIZE; 911 if (pp == NULL) { 912 if (curoff < stop) 913 ppnext = uvm_pagelookup(uobj, curoff); 914 continue; 915 } 916 917 } 918 919 /* 920 * handle case where we do not need to clean page (either 921 * because we are not clean or because page is not dirty or 922 * is busy): 923 * 924 * NOTE: we are allowed to deactivate a non-wired active 925 * PG_BUSY page, but once a PG_BUSY page is on the inactive 926 * queue it must stay put until it is !PG_BUSY (so as not to 927 * confuse pagedaemon). 928 */ 929 930 if ((flags & PGO_CLEANIT) == 0 || (pp->flags & PG_BUSY) != 0) { 931 needs_clean = FALSE; 932 if ((pp->flags & PG_BUSY) != 0 && 933 (flags & (PGO_CLEANIT|PGO_SYNCIO)) == 934 (PGO_CLEANIT|PGO_SYNCIO)) 935 need_iosync = TRUE; 936 } else { 937 /* 938 * freeing: nuke all mappings so we can sync 939 * PG_CLEAN bit with no race 940 */ 941 if ((pp->flags & PG_CLEAN) != 0 && 942 (flags & PGO_FREE) != 0 && 943 (pp->pqflags & PQ_ACTIVE) != 0) 944 pmap_page_protect(PMAP_PGARG(pp), VM_PROT_NONE); 945 if ((pp->flags & PG_CLEAN) != 0 && 946 pmap_is_modified(PMAP_PGARG(pp))) 947 pp->flags &= ~(PG_CLEAN); 948 pp->flags |= PG_CLEANCHK; /* update "hint" */ 949 950 needs_clean = ((pp->flags & PG_CLEAN) == 0); 951 } 952 953 /* 954 * if we don't need a clean... load ppnext and dispose of pp 955 */ 956 if (!needs_clean) { 957 /* load ppnext */ 958 if (by_list) 959 ppnext = pp->listq.tqe_next; 960 else { 961 if (curoff < stop) 962 ppnext = uvm_pagelookup(uobj, curoff); 963 } 964 965 /* now dispose of pp */ 966 if (flags & PGO_DEACTIVATE) { 967 if ((pp->pqflags & PQ_INACTIVE) == 0 && 968 pp->wire_count == 0) { 969 pmap_page_protect(PMAP_PGARG(pp), 970 VM_PROT_NONE); 971 uvm_pagedeactivate(pp); 972 } 973 974 } else if (flags & PGO_FREE) { 975 if (pp->flags & PG_BUSY) { 976 /* release busy pages */ 977 pp->flags |= PG_RELEASED; 978 } else { 979 pmap_page_protect(PMAP_PGARG(pp), 980 VM_PROT_NONE); 981 /* removed page from object */ 982 uvm_pagefree(pp); 983 } 984 } 985 /* ppnext is valid so we can continue... */ 986 continue; 987 } 988 989 /* 990 * pp points to a page in the locked object that we are 991 * working on. if it is !PG_CLEAN,!PG_BUSY and we asked 992 * for cleaning (PGO_CLEANIT). we clean it now. 993 * 994 * let uvm_pager_put attempted a clustered page out. 995 * note: locked: uobj and page queues. 996 */ 997 998 pp->flags |= PG_BUSY; /* we 'own' page now */ 999 UVM_PAGE_OWN(pp, "uvn_flush"); 1000 pmap_page_protect(PMAP_PGARG(pp), VM_PROT_READ); 1001 pp_version = pp->version; 1002 ReTry: 1003 ppsp = pps; 1004 npages = sizeof(pps) / sizeof(struct vm_page *); 1005 1006 /* locked: page queues, uobj */ 1007 result = uvm_pager_put(uobj, pp, &ppsp, &npages, 1008 flags | PGO_DOACTCLUST, start, stop); 1009 /* unlocked: page queues, uobj */ 1010 1011 /* 1012 * at this point nothing is locked. if we did an async I/O 1013 * it is remotely possible for the async i/o to complete and 1014 * the page "pp" be freed or what not before we get a chance 1015 * to relock the object. in order to detect this, we have 1016 * saved the version number of the page in "pp_version". 1017 */ 1018 1019 /* relock! */ 1020 simple_lock(&uobj->vmobjlock); 1021 uvm_lock_pageq(); 1022 1023 /* 1024 * VM_PAGER_AGAIN: given the structure of this pager, this 1025 * can only happen when we are doing async I/O and can't 1026 * map the pages into kernel memory (pager_map) due to lack 1027 * of vm space. if this happens we drop back to sync I/O. 1028 */ 1029 1030 if (result == VM_PAGER_AGAIN) { 1031 /* 1032 * it is unlikely, but page could have been released 1033 * while we had the object lock dropped. we ignore 1034 * this now and retry the I/O. we will detect and 1035 * handle the released page after the syncio I/O 1036 * completes. 1037 */ 1038 #ifdef DIAGNOSTIC 1039 if (flags & PGO_SYNCIO) 1040 panic("uvn_flush: PGO_SYNCIO return 'try again' error (impossible)"); 1041 #endif 1042 flags |= PGO_SYNCIO; 1043 goto ReTry; 1044 } 1045 1046 /* 1047 * the cleaning operation is now done. finish up. note that 1048 * on error (!OK, !PEND) uvm_pager_put drops the cluster for us. 1049 * if success (OK, PEND) then uvm_pager_put returns the cluster 1050 * to us in ppsp/npages. 1051 */ 1052 1053 /* 1054 * for pending async i/o if we are not deactivating/freeing 1055 * we can move on to the next page. 1056 */ 1057 1058 if (result == VM_PAGER_PEND) { 1059 1060 if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) { 1061 /* 1062 * no per-page ops: refresh ppnext and continue 1063 */ 1064 if (by_list) { 1065 if (pp->version == pp_version) 1066 ppnext = pp->listq.tqe_next; 1067 else 1068 /* reset */ 1069 ppnext = uobj->memq.tqh_first; 1070 } else { 1071 if (curoff < stop) 1072 ppnext = uvm_pagelookup(uobj, 1073 curoff); 1074 } 1075 continue; 1076 } 1077 1078 /* need to do anything here? */ 1079 } 1080 1081 /* 1082 * need to look at each page of the I/O operation. we defer 1083 * processing "pp" until the last trip through this "for" loop 1084 * so that we can load "ppnext" for the main loop after we 1085 * play with the cluster pages [thus the "npages + 1" in the 1086 * loop below]. 1087 */ 1088 1089 for (lcv = 0 ; lcv < npages + 1 ; lcv++) { 1090 1091 /* 1092 * handle ppnext for outside loop, and saving pp 1093 * until the end. 1094 */ 1095 if (lcv < npages) { 1096 if (ppsp[lcv] == pp) 1097 continue; /* skip pp until the end */ 1098 ptmp = ppsp[lcv]; 1099 } else { 1100 ptmp = pp; 1101 1102 /* set up next page for outer loop */ 1103 if (by_list) { 1104 if (pp->version == pp_version) 1105 ppnext = pp->listq.tqe_next; 1106 else 1107 /* reset */ 1108 ppnext = uobj->memq.tqh_first; 1109 } else { 1110 if (curoff < stop) 1111 ppnext = uvm_pagelookup(uobj, curoff); 1112 } 1113 } 1114 1115 /* 1116 * verify the page didn't get moved while obj was 1117 * unlocked 1118 */ 1119 if (result == VM_PAGER_PEND && ptmp->uobject != uobj) 1120 continue; 1121 1122 /* 1123 * unbusy the page if I/O is done. note that for 1124 * pending I/O it is possible that the I/O op 1125 * finished before we relocked the object (in 1126 * which case the page is no longer busy). 1127 */ 1128 1129 if (result != VM_PAGER_PEND) { 1130 if (ptmp->flags & PG_WANTED) 1131 /* still holding object lock */ 1132 thread_wakeup(ptmp); 1133 1134 ptmp->flags &= ~(PG_WANTED|PG_BUSY); 1135 UVM_PAGE_OWN(ptmp, NULL); 1136 if (ptmp->flags & PG_RELEASED) { 1137 1138 /* pgo_releasepg wants this */ 1139 uvm_unlock_pageq(); 1140 if (!uvn_releasepg(ptmp, NULL)) 1141 return (TRUE); 1142 1143 uvm_lock_pageq(); /* relock */ 1144 continue; /* next page */ 1145 1146 } else { 1147 ptmp->flags |= (PG_CLEAN|PG_CLEANCHK); 1148 if ((flags & PGO_FREE) == 0) 1149 pmap_clear_modify( 1150 PMAP_PGARG(ptmp)); 1151 } 1152 } 1153 1154 /* 1155 * dispose of page 1156 */ 1157 1158 if (flags & PGO_DEACTIVATE) { 1159 if ((pp->pqflags & PQ_INACTIVE) == 0 && 1160 pp->wire_count == 0) { 1161 pmap_page_protect(PMAP_PGARG(ptmp), 1162 VM_PROT_NONE); 1163 uvm_pagedeactivate(ptmp); 1164 } 1165 1166 } else if (flags & PGO_FREE) { 1167 if (result == VM_PAGER_PEND) { 1168 if ((ptmp->flags & PG_BUSY) != 0) 1169 /* signal for i/o done */ 1170 ptmp->flags |= PG_RELEASED; 1171 } else { 1172 if (result != VM_PAGER_OK) { 1173 printf("uvn_flush: obj=%p, " 1174 "offset=0x%lx. error " 1175 "during pageout.\n", 1176 pp->uobject, pp->offset); 1177 printf("uvn_flush: WARNING: " 1178 "changes to page may be " 1179 "lost!\n"); 1180 retval = FALSE; 1181 } 1182 pmap_page_protect(PMAP_PGARG(ptmp), 1183 VM_PROT_NONE); 1184 uvm_pagefree(ptmp); 1185 } 1186 } 1187 1188 } /* end of "lcv" for loop */ 1189 1190 } /* end of "pp" for loop */ 1191 1192 /* 1193 * done with pagequeues: unlock 1194 */ 1195 uvm_unlock_pageq(); 1196 1197 /* 1198 * now wait for all I/O if required. 1199 */ 1200 if (need_iosync) { 1201 1202 UVMHIST_LOG(maphist," <<DOING IOSYNC>>",0,0,0,0); 1203 while (uvn->u_nio != 0) { 1204 uvn->u_flags |= UVM_VNODE_IOSYNC; 1205 UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock, 1206 FALSE, "uvn_flush",0); 1207 simple_lock(&uvn->u_obj.vmobjlock); 1208 } 1209 if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED) 1210 wakeup(&uvn->u_flags); 1211 uvn->u_flags &= ~(UVM_VNODE_IOSYNC|UVM_VNODE_IOSYNCWANTED); 1212 } 1213 1214 /* return, with object locked! */ 1215 UVMHIST_LOG(maphist,"<- done (retval=0x%x)",retval,0,0,0); 1216 return(retval); 1217 } 1218 1219 /* 1220 * uvn_cluster 1221 * 1222 * we are about to do I/O in an object at offset. this function is called 1223 * to establish a range of offsets around "offset" in which we can cluster 1224 * I/O. 1225 * 1226 * - currently doesn't matter if obj locked or not. 1227 */ 1228 1229 static void 1230 uvn_cluster(uobj, offset, loffset, hoffset) 1231 struct uvm_object *uobj; 1232 vm_offset_t offset; 1233 vm_offset_t *loffset, *hoffset; /* OUT */ 1234 { 1235 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 1236 *loffset = offset; 1237 1238 if (*loffset >= uvn->u_size) 1239 panic("uvn_cluster: offset out of range"); 1240 1241 /* 1242 * XXX: old pager claims we could use VOP_BMAP to get maxcontig value. 1243 */ 1244 *hoffset = *loffset + MAXBSIZE; 1245 if (*hoffset > round_page(uvn->u_size)) /* past end? */ 1246 *hoffset = round_page(uvn->u_size); 1247 1248 return; 1249 } 1250 1251 /* 1252 * uvn_put: flush page data to backing store. 1253 * 1254 * => prefer map unlocked (not required) 1255 * => object must be locked! we will _unlock_ it before starting I/O. 1256 * => flags: PGO_SYNCIO -- use sync. I/O 1257 * => note: caller must set PG_CLEAN and pmap_clear_modify (if needed) 1258 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync. 1259 * [thus we never do async i/o! see iodone comment] 1260 */ 1261 1262 static int 1263 uvn_put(uobj, pps, npages, flags) 1264 struct uvm_object *uobj; 1265 struct vm_page **pps; 1266 int npages, flags; 1267 { 1268 int retval; 1269 1270 /* note: object locked */ 1271 retval = uvn_io((struct uvm_vnode*)uobj, pps, npages, flags, UIO_WRITE); 1272 /* note: object unlocked */ 1273 1274 return(retval); 1275 } 1276 1277 1278 /* 1279 * uvn_get: get pages (synchronously) from backing store 1280 * 1281 * => prefer map unlocked (not required) 1282 * => object must be locked! we will _unlock_ it before starting any I/O. 1283 * => flags: PGO_ALLPAGES: get all of the pages 1284 * PGO_LOCKED: fault data structures are locked 1285 * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx] 1286 * => NOTE: caller must check for released pages!! 1287 */ 1288 1289 static int 1290 uvn_get(uobj, offset, pps, npagesp, centeridx, access_type, advice, flags) 1291 struct uvm_object *uobj; 1292 vm_offset_t offset; 1293 struct vm_page **pps; /* IN/OUT */ 1294 int *npagesp; /* IN (OUT if PGO_LOCKED) */ 1295 int centeridx, advice, flags; 1296 vm_prot_t access_type; 1297 { 1298 vm_offset_t current_offset; 1299 struct vm_page *ptmp; 1300 int lcv, result, gotpages; 1301 boolean_t done; 1302 UVMHIST_FUNC("uvn_get"); UVMHIST_CALLED(maphist); 1303 UVMHIST_LOG(maphist, "flags=%d", flags,0,0,0); 1304 1305 /* 1306 * step 1: handled the case where fault data structures are locked. 1307 */ 1308 1309 if (flags & PGO_LOCKED) { 1310 1311 /* 1312 * gotpages is the current number of pages we've gotten (which 1313 * we pass back up to caller via *npagesp. 1314 */ 1315 1316 gotpages = 0; 1317 1318 /* 1319 * step 1a: get pages that are already resident. only do this 1320 * if the data structures are locked (i.e. the first time 1321 * through). 1322 */ 1323 1324 done = TRUE; /* be optimistic */ 1325 1326 for (lcv = 0, current_offset = offset ; lcv < *npagesp ; 1327 lcv++, current_offset += PAGE_SIZE) { 1328 1329 /* do we care about this page? if not, skip it */ 1330 if (pps[lcv] == PGO_DONTCARE) 1331 continue; 1332 1333 /* lookup page */ 1334 ptmp = uvm_pagelookup(uobj, current_offset); 1335 1336 /* to be useful must get a non-busy, non-released pg */ 1337 if (ptmp == NULL || 1338 (ptmp->flags & (PG_BUSY|PG_RELEASED)) != 0) { 1339 if (lcv == centeridx || (flags & PGO_ALLPAGES) 1340 != 0) 1341 done = FALSE; /* need to do a wait or I/O! */ 1342 continue; 1343 } 1344 1345 /* 1346 * useful page: busy/lock it and plug it in our 1347 * result array 1348 */ 1349 ptmp->flags |= PG_BUSY; /* loan up to caller */ 1350 UVM_PAGE_OWN(ptmp, "uvn_get1"); 1351 pps[lcv] = ptmp; 1352 gotpages++; 1353 1354 } /* "for" lcv loop */ 1355 1356 /* 1357 * XXX: given the "advice", should we consider async read-ahead? 1358 * XXX: fault current does deactive of pages behind us. is 1359 * this good (other callers might now). 1360 */ 1361 /* 1362 * XXX: read-ahead currently handled by buffer cache (bread) 1363 * level. 1364 * XXX: no async i/o available. 1365 * XXX: so we don't do anything now. 1366 */ 1367 1368 /* 1369 * step 1c: now we've either done everything needed or we to 1370 * unlock and do some waiting or I/O. 1371 */ 1372 1373 *npagesp = gotpages; /* let caller know */ 1374 if (done) 1375 return(VM_PAGER_OK); /* bingo! */ 1376 else 1377 /* EEK! Need to unlock and I/O */ 1378 return(VM_PAGER_UNLOCK); 1379 } 1380 1381 /* 1382 * step 2: get non-resident or busy pages. 1383 * object is locked. data structures are unlocked. 1384 * 1385 * XXX: because we can't do async I/O at this level we get things 1386 * page at a time (otherwise we'd chunk). the VOP_READ() will do 1387 * async-read-ahead for us at a lower level. 1388 */ 1389 1390 for (lcv = 0, current_offset = offset ; 1391 lcv < *npagesp ; lcv++, current_offset += PAGE_SIZE) { 1392 1393 /* skip over pages we've already gotten or don't want */ 1394 /* skip over pages we don't _have_ to get */ 1395 if (pps[lcv] != NULL || (lcv != centeridx && 1396 (flags & PGO_ALLPAGES) == 0)) 1397 continue; 1398 1399 /* 1400 * we have yet to locate the current page (pps[lcv]). we first 1401 * look for a page that is already at the current offset. if 1402 * we fine a page, we check to see if it is busy or released. 1403 * if that is the case, then we sleep on the page until it is 1404 * no longer busy or released and repeat the lookup. if the 1405 * page we found is neither busy nor released, then we busy it 1406 * (so we own it) and plug it into pps[lcv]. this breaks the 1407 * following while loop and indicates we are ready to move on 1408 * to the next page in the "lcv" loop above. 1409 * 1410 * if we exit the while loop with pps[lcv] still set to NULL, 1411 * then it means that we allocated a new busy/fake/clean page 1412 * ptmp in the object and we need to do I/O to fill in the data. 1413 */ 1414 1415 while (pps[lcv] == NULL) { /* top of "pps" while loop */ 1416 1417 /* look for a current page */ 1418 ptmp = uvm_pagelookup(uobj, current_offset); 1419 1420 /* nope? allocate one now (if we can) */ 1421 if (ptmp == NULL) { 1422 1423 ptmp = uvm_pagealloc(uobj, current_offset, 1424 NULL); /* alloc */ 1425 1426 /* out of RAM? */ 1427 if (ptmp == NULL) { 1428 simple_unlock(&uobj->vmobjlock); 1429 uvm_wait("uvn_getpage"); 1430 simple_lock(&uobj->vmobjlock); 1431 1432 /* goto top of pps while loop */ 1433 continue; 1434 } 1435 1436 /* 1437 * got new page ready for I/O. break pps 1438 * while loop. pps[lcv] is still NULL. 1439 */ 1440 break; 1441 } 1442 1443 /* page is there, see if we need to wait on it */ 1444 if ((ptmp->flags & (PG_BUSY|PG_RELEASED)) != 0) { 1445 ptmp->flags |= PG_WANTED; 1446 UVM_UNLOCK_AND_WAIT(ptmp, 1447 &uobj->vmobjlock, 0, "uvn_get",0); 1448 simple_lock(&uobj->vmobjlock); 1449 continue; /* goto top of pps while loop */ 1450 } 1451 1452 /* 1453 * if we get here then the page has become resident 1454 * and unbusy between steps 1 and 2. we busy it 1455 * now (so we own it) and set pps[lcv] (so that we 1456 * exit the while loop). 1457 */ 1458 ptmp->flags |= PG_BUSY; 1459 UVM_PAGE_OWN(ptmp, "uvn_get2"); 1460 pps[lcv] = ptmp; 1461 } 1462 1463 /* 1464 * if we own the a valid page at the correct offset, pps[lcv] 1465 * will point to it. nothing more to do except go to the 1466 * next page. 1467 */ 1468 1469 if (pps[lcv]) 1470 continue; /* next lcv */ 1471 1472 /* 1473 * we have a "fake/busy/clean" page that we just allocated. do 1474 * I/O to fill it with valid data. note that object must be 1475 * locked going into uvn_io, but will be unlocked afterwards. 1476 */ 1477 1478 result = uvn_io((struct uvm_vnode *) uobj, &ptmp, 1, 1479 PGO_SYNCIO, UIO_READ); 1480 1481 /* 1482 * I/O done. object is unlocked (by uvn_io). because we used 1483 * syncio the result can not be PEND or AGAIN. we must relock 1484 * and check for errors. 1485 */ 1486 1487 /* lock object. check for errors. */ 1488 simple_lock(&uobj->vmobjlock); 1489 if (result != VM_PAGER_OK) { 1490 if (ptmp->flags & PG_WANTED) 1491 /* object lock still held */ 1492 thread_wakeup(ptmp); 1493 1494 ptmp->flags &= ~(PG_WANTED|PG_BUSY); 1495 UVM_PAGE_OWN(ptmp, NULL); 1496 uvm_lock_pageq(); 1497 uvm_pagefree(ptmp); 1498 uvm_unlock_pageq(); 1499 simple_unlock(&uobj->vmobjlock); 1500 return(result); 1501 } 1502 1503 /* 1504 * we got the page! clear the fake flag (indicates valid 1505 * data now in page) and plug into our result array. note 1506 * that page is still busy. 1507 * 1508 * it is the callers job to: 1509 * => check if the page is released 1510 * => unbusy the page 1511 * => activate the page 1512 */ 1513 1514 ptmp->flags &= ~PG_FAKE; /* data is valid ... */ 1515 pmap_clear_modify(PMAP_PGARG(ptmp)); /* ... and clean */ 1516 pps[lcv] = ptmp; 1517 1518 } /* lcv loop */ 1519 1520 /* 1521 * finally, unlock object and return. 1522 */ 1523 1524 simple_unlock(&uobj->vmobjlock); 1525 return (VM_PAGER_OK); 1526 } 1527 1528 /* 1529 * uvn_asyncget: start async I/O to bring pages into ram 1530 * 1531 * => caller must lock object(???XXX: see if this is best) 1532 * => could be called from uvn_get or a madvise() fault-ahead. 1533 * => if it fails, it doesn't matter. 1534 */ 1535 1536 static int 1537 uvn_asyncget(uobj, offset, npages) 1538 struct uvm_object *uobj; 1539 vm_offset_t offset; 1540 int npages; 1541 { 1542 1543 /* 1544 * XXXCDC: we can't do async I/O yet 1545 */ 1546 printf("uvn_asyncget called\n"); 1547 return (KERN_SUCCESS); 1548 } 1549 1550 /* 1551 * uvn_io: do I/O to a vnode 1552 * 1553 * => prefer map unlocked (not required) 1554 * => object must be locked! we will _unlock_ it before starting I/O. 1555 * => flags: PGO_SYNCIO -- use sync. I/O 1556 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync. 1557 * [thus we never do async i/o! see iodone comment] 1558 */ 1559 1560 static int 1561 uvn_io(uvn, pps, npages, flags, rw) 1562 struct uvm_vnode *uvn; 1563 vm_page_t *pps; 1564 int npages, flags, rw; 1565 { 1566 struct vnode *vn; 1567 struct uio uio; 1568 struct iovec iov; 1569 vm_offset_t kva, file_offset; 1570 int waitf, result, got, wanted; 1571 UVMHIST_FUNC("uvn_io"); UVMHIST_CALLED(maphist); 1572 1573 UVMHIST_LOG(maphist, "rw=%d", rw,0,0,0); 1574 1575 /* 1576 * init values 1577 */ 1578 1579 waitf = (flags & PGO_SYNCIO) ? M_WAITOK : M_NOWAIT; 1580 vn = (struct vnode *) uvn; 1581 file_offset = pps[0]->offset; 1582 1583 /* 1584 * check for sync'ing I/O. 1585 */ 1586 1587 while (uvn->u_flags & UVM_VNODE_IOSYNC) { 1588 if (waitf == M_NOWAIT) { 1589 simple_unlock(&uvn->u_obj.vmobjlock); 1590 UVMHIST_LOG(maphist,"<- try again (iosync)",0,0,0,0); 1591 return(VM_PAGER_AGAIN); 1592 } 1593 uvn->u_flags |= UVM_VNODE_IOSYNCWANTED; 1594 UVM_UNLOCK_AND_WAIT(&uvn->u_flags, &uvn->u_obj.vmobjlock, 1595 FALSE, "uvn_iosync",0); 1596 simple_lock(&uvn->u_obj.vmobjlock); 1597 } 1598 1599 /* 1600 * check size 1601 */ 1602 1603 if (file_offset >= uvn->u_size) { 1604 simple_unlock(&uvn->u_obj.vmobjlock); 1605 UVMHIST_LOG(maphist,"<- BAD (size check)",0,0,0,0); 1606 #ifdef DIAGNOSTIC 1607 printf("uvn_io: note: size check fired\n"); 1608 #endif 1609 return(VM_PAGER_BAD); 1610 } 1611 1612 /* 1613 * first try and map the pages in (without waiting) 1614 */ 1615 1616 kva = uvm_pagermapin(pps, npages, NULL, M_NOWAIT); 1617 if (kva == NULL && waitf == M_NOWAIT) { 1618 simple_unlock(&uvn->u_obj.vmobjlock); 1619 UVMHIST_LOG(maphist,"<- mapin failed (try again)",0,0,0,0); 1620 return(VM_PAGER_AGAIN); 1621 } 1622 1623 /* 1624 * ok, now bump u_nio up. at this point we are done with uvn 1625 * and can unlock it. if we still don't have a kva, try again 1626 * (this time with sleep ok). 1627 */ 1628 1629 uvn->u_nio++; /* we have an I/O in progress! */ 1630 simple_unlock(&uvn->u_obj.vmobjlock); 1631 /* NOTE: object now unlocked */ 1632 if (kva == NULL) { 1633 kva = uvm_pagermapin(pps, npages, NULL, M_WAITOK); 1634 } 1635 1636 /* 1637 * ok, mapped in. our pages are PG_BUSY so they are not going to 1638 * get touched (so we can look at "offset" without having to lock 1639 * the object). set up for I/O. 1640 */ 1641 1642 /* 1643 * fill out uio/iov 1644 */ 1645 1646 iov.iov_base = (caddr_t) kva; 1647 wanted = npages * PAGE_SIZE; 1648 if (file_offset + wanted > uvn->u_size) 1649 wanted = uvn->u_size - file_offset; /* XXX: needed? */ 1650 iov.iov_len = wanted; 1651 uio.uio_iov = &iov; 1652 uio.uio_iovcnt = 1; 1653 uio.uio_offset = file_offset; 1654 uio.uio_segflg = UIO_SYSSPACE; 1655 uio.uio_rw = rw; 1656 uio.uio_resid = wanted; 1657 uio.uio_procp = NULL; 1658 1659 /* 1660 * do the I/O! (XXX: curproc?) 1661 */ 1662 1663 UVMHIST_LOG(maphist, "calling VOP",0,0,0,0); 1664 1665 if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0) 1666 vn_lock(vn, LK_EXCLUSIVE | LK_RETRY); 1667 /* NOTE: vnode now locked! */ 1668 1669 if (rw == UIO_READ) 1670 result = VOP_READ(vn, &uio, 0, curproc->p_ucred); 1671 else 1672 result = VOP_WRITE(vn, &uio, 0, curproc->p_ucred); 1673 1674 if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0) 1675 VOP_UNLOCK(vn, 0); 1676 /* NOTE: vnode now unlocked (unless vnislocked) */ 1677 1678 UVMHIST_LOG(maphist, "done calling VOP",0,0,0,0); 1679 1680 /* 1681 * result == unix style errno (0 == OK!) 1682 * 1683 * zero out rest of buffer (if needed) 1684 */ 1685 1686 if (result == 0) { 1687 got = wanted - uio.uio_resid; 1688 1689 if (wanted && got == 0) { 1690 result = EIO; /* XXX: error? */ 1691 } else if (got < PAGE_SIZE * npages && rw == UIO_READ) { 1692 bzero((void *) (kva + got), (PAGE_SIZE * npages) - got); 1693 } 1694 } 1695 1696 /* 1697 * now remove pager mapping 1698 */ 1699 uvm_pagermapout(kva, npages); 1700 1701 /* 1702 * now clean up the object (i.e. drop I/O count) 1703 */ 1704 1705 simple_lock(&uvn->u_obj.vmobjlock); 1706 /* NOTE: object now locked! */ 1707 1708 uvn->u_nio--; /* I/O DONE! */ 1709 if ((uvn->u_flags & UVM_VNODE_IOSYNC) != 0 && uvn->u_nio == 0) { 1710 wakeup(&uvn->u_nio); 1711 } 1712 simple_unlock(&uvn->u_obj.vmobjlock); 1713 /* NOTE: object now unlocked! */ 1714 1715 /* 1716 * done! 1717 */ 1718 1719 UVMHIST_LOG(maphist, "<- done (result %d)", result,0,0,0); 1720 if (result == 0) 1721 return(VM_PAGER_OK); 1722 else 1723 return(VM_PAGER_ERROR); 1724 } 1725 1726 /* 1727 * uvm_vnp_uncache: disable "persisting" in a vnode... when last reference 1728 * is gone we will kill the object (flushing dirty pages back to the vnode 1729 * if needed). 1730 * 1731 * => returns TRUE if there was no uvm_object attached or if there was 1732 * one and we killed it [i.e. if there is no active uvn] 1733 * => called with the vnode VOP_LOCK'd [we will unlock it for I/O, if 1734 * needed] 1735 * 1736 * => XXX: given that we now kill uvn's when a vnode is recycled (without 1737 * having to hold a reference on the vnode) and given a working 1738 * uvm_vnp_sync(), how does that effect the need for this function? 1739 * [XXXCDC: seems like it can die?] 1740 * 1741 * => XXX: this function should DIE once we merge the VM and buffer 1742 * cache. 1743 * 1744 * research shows that this is called in the following places: 1745 * ext2fs_truncate, ffs_truncate, detrunc[msdosfs]: called when vnode 1746 * changes sizes 1747 * ext2fs_write, WRITE [ufs_readwrite], msdosfs_write: called when we 1748 * are written to 1749 * ex2fs_chmod, ufs_chmod: called if VTEXT vnode and the sticky bit 1750 * is off 1751 * ffs_realloccg: when we can't extend the current block and have 1752 * to allocate a new one we call this [XXX: why?] 1753 * nfsrv_rename, rename_files: called when the target filename is there 1754 * and we want to remove it 1755 * nfsrv_remove, sys_unlink: called on file we are removing 1756 * nfsrv_access: if VTEXT and we want WRITE access and we don't uncache 1757 * then return "text busy" 1758 * nfs_open: seems to uncache any file opened with nfs 1759 * vn_writechk: if VTEXT vnode and can't uncache return "text busy" 1760 */ 1761 1762 boolean_t 1763 uvm_vnp_uncache(vp) 1764 struct vnode *vp; 1765 { 1766 struct uvm_vnode *uvn = &vp->v_uvm; 1767 1768 /* 1769 * lock uvn part of the vnode and check to see if we need to do anything 1770 */ 1771 1772 simple_lock(&uvn->u_obj.vmobjlock); 1773 if ((uvn->u_flags & UVM_VNODE_VALID) == 0 || 1774 (uvn->u_flags & UVM_VNODE_BLOCKED) != 0) { 1775 simple_unlock(&uvn->u_obj.vmobjlock); 1776 return(TRUE); 1777 } 1778 1779 /* 1780 * we have a valid, non-blocked uvn. clear persist flag. 1781 * if uvn is currently active we can return now. 1782 */ 1783 1784 uvn->u_flags &= ~UVM_VNODE_CANPERSIST; 1785 if (uvn->u_obj.uo_refs) { 1786 simple_unlock(&uvn->u_obj.vmobjlock); 1787 return(FALSE); 1788 } 1789 1790 /* 1791 * uvn is currently persisting! we have to gain a reference to 1792 * it so that we can call uvn_detach to kill the uvn. 1793 */ 1794 1795 VREF(vp); /* seems ok, even with VOP_LOCK */ 1796 uvn->u_obj.uo_refs++; /* value is now 1 */ 1797 simple_unlock(&uvn->u_obj.vmobjlock); 1798 1799 1800 #ifdef DEBUG 1801 /* 1802 * carry over sanity check from old vnode pager: the vnode should 1803 * be VOP_LOCK'd, and we confirm it here. 1804 */ 1805 if (!VOP_ISLOCKED(vp)) { 1806 boolean_t is_ok_anyway = FALSE; 1807 #ifdef NFS 1808 extern int (**nfsv2_vnodeop_p) __P((void *)); 1809 extern int (**spec_nfsv2nodeop_p) __P((void *)); 1810 #ifdef FIFO 1811 extern int (**fifo_nfsv2nodeop_p) __P((void *)); 1812 #endif /* FIFO */ 1813 1814 /* vnode is NOT VOP_LOCKed: some vnode types _never_ lock */ 1815 if (vp->v_op == nfsv2_vnodeop_p || 1816 vp->v_op == spec_nfsv2nodeop_p) { 1817 is_ok_anyway = TRUE; 1818 } 1819 #ifdef FIFO 1820 if (vp->v_op == fifo_nfsv2nodeop_p) { 1821 is_ok_anyway = TRUE; 1822 } 1823 #endif /* FIFO */ 1824 #endif /* NFS */ 1825 if (!is_ok_anyway) 1826 panic("uvm_vnp_uncache: vnode not locked!"); 1827 } 1828 #endif /* DEBUG */ 1829 1830 /* 1831 * now drop our reference to the vnode. if we have the sole 1832 * reference to the vnode then this will cause it to die [as we 1833 * just cleared the persist flag]. we have to unlock the vnode 1834 * while we are doing this as it may trigger I/O. 1835 * 1836 * XXX: it might be possible for uvn to get reclaimed while we are 1837 * unlocked causing us to return TRUE when we should not. we ignore 1838 * this as a false-positive return value doesn't hurt us. 1839 */ 1840 VOP_UNLOCK(vp, 0); 1841 uvn_detach(&uvn->u_obj); 1842 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 1843 1844 /* 1845 * and return... 1846 */ 1847 1848 return(TRUE); 1849 } 1850 1851 /* 1852 * uvm_vnp_setsize: grow or shrink a vnode uvn 1853 * 1854 * grow => just update size value 1855 * shrink => toss un-needed pages 1856 * 1857 * => we assume that the caller has a reference of some sort to the 1858 * vnode in question so that it will not be yanked out from under 1859 * us. 1860 * 1861 * called from: 1862 * => truncate fns (ext2fs_truncate, ffs_truncate, detrunc[msdos]) 1863 * => "write" fns (ext2fs_write, WRITE [ufs/ufs], msdosfs_write, nfs_write) 1864 * => ffs_balloc [XXX: why? doesn't WRITE handle?] 1865 * => NFS: nfs_loadattrcache, nfs_getattrcache, nfs_setattr 1866 * => union fs: union_newsize 1867 */ 1868 1869 void 1870 uvm_vnp_setsize(vp, newsize) 1871 struct vnode *vp; 1872 u_quad_t newsize; 1873 { 1874 struct uvm_vnode *uvn = &vp->v_uvm; 1875 1876 /* 1877 * lock uvn and check for valid object, and if valid: do it! 1878 */ 1879 simple_lock(&uvn->u_obj.vmobjlock); 1880 if (uvn->u_flags & UVM_VNODE_VALID) { 1881 1882 /* 1883 * make sure that the newsize fits within a vm_offset_t 1884 * XXX: need to revise addressing data types 1885 */ 1886 1887 if (newsize > (vm_offset_t) -PAGE_SIZE) { 1888 #ifdef DEBUG 1889 printf("uvm_vnp_setsize: vn %p size truncated " 1890 "%qx->%lx\n", vp, newsize, (vm_offset_t)-PAGE_SIZE); 1891 #endif 1892 newsize = (vm_offset_t)-PAGE_SIZE; 1893 } 1894 1895 /* 1896 * now check if the size has changed: if we shrink we had better 1897 * toss some pages... 1898 */ 1899 1900 if (uvn->u_size > newsize) { 1901 (void)uvn_flush(&uvn->u_obj, (vm_offset_t) newsize, 1902 uvn->u_size, PGO_FREE); 1903 } 1904 uvn->u_size = (vm_offset_t)newsize; 1905 } 1906 simple_unlock(&uvn->u_obj.vmobjlock); 1907 1908 /* 1909 * done 1910 */ 1911 return; 1912 } 1913 1914 /* 1915 * uvm_vnp_sync: flush all dirty VM pages back to their backing vnodes. 1916 * 1917 * => called from sys_sync with no VM structures locked 1918 * => only one process can do a sync at a time (because the uvn 1919 * structure only has one queue for sync'ing). we ensure this 1920 * by holding the uvn_sync_lock while the sync is in progress. 1921 * other processes attempting a sync will sleep on this lock 1922 * until we are done. 1923 */ 1924 1925 void 1926 uvm_vnp_sync(mp) 1927 struct mount *mp; 1928 { 1929 struct uvm_vnode *uvn; 1930 struct vnode *vp; 1931 boolean_t got_lock; 1932 1933 /* 1934 * step 1: ensure we are only ones using the uvn_sync_q by locking 1935 * our lock... 1936 */ 1937 lockmgr(&uvn_sync_lock, LK_EXCLUSIVE, (void *)0); 1938 1939 /* 1940 * step 2: build up a simpleq of uvns of interest based on the 1941 * write list. we gain a reference to uvns of interest. must 1942 * be careful about locking uvn's since we will be holding uvn_wl_lock 1943 * in the body of the loop. 1944 */ 1945 SIMPLEQ_INIT(&uvn_sync_q); 1946 simple_lock(&uvn_wl_lock); 1947 for (uvn = uvn_wlist.lh_first ; uvn != NULL ; 1948 uvn = uvn->u_wlist.le_next) { 1949 1950 vp = (struct vnode *) uvn; 1951 if (mp && vp->v_mount != mp) 1952 continue; 1953 1954 /* attempt to gain reference */ 1955 while ((got_lock = simple_lock_try(&uvn->u_obj.vmobjlock)) == 1956 FALSE && 1957 (uvn->u_flags & UVM_VNODE_BLOCKED) == 0) 1958 /* spin */ ; 1959 1960 /* 1961 * we will exit the loop if either if the following are true: 1962 * - we got the lock [always true if NCPU == 1] 1963 * - we failed to get the lock but noticed the vnode was 1964 * "blocked" -- in this case the vnode must be a dying 1965 * vnode, and since dying vnodes are in the process of 1966 * being flushed out, we can safely skip this one 1967 * 1968 * we want to skip over the vnode if we did not get the lock, 1969 * or if the vnode is already dying (due to the above logic). 1970 * 1971 * note that uvn must already be valid because we found it on 1972 * the wlist (this also means it can't be ALOCK'd). 1973 */ 1974 if (!got_lock || (uvn->u_flags & UVM_VNODE_BLOCKED) != 0) { 1975 if (got_lock) 1976 simple_unlock(&uvn->u_obj.vmobjlock); 1977 continue; /* skip it */ 1978 } 1979 1980 /* 1981 * gain reference. watch out for persisting uvns (need to 1982 * regain vnode REF). 1983 */ 1984 if (uvn->u_obj.uo_refs == 0) 1985 VREF(vp); 1986 uvn->u_obj.uo_refs++; 1987 simple_unlock(&uvn->u_obj.vmobjlock); 1988 1989 /* 1990 * got it! 1991 */ 1992 SIMPLEQ_INSERT_HEAD(&uvn_sync_q, uvn, u_syncq); 1993 } 1994 simple_unlock(&uvn_wl_lock); 1995 1996 /* 1997 * step 3: we now have a list of uvn's that may need cleaning. 1998 * we are holding the uvn_sync_lock, but have dropped the uvn_wl_lock 1999 * (so we can now safely lock uvn's again). 2000 */ 2001 2002 for (uvn = uvn_sync_q.sqh_first ; uvn ; uvn = uvn->u_syncq.sqe_next) { 2003 simple_lock(&uvn->u_obj.vmobjlock); 2004 #ifdef DIAGNOSTIC 2005 if (uvn->u_flags & UVM_VNODE_DYING) { 2006 printf("uvm_vnp_sync: dying vnode on sync list\n"); 2007 } 2008 #endif 2009 uvn_flush(&uvn->u_obj, 0, 0, 2010 PGO_CLEANIT|PGO_ALLPAGES|PGO_DOACTCLUST); 2011 2012 /* 2013 * if we have the only reference and we just cleaned the uvn, 2014 * then we can pull it out of the UVM_VNODE_WRITEABLE state 2015 * thus allowing us to avoid thinking about flushing it again 2016 * on later sync ops. 2017 */ 2018 if (uvn->u_obj.uo_refs == 1 && 2019 (uvn->u_flags & UVM_VNODE_WRITEABLE)) { 2020 LIST_REMOVE(uvn, u_wlist); 2021 uvn->u_flags &= ~UVM_VNODE_WRITEABLE; 2022 } 2023 2024 simple_unlock(&uvn->u_obj.vmobjlock); 2025 2026 /* now drop our reference to the uvn */ 2027 uvn_detach(&uvn->u_obj); 2028 } 2029 2030 /* 2031 * done! release sync lock 2032 */ 2033 lockmgr(&uvn_sync_lock, LK_RELEASE, (void *)0); 2034 } 2035