1 /* $NetBSD: uvm_vnode.c,v 1.33 2000/05/19 03:45:05 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 1997 Charles D. Cranor and Washington University. 5 * Copyright (c) 1991, 1993 6 * The Regents of the University of California. 7 * Copyright (c) 1990 University of Utah. 8 * 9 * All rights reserved. 10 * 11 * This code is derived from software contributed to Berkeley by 12 * the Systems Programming Group of the University of Utah Computer 13 * Science Department. 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 3. All advertising materials mentioning features or use of this software 24 * must display the following acknowledgement: 25 * This product includes software developed by Charles D. Cranor, 26 * Washington University, the University of California, Berkeley and 27 * its contributors. 28 * 4. Neither the name of the University nor the names of its contributors 29 * may be used to endorse or promote products derived from this software 30 * without specific prior written permission. 31 * 32 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 33 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 34 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 35 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 36 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 37 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 38 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 39 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 40 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 41 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 42 * SUCH DAMAGE. 43 * 44 * @(#)vnode_pager.c 8.8 (Berkeley) 2/13/94 45 * from: Id: uvm_vnode.c,v 1.1.2.26 1998/02/02 20:38:07 chuck Exp 46 */ 47 48 #include "fs_nfs.h" 49 #include "opt_uvmhist.h" 50 51 /* 52 * uvm_vnode.c: the vnode pager. 53 */ 54 55 #include <sys/param.h> 56 #include <sys/systm.h> 57 #include <sys/proc.h> 58 #include <sys/malloc.h> 59 #include <sys/vnode.h> 60 #include <sys/disklabel.h> 61 #include <sys/ioctl.h> 62 #include <sys/fcntl.h> 63 #include <sys/conf.h> 64 65 #include <miscfs/specfs/specdev.h> 66 67 #include <vm/vm.h> 68 #include <vm/vm_page.h> 69 #include <vm/vm_kern.h> 70 71 #include <uvm/uvm.h> 72 #include <uvm/uvm_vnode.h> 73 74 /* 75 * private global data structure 76 * 77 * we keep a list of writeable active vnode-backed VM objects for sync op. 78 * we keep a simpleq of vnodes that are currently being sync'd. 79 */ 80 81 LIST_HEAD(uvn_list_struct, uvm_vnode); 82 static struct uvn_list_struct uvn_wlist; /* writeable uvns */ 83 static simple_lock_data_t uvn_wl_lock; /* locks uvn_wlist */ 84 85 SIMPLEQ_HEAD(uvn_sq_struct, uvm_vnode); 86 static struct uvn_sq_struct uvn_sync_q; /* sync'ing uvns */ 87 lock_data_t uvn_sync_lock; /* locks sync operation */ 88 89 /* 90 * functions 91 */ 92 93 static int uvn_asyncget __P((struct uvm_object *, voff_t, 94 int)); 95 static void uvn_cluster __P((struct uvm_object *, voff_t, 96 voff_t *, voff_t *)); 97 static void uvn_detach __P((struct uvm_object *)); 98 static boolean_t uvn_flush __P((struct uvm_object *, voff_t, 99 voff_t, int)); 100 static int uvn_get __P((struct uvm_object *, voff_t, 101 vm_page_t *, int *, int, 102 vm_prot_t, int, int)); 103 static void uvn_init __P((void)); 104 static int uvn_io __P((struct uvm_vnode *, vm_page_t *, 105 int, int, int)); 106 static int uvn_put __P((struct uvm_object *, vm_page_t *, 107 int, boolean_t)); 108 static void uvn_reference __P((struct uvm_object *)); 109 static boolean_t uvn_releasepg __P((struct vm_page *, 110 struct vm_page **)); 111 112 /* 113 * master pager structure 114 */ 115 116 struct uvm_pagerops uvm_vnodeops = { 117 uvn_init, 118 uvn_reference, 119 uvn_detach, 120 NULL, /* no specialized fault routine required */ 121 uvn_flush, 122 uvn_get, 123 uvn_asyncget, 124 uvn_put, 125 uvn_cluster, 126 uvm_mk_pcluster, /* use generic version of this: see uvm_pager.c */ 127 NULL, /* AIO-DONE function (not until we have asyncio) */ 128 uvn_releasepg, 129 }; 130 131 /* 132 * the ops! 133 */ 134 135 /* 136 * uvn_init 137 * 138 * init pager private data structures. 139 */ 140 141 static void 142 uvn_init() 143 { 144 145 LIST_INIT(&uvn_wlist); 146 simple_lock_init(&uvn_wl_lock); 147 /* note: uvn_sync_q init'd in uvm_vnp_sync() */ 148 lockinit(&uvn_sync_lock, PVM, "uvnsync", 0, 0); 149 } 150 151 /* 152 * uvn_attach 153 * 154 * attach a vnode structure to a VM object. if the vnode is already 155 * attached, then just bump the reference count by one and return the 156 * VM object. if not already attached, attach and return the new VM obj. 157 * the "accessprot" tells the max access the attaching thread wants to 158 * our pages. 159 * 160 * => caller must _not_ already be holding the lock on the uvm_object. 161 * => in fact, nothing should be locked so that we can sleep here. 162 * => note that uvm_object is first thing in vnode structure, so their 163 * pointers are equiv. 164 */ 165 166 struct uvm_object * 167 uvn_attach(arg, accessprot) 168 void *arg; 169 vm_prot_t accessprot; 170 { 171 struct vnode *vp = arg; 172 struct uvm_vnode *uvn = &vp->v_uvm; 173 struct vattr vattr; 174 int oldflags, result; 175 struct partinfo pi; 176 u_quad_t used_vnode_size; 177 UVMHIST_FUNC("uvn_attach"); UVMHIST_CALLED(maphist); 178 179 UVMHIST_LOG(maphist, "(vn=0x%x)", arg,0,0,0); 180 181 used_vnode_size = (u_quad_t)0; /* XXX gcc -Wuninitialized */ 182 183 /* 184 * first get a lock on the uvn. 185 */ 186 simple_lock(&uvn->u_obj.vmobjlock); 187 while (uvn->u_flags & UVM_VNODE_BLOCKED) { 188 printf("uvn_attach: blocked at 0x%p flags 0x%x\n", 189 uvn, uvn->u_flags); 190 uvn->u_flags |= UVM_VNODE_WANTED; 191 UVMHIST_LOG(maphist, " SLEEPING on blocked vn",0,0,0,0); 192 UVM_UNLOCK_AND_WAIT(uvn, &uvn->u_obj.vmobjlock, FALSE, 193 "uvn_attach", 0); 194 simple_lock(&uvn->u_obj.vmobjlock); 195 UVMHIST_LOG(maphist," WOKE UP",0,0,0,0); 196 } 197 198 /* 199 * if we're mapping a BLK device, make sure it is a disk. 200 */ 201 if (vp->v_type == VBLK && bdevsw[major(vp->v_rdev)].d_type != D_DISK) { 202 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock */ 203 UVMHIST_LOG(maphist,"<- done (VBLK not D_DISK!)", 0,0,0,0); 204 return(NULL); 205 } 206 207 /* 208 * now we have lock and uvn must not be in a blocked state. 209 * first check to see if it is already active, in which case 210 * we can bump the reference count, check to see if we need to 211 * add it to the writeable list, and then return. 212 */ 213 if (uvn->u_flags & UVM_VNODE_VALID) { /* already active? */ 214 215 /* regain VREF if we were persisting */ 216 if (uvn->u_obj.uo_refs == 0) { 217 VREF(vp); 218 UVMHIST_LOG(maphist," VREF (reclaim persisting vnode)", 219 0,0,0,0); 220 } 221 uvn->u_obj.uo_refs++; /* bump uvn ref! */ 222 223 /* check for new writeable uvn */ 224 if ((accessprot & VM_PROT_WRITE) != 0 && 225 (uvn->u_flags & UVM_VNODE_WRITEABLE) == 0) { 226 simple_lock(&uvn_wl_lock); 227 LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist); 228 simple_unlock(&uvn_wl_lock); 229 /* we are now on wlist! */ 230 uvn->u_flags |= UVM_VNODE_WRITEABLE; 231 } 232 233 /* unlock and return */ 234 simple_unlock(&uvn->u_obj.vmobjlock); 235 UVMHIST_LOG(maphist,"<- done, refcnt=%d", uvn->u_obj.uo_refs, 236 0, 0, 0); 237 return (&uvn->u_obj); 238 } 239 240 /* 241 * need to call VOP_GETATTR() to get the attributes, but that could 242 * block (due to I/O), so we want to unlock the object before calling. 243 * however, we want to keep anyone else from playing with the object 244 * while it is unlocked. to do this we set UVM_VNODE_ALOCK which 245 * prevents anyone from attaching to the vnode until we are done with 246 * it. 247 */ 248 uvn->u_flags = UVM_VNODE_ALOCK; 249 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock in case we sleep */ 250 /* XXX: curproc? */ 251 252 if (vp->v_type == VBLK) { 253 /* 254 * We could implement this as a specfs getattr call, but: 255 * 256 * (1) VOP_GETATTR() would get the file system 257 * vnode operation, not the specfs operation. 258 * 259 * (2) All we want is the size, anyhow. 260 */ 261 result = (*bdevsw[major(vp->v_rdev)].d_ioctl)(vp->v_rdev, 262 DIOCGPART, (caddr_t)&pi, FREAD, curproc); 263 if (result == 0) { 264 /* XXX should remember blocksize */ 265 used_vnode_size = (u_quad_t)pi.disklab->d_secsize * 266 (u_quad_t)pi.part->p_size; 267 } 268 } else { 269 result = VOP_GETATTR(vp, &vattr, curproc->p_ucred, curproc); 270 if (result == 0) 271 used_vnode_size = vattr.va_size; 272 } 273 274 /* relock object */ 275 simple_lock(&uvn->u_obj.vmobjlock); 276 277 if (result != 0) { 278 if (uvn->u_flags & UVM_VNODE_WANTED) 279 wakeup(uvn); 280 uvn->u_flags = 0; 281 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock */ 282 UVMHIST_LOG(maphist,"<- done (VOP_GETATTR FAILED!)", 0,0,0,0); 283 return(NULL); 284 } 285 286 /* 287 * make sure that the newsize fits within a vaddr_t 288 * XXX: need to revise addressing data types 289 */ 290 #ifdef DEBUG 291 if (vp->v_type == VBLK) 292 printf("used_vnode_size = %qu\n", (long long)used_vnode_size); 293 #endif 294 295 /* 296 * now set up the uvn. 297 */ 298 uvn->u_obj.pgops = &uvm_vnodeops; 299 TAILQ_INIT(&uvn->u_obj.memq); 300 uvn->u_obj.uo_npages = 0; 301 uvn->u_obj.uo_refs = 1; /* just us... */ 302 oldflags = uvn->u_flags; 303 uvn->u_flags = UVM_VNODE_VALID|UVM_VNODE_CANPERSIST; 304 uvn->u_nio = 0; 305 uvn->u_size = used_vnode_size; 306 307 /* if write access, we need to add it to the wlist */ 308 if (accessprot & VM_PROT_WRITE) { 309 simple_lock(&uvn_wl_lock); 310 LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist); 311 simple_unlock(&uvn_wl_lock); 312 uvn->u_flags |= UVM_VNODE_WRITEABLE; /* we are on wlist! */ 313 } 314 315 /* 316 * add a reference to the vnode. this reference will stay as long 317 * as there is a valid mapping of the vnode. dropped when the 318 * reference count goes to zero [and we either free or persist]. 319 */ 320 VREF(vp); 321 simple_unlock(&uvn->u_obj.vmobjlock); 322 if (oldflags & UVM_VNODE_WANTED) 323 wakeup(uvn); 324 325 UVMHIST_LOG(maphist,"<- done/VREF, ret 0x%x", &uvn->u_obj,0,0,0); 326 return(&uvn->u_obj); 327 } 328 329 330 /* 331 * uvn_reference 332 * 333 * duplicate a reference to a VM object. Note that the reference 334 * count must already be at least one (the passed in reference) so 335 * there is no chance of the uvn being killed or locked out here. 336 * 337 * => caller must call with object unlocked. 338 * => caller must be using the same accessprot as was used at attach time 339 */ 340 341 342 static void 343 uvn_reference(uobj) 344 struct uvm_object *uobj; 345 { 346 #ifdef DEBUG 347 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 348 #endif 349 UVMHIST_FUNC("uvn_reference"); UVMHIST_CALLED(maphist); 350 351 simple_lock(&uobj->vmobjlock); 352 #ifdef DEBUG 353 if ((uvn->u_flags & UVM_VNODE_VALID) == 0) { 354 printf("uvn_reference: ref=%d, flags=0x%x\n", uvn->u_flags, 355 uobj->uo_refs); 356 panic("uvn_reference: invalid state"); 357 } 358 #endif 359 uobj->uo_refs++; 360 UVMHIST_LOG(maphist, "<- done (uobj=0x%x, ref = %d)", 361 uobj, uobj->uo_refs,0,0); 362 simple_unlock(&uobj->vmobjlock); 363 } 364 365 /* 366 * uvn_detach 367 * 368 * remove a reference to a VM object. 369 * 370 * => caller must call with object unlocked and map locked. 371 * => this starts the detach process, but doesn't have to finish it 372 * (async i/o could still be pending). 373 */ 374 static void 375 uvn_detach(uobj) 376 struct uvm_object *uobj; 377 { 378 struct uvm_vnode *uvn; 379 struct vnode *vp; 380 int oldflags; 381 UVMHIST_FUNC("uvn_detach"); UVMHIST_CALLED(maphist); 382 383 simple_lock(&uobj->vmobjlock); 384 385 UVMHIST_LOG(maphist," (uobj=0x%x) ref=%d", uobj,uobj->uo_refs,0,0); 386 uobj->uo_refs--; /* drop ref! */ 387 if (uobj->uo_refs) { /* still more refs */ 388 simple_unlock(&uobj->vmobjlock); 389 UVMHIST_LOG(maphist, "<- done (rc>0)", 0,0,0,0); 390 return; 391 } 392 393 /* 394 * get other pointers ... 395 */ 396 397 uvn = (struct uvm_vnode *) uobj; 398 vp = (struct vnode *) uobj; 399 400 /* 401 * clear VTEXT flag now that there are no mappings left (VTEXT is used 402 * to keep an active text file from being overwritten). 403 */ 404 vp->v_flag &= ~VTEXT; 405 406 /* 407 * we just dropped the last reference to the uvn. see if we can 408 * let it "stick around". 409 */ 410 411 if (uvn->u_flags & UVM_VNODE_CANPERSIST) { 412 /* won't block */ 413 uvn_flush(uobj, 0, 0, PGO_DEACTIVATE|PGO_ALLPAGES); 414 simple_unlock(&uobj->vmobjlock); 415 vrele(vp); /* drop vnode reference */ 416 UVMHIST_LOG(maphist,"<- done/vrele! (persist)", 0,0,0,0); 417 return; 418 } 419 420 /* 421 * its a goner! 422 */ 423 424 UVMHIST_LOG(maphist," its a goner (flushing)!", 0,0,0,0); 425 426 uvn->u_flags |= UVM_VNODE_DYING; 427 428 /* 429 * even though we may unlock in flush, no one can gain a reference 430 * to us until we clear the "dying" flag [because it blocks 431 * attaches]. we will not do that until after we've disposed of all 432 * the pages with uvn_flush(). note that before the flush the only 433 * pages that could be marked PG_BUSY are ones that are in async 434 * pageout by the daemon. (there can't be any pending "get"'s 435 * because there are no references to the object). 436 */ 437 438 (void) uvn_flush(uobj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES); 439 440 UVMHIST_LOG(maphist," its a goner (done flush)!", 0,0,0,0); 441 442 /* 443 * given the structure of this pager, the above flush request will 444 * create the following state: all the pages that were in the object 445 * have either been free'd or they are marked PG_BUSY|PG_RELEASED. 446 * the PG_BUSY bit was set either by us or the daemon for async I/O. 447 * in either case, if we have pages left we can't kill the object 448 * yet because i/o is pending. in this case we set the "relkill" 449 * flag which will cause pgo_releasepg to kill the object once all 450 * the I/O's are done [pgo_releasepg will be called from the aiodone 451 * routine or from the page daemon]. 452 */ 453 454 if (uobj->uo_npages) { /* I/O pending. iodone will free */ 455 #ifdef DEBUG 456 /* 457 * XXXCDC: very unlikely to happen until we have async i/o 458 * so print a little info message in case it does. 459 */ 460 printf("uvn_detach: vn %p has pages left after flush - " 461 "relkill mode\n", uobj); 462 #endif 463 uvn->u_flags |= UVM_VNODE_RELKILL; 464 simple_unlock(&uobj->vmobjlock); 465 UVMHIST_LOG(maphist,"<- done! (releasepg will kill obj)", 0, 0, 466 0, 0); 467 return; 468 } 469 470 /* 471 * kill object now. note that we can't be on the sync q because 472 * all references are gone. 473 */ 474 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 475 simple_lock(&uvn_wl_lock); /* protect uvn_wlist */ 476 LIST_REMOVE(uvn, u_wlist); 477 simple_unlock(&uvn_wl_lock); 478 } 479 #ifdef DIAGNOSTIC 480 if (uobj->memq.tqh_first != NULL) 481 panic("uvn_deref: vnode VM object still has pages afer " 482 "syncio/free flush"); 483 #endif 484 oldflags = uvn->u_flags; 485 uvn->u_flags = 0; 486 simple_unlock(&uobj->vmobjlock); 487 488 /* wake up any sleepers */ 489 if (oldflags & UVM_VNODE_WANTED) 490 wakeup(uvn); 491 492 /* 493 * drop our reference to the vnode. 494 */ 495 vrele(vp); 496 UVMHIST_LOG(maphist,"<- done (vrele) final", 0,0,0,0); 497 498 return; 499 } 500 501 /* 502 * uvm_vnp_terminate: external hook to clear out a vnode's VM 503 * 504 * called in two cases: 505 * [1] when a persisting vnode vm object (i.e. one with a zero reference 506 * count) needs to be freed so that a vnode can be reused. this 507 * happens under "getnewvnode" in vfs_subr.c. if the vnode from 508 * the free list is still attached (i.e. not VBAD) then vgone is 509 * called. as part of the vgone trace this should get called to 510 * free the vm object. this is the common case. 511 * [2] when a filesystem is being unmounted by force (MNT_FORCE, 512 * "umount -f") the vgone() function is called on active vnodes 513 * on the mounted file systems to kill their data (the vnodes become 514 * "dead" ones [see src/sys/miscfs/deadfs/...]). that results in a 515 * call here (even if the uvn is still in use -- i.e. has a non-zero 516 * reference count). this case happens at "umount -f" and during a 517 * "reboot/halt" operation. 518 * 519 * => the caller must XLOCK and VOP_LOCK the vnode before calling us 520 * [protects us from getting a vnode that is already in the DYING 521 * state...] 522 * => unlike uvn_detach, this function must not return until all the 523 * uvn's pages are disposed of. 524 * => in case [2] the uvn is still alive after this call, but all I/O 525 * ops will fail (due to the backing vnode now being "dead"). this 526 * will prob. kill any process using the uvn due to pgo_get failing. 527 */ 528 529 void 530 uvm_vnp_terminate(vp) 531 struct vnode *vp; 532 { 533 struct uvm_vnode *uvn = &vp->v_uvm; 534 int oldflags; 535 UVMHIST_FUNC("uvm_vnp_terminate"); UVMHIST_CALLED(maphist); 536 537 /* 538 * lock object and check if it is valid 539 */ 540 simple_lock(&uvn->u_obj.vmobjlock); 541 UVMHIST_LOG(maphist, " vp=0x%x, ref=%d, flag=0x%x", vp, 542 uvn->u_obj.uo_refs, uvn->u_flags, 0); 543 if ((uvn->u_flags & UVM_VNODE_VALID) == 0) { 544 simple_unlock(&uvn->u_obj.vmobjlock); 545 UVMHIST_LOG(maphist, "<- done (not active)", 0, 0, 0, 0); 546 return; 547 } 548 549 /* 550 * must be a valid uvn that is not already dying (because XLOCK 551 * protects us from that). the uvn can't in the ALOCK state 552 * because it is valid, and uvn's that are in the ALOCK state haven't 553 * been marked valid yet. 554 */ 555 556 #ifdef DEBUG 557 /* 558 * debug check: are we yanking the vnode out from under our uvn? 559 */ 560 if (uvn->u_obj.uo_refs) { 561 printf("uvm_vnp_terminate(%p): terminating active vnode " 562 "(refs=%d)\n", uvn, uvn->u_obj.uo_refs); 563 } 564 #endif 565 566 /* 567 * it is possible that the uvn was detached and is in the relkill 568 * state [i.e. waiting for async i/o to finish so that releasepg can 569 * kill object]. we take over the vnode now and cancel the relkill. 570 * we want to know when the i/o is done so we can recycle right 571 * away. note that a uvn can only be in the RELKILL state if it 572 * has a zero reference count. 573 */ 574 575 if (uvn->u_flags & UVM_VNODE_RELKILL) 576 uvn->u_flags &= ~UVM_VNODE_RELKILL; /* cancel RELKILL */ 577 578 /* 579 * block the uvn by setting the dying flag, and then flush the 580 * pages. (note that flush may unlock object while doing I/O, but 581 * it will re-lock it before it returns control here). 582 * 583 * also, note that we tell I/O that we are already VOP_LOCK'd so 584 * that uvn_io doesn't attempt to VOP_LOCK again. 585 * 586 * XXXCDC: setting VNISLOCKED on an active uvn which is being terminated 587 * due to a forceful unmount might not be a good idea. maybe we 588 * need a way to pass in this info to uvn_flush through a 589 * pager-defined PGO_ constant [currently there are none]. 590 */ 591 uvn->u_flags |= UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED; 592 593 (void) uvn_flush(&uvn->u_obj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES); 594 595 /* 596 * as we just did a flush we expect all the pages to be gone or in 597 * the process of going. sleep to wait for the rest to go [via iosync]. 598 */ 599 600 while (uvn->u_obj.uo_npages) { 601 #ifdef DEBUG 602 struct vm_page *pp; 603 for (pp = uvn->u_obj.memq.tqh_first ; pp != NULL ; 604 pp = pp->listq.tqe_next) { 605 if ((pp->flags & PG_BUSY) == 0) 606 panic("uvm_vnp_terminate: detected unbusy pg"); 607 } 608 if (uvn->u_nio == 0) 609 panic("uvm_vnp_terminate: no I/O to wait for?"); 610 printf("uvm_vnp_terminate: waiting for I/O to fin.\n"); 611 /* 612 * XXXCDC: this is unlikely to happen without async i/o so we 613 * put a printf in just to keep an eye on it. 614 */ 615 #endif 616 uvn->u_flags |= UVM_VNODE_IOSYNC; 617 UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock, FALSE, 618 "uvn_term",0); 619 simple_lock(&uvn->u_obj.vmobjlock); 620 } 621 622 /* 623 * done. now we free the uvn if its reference count is zero 624 * (true if we are zapping a persisting uvn). however, if we are 625 * terminating a uvn with active mappings we let it live ... future 626 * calls down to the vnode layer will fail. 627 */ 628 629 oldflags = uvn->u_flags; 630 if (uvn->u_obj.uo_refs) { 631 632 /* 633 * uvn must live on it is dead-vnode state until all references 634 * are gone. restore flags. clear CANPERSIST state. 635 */ 636 637 uvn->u_flags &= ~(UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED| 638 UVM_VNODE_WANTED|UVM_VNODE_CANPERSIST); 639 640 } else { 641 642 /* 643 * free the uvn now. note that the VREF reference is already 644 * gone [it is dropped when we enter the persist state]. 645 */ 646 if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED) 647 panic("uvm_vnp_terminate: io sync wanted bit set"); 648 649 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 650 simple_lock(&uvn_wl_lock); 651 LIST_REMOVE(uvn, u_wlist); 652 simple_unlock(&uvn_wl_lock); 653 } 654 uvn->u_flags = 0; /* uvn is history, clear all bits */ 655 } 656 657 if (oldflags & UVM_VNODE_WANTED) 658 wakeup(uvn); /* object lock still held */ 659 660 simple_unlock(&uvn->u_obj.vmobjlock); 661 UVMHIST_LOG(maphist, "<- done", 0, 0, 0, 0); 662 663 } 664 665 /* 666 * uvn_releasepg: handled a released page in a uvn 667 * 668 * => "pg" is a PG_BUSY [caller owns it], PG_RELEASED page that we need 669 * to dispose of. 670 * => caller must handled PG_WANTED case 671 * => called with page's object locked, pageq's unlocked 672 * => returns TRUE if page's object is still alive, FALSE if we 673 * killed the page's object. if we return TRUE, then we 674 * return with the object locked. 675 * => if (nextpgp != NULL) => we return pageq.tqe_next here, and return 676 * with the page queues locked [for pagedaemon] 677 * => if (nextpgp == NULL) => we return with page queues unlocked [normal case] 678 * => we kill the uvn if it is not referenced and we are suppose to 679 * kill it ("relkill"). 680 */ 681 682 boolean_t 683 uvn_releasepg(pg, nextpgp) 684 struct vm_page *pg; 685 struct vm_page **nextpgp; /* OUT */ 686 { 687 struct uvm_vnode *uvn = (struct uvm_vnode *) pg->uobject; 688 #ifdef DIAGNOSTIC 689 if ((pg->flags & PG_RELEASED) == 0) 690 panic("uvn_releasepg: page not released!"); 691 #endif 692 693 /* 694 * dispose of the page [caller handles PG_WANTED] 695 */ 696 pmap_page_protect(pg, VM_PROT_NONE); 697 uvm_lock_pageq(); 698 if (nextpgp) 699 *nextpgp = pg->pageq.tqe_next; /* next page for daemon */ 700 uvm_pagefree(pg); 701 if (!nextpgp) 702 uvm_unlock_pageq(); 703 704 /* 705 * now see if we need to kill the object 706 */ 707 if (uvn->u_flags & UVM_VNODE_RELKILL) { 708 if (uvn->u_obj.uo_refs) 709 panic("uvn_releasepg: kill flag set on referenced " 710 "object!"); 711 if (uvn->u_obj.uo_npages == 0) { 712 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 713 simple_lock(&uvn_wl_lock); 714 LIST_REMOVE(uvn, u_wlist); 715 simple_unlock(&uvn_wl_lock); 716 } 717 #ifdef DIAGNOSTIC 718 if (uvn->u_obj.memq.tqh_first) 719 panic("uvn_releasepg: pages in object with npages == 0"); 720 #endif 721 if (uvn->u_flags & UVM_VNODE_WANTED) 722 /* still holding object lock */ 723 wakeup(uvn); 724 725 uvn->u_flags = 0; /* DEAD! */ 726 simple_unlock(&uvn->u_obj.vmobjlock); 727 return (FALSE); 728 } 729 } 730 return (TRUE); 731 } 732 733 /* 734 * NOTE: currently we have to use VOP_READ/VOP_WRITE because they go 735 * through the buffer cache and allow I/O in any size. These VOPs use 736 * synchronous i/o. [vs. VOP_STRATEGY which can be async, but doesn't 737 * go through the buffer cache or allow I/O sizes larger than a 738 * block]. we will eventually want to change this. 739 * 740 * issues to consider: 741 * uvm provides the uvm_aiodesc structure for async i/o management. 742 * there are two tailq's in the uvm. structure... one for pending async 743 * i/o and one for "done" async i/o. to do an async i/o one puts 744 * an aiodesc on the "pending" list (protected by splbio()), starts the 745 * i/o and returns VM_PAGER_PEND. when the i/o is done, we expect 746 * some sort of "i/o done" function to be called (at splbio(), interrupt 747 * time). this function should remove the aiodesc from the pending list 748 * and place it on the "done" list and wakeup the daemon. the daemon 749 * will run at normal spl() and will remove all items from the "done" 750 * list and call the "aiodone" hook for each done request (see uvm_pager.c). 751 * [in the old vm code, this was done by calling the "put" routine with 752 * null arguments which made the code harder to read and understand because 753 * you had one function ("put") doing two things.] 754 * 755 * so the current pager needs: 756 * int uvn_aiodone(struct uvm_aiodesc *) 757 * 758 * => return KERN_SUCCESS (aio finished, free it). otherwise requeue for 759 * later collection. 760 * => called with pageq's locked by the daemon. 761 * 762 * general outline: 763 * - "try" to lock object. if fail, just return (will try again later) 764 * - drop "u_nio" (this req is done!) 765 * - if (object->iosync && u_naio == 0) { wakeup &uvn->u_naio } 766 * - get "page" structures (atop?). 767 * - handle "wanted" pages 768 * - handle "released" pages [using pgo_releasepg] 769 * >>> pgo_releasepg may kill the object 770 * dont forget to look at "object" wanted flag in all cases. 771 */ 772 773 774 /* 775 * uvn_flush: flush pages out of a uvm object. 776 * 777 * => object should be locked by caller. we may _unlock_ the object 778 * if (and only if) we need to clean a page (PGO_CLEANIT). 779 * we return with the object locked. 780 * => if PGO_CLEANIT is set, we may block (due to I/O). thus, a caller 781 * might want to unlock higher level resources (e.g. vm_map) 782 * before calling flush. 783 * => if PGO_CLEANIT is not set, then we will neither unlock the object 784 * or block. 785 * => if PGO_ALLPAGE is set, then all pages in the object are valid targets 786 * for flushing. 787 * => NOTE: we rely on the fact that the object's memq is a TAILQ and 788 * that new pages are inserted on the tail end of the list. thus, 789 * we can make a complete pass through the object in one go by starting 790 * at the head and working towards the tail (new pages are put in 791 * front of us). 792 * => NOTE: we are allowed to lock the page queues, so the caller 793 * must not be holding the lock on them [e.g. pagedaemon had 794 * better not call us with the queues locked] 795 * => we return TRUE unless we encountered some sort of I/O error 796 * 797 * comment on "cleaning" object and PG_BUSY pages: 798 * this routine is holding the lock on the object. the only time 799 * that it can run into a PG_BUSY page that it does not own is if 800 * some other process has started I/O on the page (e.g. either 801 * a pagein, or a pageout). if the PG_BUSY page is being paged 802 * in, then it can not be dirty (!PG_CLEAN) because no one has 803 * had a chance to modify it yet. if the PG_BUSY page is being 804 * paged out then it means that someone else has already started 805 * cleaning the page for us (how nice!). in this case, if we 806 * have syncio specified, then after we make our pass through the 807 * object we need to wait for the other PG_BUSY pages to clear 808 * off (i.e. we need to do an iosync). also note that once a 809 * page is PG_BUSY it must stay in its object until it is un-busyed. 810 * 811 * note on page traversal: 812 * we can traverse the pages in an object either by going down the 813 * linked list in "uobj->memq", or we can go over the address range 814 * by page doing hash table lookups for each address. depending 815 * on how many pages are in the object it may be cheaper to do one 816 * or the other. we set "by_list" to true if we are using memq. 817 * if the cost of a hash lookup was equal to the cost of the list 818 * traversal we could compare the number of pages in the start->stop 819 * range to the total number of pages in the object. however, it 820 * seems that a hash table lookup is more expensive than the linked 821 * list traversal, so we multiply the number of pages in the 822 * start->stop range by a penalty which we define below. 823 */ 824 825 #define UVN_HASH_PENALTY 4 /* XXX: a guess */ 826 827 static boolean_t 828 uvn_flush(uobj, start, stop, flags) 829 struct uvm_object *uobj; 830 voff_t start, stop; 831 int flags; 832 { 833 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 834 struct vm_page *pp, *ppnext, *ptmp; 835 struct vm_page *pps[MAXBSIZE >> PAGE_SHIFT], **ppsp; 836 int npages, result, lcv; 837 boolean_t retval, need_iosync, by_list, needs_clean, all; 838 voff_t curoff; 839 u_short pp_version; 840 UVMHIST_FUNC("uvn_flush"); UVMHIST_CALLED(maphist); 841 842 curoff = 0; /* XXX: shut up gcc */ 843 /* 844 * get init vals and determine how we are going to traverse object 845 */ 846 847 need_iosync = FALSE; 848 retval = TRUE; /* return value */ 849 if (flags & PGO_ALLPAGES) { 850 all = TRUE; 851 by_list = TRUE; /* always go by the list */ 852 } else { 853 start = trunc_page(start); 854 stop = round_page(stop); 855 #ifdef DEBUG 856 if (stop > round_page(uvn->u_size)) 857 printf("uvn_flush: strange, got an out of range " 858 "flush (fixed)\n"); 859 #endif 860 all = FALSE; 861 by_list = (uobj->uo_npages <= 862 ((stop - start) >> PAGE_SHIFT) * UVN_HASH_PENALTY); 863 } 864 865 UVMHIST_LOG(maphist, 866 " flush start=0x%x, stop=0x%x, by_list=%d, flags=0x%x", 867 start, stop, by_list, flags); 868 869 /* 870 * PG_CLEANCHK: this bit is used by the pgo_mk_pcluster function as 871 * a _hint_ as to how up to date the PG_CLEAN bit is. if the hint 872 * is wrong it will only prevent us from clustering... it won't break 873 * anything. we clear all PG_CLEANCHK bits here, and pgo_mk_pcluster 874 * will set them as it syncs PG_CLEAN. This is only an issue if we 875 * are looking at non-inactive pages (because inactive page's PG_CLEAN 876 * bit is always up to date since there are no mappings). 877 * [borrowed PG_CLEANCHK idea from FreeBSD VM] 878 */ 879 880 if ((flags & PGO_CLEANIT) != 0 && 881 uobj->pgops->pgo_mk_pcluster != NULL) { 882 if (by_list) { 883 for (pp = uobj->memq.tqh_first ; pp != NULL ; 884 pp = pp->listq.tqe_next) { 885 if (!all && 886 (pp->offset < start || pp->offset >= stop)) 887 continue; 888 pp->flags &= ~PG_CLEANCHK; 889 } 890 891 } else { /* by hash */ 892 for (curoff = start ; curoff < stop; 893 curoff += PAGE_SIZE) { 894 pp = uvm_pagelookup(uobj, curoff); 895 if (pp) 896 pp->flags &= ~PG_CLEANCHK; 897 } 898 } 899 } 900 901 /* 902 * now do it. note: we must update ppnext in body of loop or we 903 * will get stuck. we need to use ppnext because we may free "pp" 904 * before doing the next loop. 905 */ 906 907 if (by_list) { 908 pp = uobj->memq.tqh_first; 909 } else { 910 curoff = start; 911 pp = uvm_pagelookup(uobj, curoff); 912 } 913 914 ppnext = NULL; /* XXX: shut up gcc */ 915 ppsp = NULL; /* XXX: shut up gcc */ 916 uvm_lock_pageq(); /* page queues locked */ 917 918 /* locked: both page queues and uobj */ 919 for ( ; (by_list && pp != NULL) || 920 (!by_list && curoff < stop) ; pp = ppnext) { 921 922 if (by_list) { 923 924 /* 925 * range check 926 */ 927 928 if (!all && 929 (pp->offset < start || pp->offset >= stop)) { 930 ppnext = pp->listq.tqe_next; 931 continue; 932 } 933 934 } else { 935 936 /* 937 * null check 938 */ 939 940 curoff += PAGE_SIZE; 941 if (pp == NULL) { 942 if (curoff < stop) 943 ppnext = uvm_pagelookup(uobj, curoff); 944 continue; 945 } 946 947 } 948 949 /* 950 * handle case where we do not need to clean page (either 951 * because we are not clean or because page is not dirty or 952 * is busy): 953 * 954 * NOTE: we are allowed to deactivate a non-wired active 955 * PG_BUSY page, but once a PG_BUSY page is on the inactive 956 * queue it must stay put until it is !PG_BUSY (so as not to 957 * confuse pagedaemon). 958 */ 959 960 if ((flags & PGO_CLEANIT) == 0 || (pp->flags & PG_BUSY) != 0) { 961 needs_clean = FALSE; 962 if ((pp->flags & PG_BUSY) != 0 && 963 (flags & (PGO_CLEANIT|PGO_SYNCIO)) == 964 (PGO_CLEANIT|PGO_SYNCIO)) 965 need_iosync = TRUE; 966 } else { 967 /* 968 * freeing: nuke all mappings so we can sync 969 * PG_CLEAN bit with no race 970 */ 971 if ((pp->flags & PG_CLEAN) != 0 && 972 (flags & PGO_FREE) != 0 && 973 (pp->pqflags & PQ_ACTIVE) != 0) 974 pmap_page_protect(pp, VM_PROT_NONE); 975 if ((pp->flags & PG_CLEAN) != 0 && 976 pmap_is_modified(pp)) 977 pp->flags &= ~(PG_CLEAN); 978 pp->flags |= PG_CLEANCHK; /* update "hint" */ 979 980 needs_clean = ((pp->flags & PG_CLEAN) == 0); 981 } 982 983 /* 984 * if we don't need a clean... load ppnext and dispose of pp 985 */ 986 if (!needs_clean) { 987 /* load ppnext */ 988 if (by_list) 989 ppnext = pp->listq.tqe_next; 990 else { 991 if (curoff < stop) 992 ppnext = uvm_pagelookup(uobj, curoff); 993 } 994 995 /* now dispose of pp */ 996 if (flags & PGO_DEACTIVATE) { 997 if ((pp->pqflags & PQ_INACTIVE) == 0 && 998 pp->wire_count == 0) { 999 pmap_page_protect(pp, VM_PROT_NONE); 1000 uvm_pagedeactivate(pp); 1001 } 1002 1003 } else if (flags & PGO_FREE) { 1004 if (pp->flags & PG_BUSY) { 1005 /* release busy pages */ 1006 pp->flags |= PG_RELEASED; 1007 } else { 1008 pmap_page_protect(pp, VM_PROT_NONE); 1009 /* removed page from object */ 1010 uvm_pagefree(pp); 1011 } 1012 } 1013 /* ppnext is valid so we can continue... */ 1014 continue; 1015 } 1016 1017 /* 1018 * pp points to a page in the locked object that we are 1019 * working on. if it is !PG_CLEAN,!PG_BUSY and we asked 1020 * for cleaning (PGO_CLEANIT). we clean it now. 1021 * 1022 * let uvm_pager_put attempted a clustered page out. 1023 * note: locked: uobj and page queues. 1024 */ 1025 1026 pp->flags |= PG_BUSY; /* we 'own' page now */ 1027 UVM_PAGE_OWN(pp, "uvn_flush"); 1028 pmap_page_protect(pp, VM_PROT_READ); 1029 pp_version = pp->version; 1030 ReTry: 1031 ppsp = pps; 1032 npages = sizeof(pps) / sizeof(struct vm_page *); 1033 1034 /* locked: page queues, uobj */ 1035 result = uvm_pager_put(uobj, pp, &ppsp, &npages, 1036 flags | PGO_DOACTCLUST, start, stop); 1037 /* unlocked: page queues, uobj */ 1038 1039 /* 1040 * at this point nothing is locked. if we did an async I/O 1041 * it is remotely possible for the async i/o to complete and 1042 * the page "pp" be freed or what not before we get a chance 1043 * to relock the object. in order to detect this, we have 1044 * saved the version number of the page in "pp_version". 1045 */ 1046 1047 /* relock! */ 1048 simple_lock(&uobj->vmobjlock); 1049 uvm_lock_pageq(); 1050 1051 /* 1052 * VM_PAGER_AGAIN: given the structure of this pager, this 1053 * can only happen when we are doing async I/O and can't 1054 * map the pages into kernel memory (pager_map) due to lack 1055 * of vm space. if this happens we drop back to sync I/O. 1056 */ 1057 1058 if (result == VM_PAGER_AGAIN) { 1059 /* 1060 * it is unlikely, but page could have been released 1061 * while we had the object lock dropped. we ignore 1062 * this now and retry the I/O. we will detect and 1063 * handle the released page after the syncio I/O 1064 * completes. 1065 */ 1066 #ifdef DIAGNOSTIC 1067 if (flags & PGO_SYNCIO) 1068 panic("uvn_flush: PGO_SYNCIO return 'try again' error (impossible)"); 1069 #endif 1070 flags |= PGO_SYNCIO; 1071 goto ReTry; 1072 } 1073 1074 /* 1075 * the cleaning operation is now done. finish up. note that 1076 * on error (!OK, !PEND) uvm_pager_put drops the cluster for us. 1077 * if success (OK, PEND) then uvm_pager_put returns the cluster 1078 * to us in ppsp/npages. 1079 */ 1080 1081 /* 1082 * for pending async i/o if we are not deactivating/freeing 1083 * we can move on to the next page. 1084 */ 1085 1086 if (result == VM_PAGER_PEND) { 1087 1088 if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) { 1089 /* 1090 * no per-page ops: refresh ppnext and continue 1091 */ 1092 if (by_list) { 1093 if (pp->version == pp_version) 1094 ppnext = pp->listq.tqe_next; 1095 else 1096 /* reset */ 1097 ppnext = uobj->memq.tqh_first; 1098 } else { 1099 if (curoff < stop) 1100 ppnext = uvm_pagelookup(uobj, 1101 curoff); 1102 } 1103 continue; 1104 } 1105 1106 /* need to do anything here? */ 1107 } 1108 1109 /* 1110 * need to look at each page of the I/O operation. we defer 1111 * processing "pp" until the last trip through this "for" loop 1112 * so that we can load "ppnext" for the main loop after we 1113 * play with the cluster pages [thus the "npages + 1" in the 1114 * loop below]. 1115 */ 1116 1117 for (lcv = 0 ; lcv < npages + 1 ; lcv++) { 1118 1119 /* 1120 * handle ppnext for outside loop, and saving pp 1121 * until the end. 1122 */ 1123 if (lcv < npages) { 1124 if (ppsp[lcv] == pp) 1125 continue; /* skip pp until the end */ 1126 ptmp = ppsp[lcv]; 1127 } else { 1128 ptmp = pp; 1129 1130 /* set up next page for outer loop */ 1131 if (by_list) { 1132 if (pp->version == pp_version) 1133 ppnext = pp->listq.tqe_next; 1134 else 1135 /* reset */ 1136 ppnext = uobj->memq.tqh_first; 1137 } else { 1138 if (curoff < stop) 1139 ppnext = uvm_pagelookup(uobj, curoff); 1140 } 1141 } 1142 1143 /* 1144 * verify the page didn't get moved while obj was 1145 * unlocked 1146 */ 1147 if (result == VM_PAGER_PEND && ptmp->uobject != uobj) 1148 continue; 1149 1150 /* 1151 * unbusy the page if I/O is done. note that for 1152 * pending I/O it is possible that the I/O op 1153 * finished before we relocked the object (in 1154 * which case the page is no longer busy). 1155 */ 1156 1157 if (result != VM_PAGER_PEND) { 1158 if (ptmp->flags & PG_WANTED) 1159 /* still holding object lock */ 1160 wakeup(ptmp); 1161 1162 ptmp->flags &= ~(PG_WANTED|PG_BUSY); 1163 UVM_PAGE_OWN(ptmp, NULL); 1164 if (ptmp->flags & PG_RELEASED) { 1165 1166 /* pgo_releasepg wants this */ 1167 uvm_unlock_pageq(); 1168 if (!uvn_releasepg(ptmp, NULL)) 1169 return (TRUE); 1170 1171 uvm_lock_pageq(); /* relock */ 1172 continue; /* next page */ 1173 1174 } else { 1175 ptmp->flags |= (PG_CLEAN|PG_CLEANCHK); 1176 if ((flags & PGO_FREE) == 0) 1177 pmap_clear_modify(ptmp); 1178 } 1179 } 1180 1181 /* 1182 * dispose of page 1183 */ 1184 1185 if (flags & PGO_DEACTIVATE) { 1186 if ((pp->pqflags & PQ_INACTIVE) == 0 && 1187 pp->wire_count == 0) { 1188 pmap_page_protect(ptmp, VM_PROT_NONE); 1189 uvm_pagedeactivate(ptmp); 1190 } 1191 1192 } else if (flags & PGO_FREE) { 1193 if (result == VM_PAGER_PEND) { 1194 if ((ptmp->flags & PG_BUSY) != 0) 1195 /* signal for i/o done */ 1196 ptmp->flags |= PG_RELEASED; 1197 } else { 1198 if (result != VM_PAGER_OK) { 1199 printf("uvn_flush: obj=%p, " 1200 "offset=0x%llx. error " 1201 "during pageout.\n", 1202 pp->uobject, 1203 (long long)pp->offset); 1204 printf("uvn_flush: WARNING: " 1205 "changes to page may be " 1206 "lost!\n"); 1207 retval = FALSE; 1208 } 1209 pmap_page_protect(ptmp, VM_PROT_NONE); 1210 uvm_pagefree(ptmp); 1211 } 1212 } 1213 1214 } /* end of "lcv" for loop */ 1215 1216 } /* end of "pp" for loop */ 1217 1218 /* 1219 * done with pagequeues: unlock 1220 */ 1221 uvm_unlock_pageq(); 1222 1223 /* 1224 * now wait for all I/O if required. 1225 */ 1226 if (need_iosync) { 1227 1228 UVMHIST_LOG(maphist," <<DOING IOSYNC>>",0,0,0,0); 1229 while (uvn->u_nio != 0) { 1230 uvn->u_flags |= UVM_VNODE_IOSYNC; 1231 UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock, 1232 FALSE, "uvn_flush",0); 1233 simple_lock(&uvn->u_obj.vmobjlock); 1234 } 1235 if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED) 1236 wakeup(&uvn->u_flags); 1237 uvn->u_flags &= ~(UVM_VNODE_IOSYNC|UVM_VNODE_IOSYNCWANTED); 1238 } 1239 1240 /* return, with object locked! */ 1241 UVMHIST_LOG(maphist,"<- done (retval=0x%x)",retval,0,0,0); 1242 return(retval); 1243 } 1244 1245 /* 1246 * uvn_cluster 1247 * 1248 * we are about to do I/O in an object at offset. this function is called 1249 * to establish a range of offsets around "offset" in which we can cluster 1250 * I/O. 1251 * 1252 * - currently doesn't matter if obj locked or not. 1253 */ 1254 1255 static void 1256 uvn_cluster(uobj, offset, loffset, hoffset) 1257 struct uvm_object *uobj; 1258 voff_t offset; 1259 voff_t *loffset, *hoffset; /* OUT */ 1260 { 1261 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 1262 *loffset = offset; 1263 1264 if (*loffset >= uvn->u_size) 1265 panic("uvn_cluster: offset out of range"); 1266 1267 /* 1268 * XXX: old pager claims we could use VOP_BMAP to get maxcontig value. 1269 */ 1270 *hoffset = *loffset + MAXBSIZE; 1271 if (*hoffset > round_page(uvn->u_size)) /* past end? */ 1272 *hoffset = round_page(uvn->u_size); 1273 1274 return; 1275 } 1276 1277 /* 1278 * uvn_put: flush page data to backing store. 1279 * 1280 * => prefer map unlocked (not required) 1281 * => object must be locked! we will _unlock_ it before starting I/O. 1282 * => flags: PGO_SYNCIO -- use sync. I/O 1283 * => note: caller must set PG_CLEAN and pmap_clear_modify (if needed) 1284 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync. 1285 * [thus we never do async i/o! see iodone comment] 1286 */ 1287 1288 static int 1289 uvn_put(uobj, pps, npages, flags) 1290 struct uvm_object *uobj; 1291 struct vm_page **pps; 1292 int npages, flags; 1293 { 1294 int retval; 1295 1296 /* note: object locked */ 1297 retval = uvn_io((struct uvm_vnode*)uobj, pps, npages, flags, UIO_WRITE); 1298 /* note: object unlocked */ 1299 1300 return(retval); 1301 } 1302 1303 1304 /* 1305 * uvn_get: get pages (synchronously) from backing store 1306 * 1307 * => prefer map unlocked (not required) 1308 * => object must be locked! we will _unlock_ it before starting any I/O. 1309 * => flags: PGO_ALLPAGES: get all of the pages 1310 * PGO_LOCKED: fault data structures are locked 1311 * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx] 1312 * => NOTE: caller must check for released pages!! 1313 */ 1314 1315 static int 1316 uvn_get(uobj, offset, pps, npagesp, centeridx, access_type, advice, flags) 1317 struct uvm_object *uobj; 1318 voff_t offset; 1319 struct vm_page **pps; /* IN/OUT */ 1320 int *npagesp; /* IN (OUT if PGO_LOCKED) */ 1321 int centeridx, advice, flags; 1322 vm_prot_t access_type; 1323 { 1324 voff_t current_offset; 1325 struct vm_page *ptmp; 1326 int lcv, result, gotpages; 1327 boolean_t done; 1328 UVMHIST_FUNC("uvn_get"); UVMHIST_CALLED(maphist); 1329 UVMHIST_LOG(maphist, "flags=%d", flags,0,0,0); 1330 1331 /* 1332 * step 1: handled the case where fault data structures are locked. 1333 */ 1334 1335 if (flags & PGO_LOCKED) { 1336 1337 /* 1338 * gotpages is the current number of pages we've gotten (which 1339 * we pass back up to caller via *npagesp. 1340 */ 1341 1342 gotpages = 0; 1343 1344 /* 1345 * step 1a: get pages that are already resident. only do this 1346 * if the data structures are locked (i.e. the first time 1347 * through). 1348 */ 1349 1350 done = TRUE; /* be optimistic */ 1351 1352 for (lcv = 0, current_offset = offset ; lcv < *npagesp ; 1353 lcv++, current_offset += PAGE_SIZE) { 1354 1355 /* do we care about this page? if not, skip it */ 1356 if (pps[lcv] == PGO_DONTCARE) 1357 continue; 1358 1359 /* lookup page */ 1360 ptmp = uvm_pagelookup(uobj, current_offset); 1361 1362 /* to be useful must get a non-busy, non-released pg */ 1363 if (ptmp == NULL || 1364 (ptmp->flags & (PG_BUSY|PG_RELEASED)) != 0) { 1365 if (lcv == centeridx || (flags & PGO_ALLPAGES) 1366 != 0) 1367 done = FALSE; /* need to do a wait or I/O! */ 1368 continue; 1369 } 1370 1371 /* 1372 * useful page: busy/lock it and plug it in our 1373 * result array 1374 */ 1375 ptmp->flags |= PG_BUSY; /* loan up to caller */ 1376 UVM_PAGE_OWN(ptmp, "uvn_get1"); 1377 pps[lcv] = ptmp; 1378 gotpages++; 1379 1380 } /* "for" lcv loop */ 1381 1382 /* 1383 * XXX: given the "advice", should we consider async read-ahead? 1384 * XXX: fault current does deactive of pages behind us. is 1385 * this good (other callers might now). 1386 */ 1387 /* 1388 * XXX: read-ahead currently handled by buffer cache (bread) 1389 * level. 1390 * XXX: no async i/o available. 1391 * XXX: so we don't do anything now. 1392 */ 1393 1394 /* 1395 * step 1c: now we've either done everything needed or we to 1396 * unlock and do some waiting or I/O. 1397 */ 1398 1399 *npagesp = gotpages; /* let caller know */ 1400 if (done) 1401 return(VM_PAGER_OK); /* bingo! */ 1402 else 1403 /* EEK! Need to unlock and I/O */ 1404 return(VM_PAGER_UNLOCK); 1405 } 1406 1407 /* 1408 * step 2: get non-resident or busy pages. 1409 * object is locked. data structures are unlocked. 1410 * 1411 * XXX: because we can't do async I/O at this level we get things 1412 * page at a time (otherwise we'd chunk). the VOP_READ() will do 1413 * async-read-ahead for us at a lower level. 1414 */ 1415 1416 for (lcv = 0, current_offset = offset ; 1417 lcv < *npagesp ; lcv++, current_offset += PAGE_SIZE) { 1418 1419 /* skip over pages we've already gotten or don't want */ 1420 /* skip over pages we don't _have_ to get */ 1421 if (pps[lcv] != NULL || (lcv != centeridx && 1422 (flags & PGO_ALLPAGES) == 0)) 1423 continue; 1424 1425 /* 1426 * we have yet to locate the current page (pps[lcv]). we first 1427 * look for a page that is already at the current offset. if 1428 * we fine a page, we check to see if it is busy or released. 1429 * if that is the case, then we sleep on the page until it is 1430 * no longer busy or released and repeat the lookup. if the 1431 * page we found is neither busy nor released, then we busy it 1432 * (so we own it) and plug it into pps[lcv]. this breaks the 1433 * following while loop and indicates we are ready to move on 1434 * to the next page in the "lcv" loop above. 1435 * 1436 * if we exit the while loop with pps[lcv] still set to NULL, 1437 * then it means that we allocated a new busy/fake/clean page 1438 * ptmp in the object and we need to do I/O to fill in the data. 1439 */ 1440 1441 while (pps[lcv] == NULL) { /* top of "pps" while loop */ 1442 1443 /* look for a current page */ 1444 ptmp = uvm_pagelookup(uobj, current_offset); 1445 1446 /* nope? allocate one now (if we can) */ 1447 if (ptmp == NULL) { 1448 1449 ptmp = uvm_pagealloc(uobj, current_offset, 1450 NULL, 0); 1451 1452 /* out of RAM? */ 1453 if (ptmp == NULL) { 1454 simple_unlock(&uobj->vmobjlock); 1455 uvm_wait("uvn_getpage"); 1456 simple_lock(&uobj->vmobjlock); 1457 1458 /* goto top of pps while loop */ 1459 continue; 1460 } 1461 1462 /* 1463 * got new page ready for I/O. break pps 1464 * while loop. pps[lcv] is still NULL. 1465 */ 1466 break; 1467 } 1468 1469 /* page is there, see if we need to wait on it */ 1470 if ((ptmp->flags & (PG_BUSY|PG_RELEASED)) != 0) { 1471 ptmp->flags |= PG_WANTED; 1472 UVM_UNLOCK_AND_WAIT(ptmp, 1473 &uobj->vmobjlock, FALSE, "uvn_get",0); 1474 simple_lock(&uobj->vmobjlock); 1475 continue; /* goto top of pps while loop */ 1476 } 1477 1478 /* 1479 * if we get here then the page has become resident 1480 * and unbusy between steps 1 and 2. we busy it 1481 * now (so we own it) and set pps[lcv] (so that we 1482 * exit the while loop). 1483 */ 1484 ptmp->flags |= PG_BUSY; 1485 UVM_PAGE_OWN(ptmp, "uvn_get2"); 1486 pps[lcv] = ptmp; 1487 } 1488 1489 /* 1490 * if we own the a valid page at the correct offset, pps[lcv] 1491 * will point to it. nothing more to do except go to the 1492 * next page. 1493 */ 1494 1495 if (pps[lcv]) 1496 continue; /* next lcv */ 1497 1498 /* 1499 * we have a "fake/busy/clean" page that we just allocated. do 1500 * I/O to fill it with valid data. note that object must be 1501 * locked going into uvn_io, but will be unlocked afterwards. 1502 */ 1503 1504 result = uvn_io((struct uvm_vnode *) uobj, &ptmp, 1, 1505 PGO_SYNCIO, UIO_READ); 1506 1507 /* 1508 * I/O done. object is unlocked (by uvn_io). because we used 1509 * syncio the result can not be PEND or AGAIN. we must relock 1510 * and check for errors. 1511 */ 1512 1513 /* lock object. check for errors. */ 1514 simple_lock(&uobj->vmobjlock); 1515 if (result != VM_PAGER_OK) { 1516 if (ptmp->flags & PG_WANTED) 1517 /* object lock still held */ 1518 wakeup(ptmp); 1519 1520 ptmp->flags &= ~(PG_WANTED|PG_BUSY); 1521 UVM_PAGE_OWN(ptmp, NULL); 1522 uvm_lock_pageq(); 1523 uvm_pagefree(ptmp); 1524 uvm_unlock_pageq(); 1525 simple_unlock(&uobj->vmobjlock); 1526 return(result); 1527 } 1528 1529 /* 1530 * we got the page! clear the fake flag (indicates valid 1531 * data now in page) and plug into our result array. note 1532 * that page is still busy. 1533 * 1534 * it is the callers job to: 1535 * => check if the page is released 1536 * => unbusy the page 1537 * => activate the page 1538 */ 1539 1540 ptmp->flags &= ~PG_FAKE; /* data is valid ... */ 1541 pmap_clear_modify(ptmp); /* ... and clean */ 1542 pps[lcv] = ptmp; 1543 1544 } /* lcv loop */ 1545 1546 /* 1547 * finally, unlock object and return. 1548 */ 1549 1550 simple_unlock(&uobj->vmobjlock); 1551 return (VM_PAGER_OK); 1552 } 1553 1554 /* 1555 * uvn_asyncget: start async I/O to bring pages into ram 1556 * 1557 * => caller must lock object(???XXX: see if this is best) 1558 * => could be called from uvn_get or a madvise() fault-ahead. 1559 * => if it fails, it doesn't matter. 1560 */ 1561 1562 static int 1563 uvn_asyncget(uobj, offset, npages) 1564 struct uvm_object *uobj; 1565 voff_t offset; 1566 int npages; 1567 { 1568 1569 /* 1570 * XXXCDC: we can't do async I/O yet 1571 */ 1572 printf("uvn_asyncget called\n"); 1573 return (KERN_SUCCESS); 1574 } 1575 1576 /* 1577 * uvn_io: do I/O to a vnode 1578 * 1579 * => prefer map unlocked (not required) 1580 * => object must be locked! we will _unlock_ it before starting I/O. 1581 * => flags: PGO_SYNCIO -- use sync. I/O 1582 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync. 1583 * [thus we never do async i/o! see iodone comment] 1584 */ 1585 1586 static int 1587 uvn_io(uvn, pps, npages, flags, rw) 1588 struct uvm_vnode *uvn; 1589 vm_page_t *pps; 1590 int npages, flags, rw; 1591 { 1592 struct vnode *vn; 1593 struct uio uio; 1594 struct iovec iov; 1595 vaddr_t kva; 1596 off_t file_offset; 1597 int waitf, result, mapinflags; 1598 size_t got, wanted; 1599 UVMHIST_FUNC("uvn_io"); UVMHIST_CALLED(maphist); 1600 1601 UVMHIST_LOG(maphist, "rw=%d", rw,0,0,0); 1602 1603 /* 1604 * init values 1605 */ 1606 1607 waitf = (flags & PGO_SYNCIO) ? M_WAITOK : M_NOWAIT; 1608 vn = (struct vnode *) uvn; 1609 file_offset = pps[0]->offset; 1610 1611 /* 1612 * check for sync'ing I/O. 1613 */ 1614 1615 while (uvn->u_flags & UVM_VNODE_IOSYNC) { 1616 if (waitf == M_NOWAIT) { 1617 simple_unlock(&uvn->u_obj.vmobjlock); 1618 UVMHIST_LOG(maphist,"<- try again (iosync)",0,0,0,0); 1619 return(VM_PAGER_AGAIN); 1620 } 1621 uvn->u_flags |= UVM_VNODE_IOSYNCWANTED; 1622 UVM_UNLOCK_AND_WAIT(&uvn->u_flags, &uvn->u_obj.vmobjlock, 1623 FALSE, "uvn_iosync",0); 1624 simple_lock(&uvn->u_obj.vmobjlock); 1625 } 1626 1627 /* 1628 * check size 1629 */ 1630 1631 if (file_offset >= uvn->u_size) { 1632 simple_unlock(&uvn->u_obj.vmobjlock); 1633 UVMHIST_LOG(maphist,"<- BAD (size check)",0,0,0,0); 1634 return(VM_PAGER_BAD); 1635 } 1636 1637 /* 1638 * first try and map the pages in (without waiting) 1639 */ 1640 1641 mapinflags = (rw == UIO_READ) ? 1642 UVMPAGER_MAPIN_READ : UVMPAGER_MAPIN_WRITE; 1643 1644 kva = uvm_pagermapin(pps, npages, NULL, mapinflags); 1645 if (kva == 0 && waitf == M_NOWAIT) { 1646 simple_unlock(&uvn->u_obj.vmobjlock); 1647 UVMHIST_LOG(maphist,"<- mapin failed (try again)",0,0,0,0); 1648 return(VM_PAGER_AGAIN); 1649 } 1650 1651 /* 1652 * ok, now bump u_nio up. at this point we are done with uvn 1653 * and can unlock it. if we still don't have a kva, try again 1654 * (this time with sleep ok). 1655 */ 1656 1657 uvn->u_nio++; /* we have an I/O in progress! */ 1658 simple_unlock(&uvn->u_obj.vmobjlock); 1659 /* NOTE: object now unlocked */ 1660 if (kva == 0) 1661 kva = uvm_pagermapin(pps, npages, NULL, 1662 mapinflags | UVMPAGER_MAPIN_WAITOK); 1663 1664 /* 1665 * ok, mapped in. our pages are PG_BUSY so they are not going to 1666 * get touched (so we can look at "offset" without having to lock 1667 * the object). set up for I/O. 1668 */ 1669 1670 /* 1671 * fill out uio/iov 1672 */ 1673 1674 iov.iov_base = (caddr_t) kva; 1675 wanted = npages << PAGE_SHIFT; 1676 if (file_offset + wanted > uvn->u_size) 1677 wanted = uvn->u_size - file_offset; /* XXX: needed? */ 1678 iov.iov_len = wanted; 1679 uio.uio_iov = &iov; 1680 uio.uio_iovcnt = 1; 1681 uio.uio_offset = file_offset; 1682 uio.uio_segflg = UIO_SYSSPACE; 1683 uio.uio_rw = rw; 1684 uio.uio_resid = wanted; 1685 uio.uio_procp = NULL; 1686 1687 /* 1688 * do the I/O! (XXX: curproc?) 1689 */ 1690 1691 UVMHIST_LOG(maphist, "calling VOP",0,0,0,0); 1692 1693 /* 1694 * This process may already have this vnode locked, if we faulted in 1695 * copyin() or copyout() on a region backed by this vnode 1696 * while doing I/O to the vnode. If this is the case, don't 1697 * panic.. instead, return the error to the user. 1698 * 1699 * XXX this is a stopgap to prevent a panic. 1700 * Ideally, this kind of operation *should* work. 1701 */ 1702 result = 0; 1703 if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0) 1704 result = vn_lock(vn, LK_EXCLUSIVE | LK_RETRY | LK_RECURSEFAIL); 1705 1706 if (result == 0) { 1707 /* NOTE: vnode now locked! */ 1708 1709 if (rw == UIO_READ) 1710 result = VOP_READ(vn, &uio, 0, curproc->p_ucred); 1711 else 1712 result = VOP_WRITE(vn, &uio, 0, curproc->p_ucred); 1713 1714 if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0) 1715 VOP_UNLOCK(vn, 0); 1716 } 1717 1718 /* NOTE: vnode now unlocked (unless vnislocked) */ 1719 1720 UVMHIST_LOG(maphist, "done calling VOP",0,0,0,0); 1721 1722 /* 1723 * result == unix style errno (0 == OK!) 1724 * 1725 * zero out rest of buffer (if needed) 1726 */ 1727 1728 if (result == 0) { 1729 got = wanted - uio.uio_resid; 1730 1731 if (wanted && got == 0) { 1732 result = EIO; /* XXX: error? */ 1733 } else if (got < PAGE_SIZE * npages && rw == UIO_READ) { 1734 memset((void *) (kva + got), 0, 1735 (npages << PAGE_SHIFT) - got); 1736 } 1737 } 1738 1739 /* 1740 * now remove pager mapping 1741 */ 1742 uvm_pagermapout(kva, npages); 1743 1744 /* 1745 * now clean up the object (i.e. drop I/O count) 1746 */ 1747 1748 simple_lock(&uvn->u_obj.vmobjlock); 1749 /* NOTE: object now locked! */ 1750 1751 uvn->u_nio--; /* I/O DONE! */ 1752 if ((uvn->u_flags & UVM_VNODE_IOSYNC) != 0 && uvn->u_nio == 0) { 1753 wakeup(&uvn->u_nio); 1754 } 1755 simple_unlock(&uvn->u_obj.vmobjlock); 1756 /* NOTE: object now unlocked! */ 1757 1758 /* 1759 * done! 1760 */ 1761 1762 UVMHIST_LOG(maphist, "<- done (result %d)", result,0,0,0); 1763 if (result == 0) 1764 return(VM_PAGER_OK); 1765 else 1766 return(VM_PAGER_ERROR); 1767 } 1768 1769 /* 1770 * uvm_vnp_uncache: disable "persisting" in a vnode... when last reference 1771 * is gone we will kill the object (flushing dirty pages back to the vnode 1772 * if needed). 1773 * 1774 * => returns TRUE if there was no uvm_object attached or if there was 1775 * one and we killed it [i.e. if there is no active uvn] 1776 * => called with the vnode VOP_LOCK'd [we will unlock it for I/O, if 1777 * needed] 1778 * 1779 * => XXX: given that we now kill uvn's when a vnode is recycled (without 1780 * having to hold a reference on the vnode) and given a working 1781 * uvm_vnp_sync(), how does that effect the need for this function? 1782 * [XXXCDC: seems like it can die?] 1783 * 1784 * => XXX: this function should DIE once we merge the VM and buffer 1785 * cache. 1786 * 1787 * research shows that this is called in the following places: 1788 * ext2fs_truncate, ffs_truncate, detrunc[msdosfs]: called when vnode 1789 * changes sizes 1790 * ext2fs_write, WRITE [ufs_readwrite], msdosfs_write: called when we 1791 * are written to 1792 * ex2fs_chmod, ufs_chmod: called if VTEXT vnode and the sticky bit 1793 * is off 1794 * ffs_realloccg: when we can't extend the current block and have 1795 * to allocate a new one we call this [XXX: why?] 1796 * nfsrv_rename, rename_files: called when the target filename is there 1797 * and we want to remove it 1798 * nfsrv_remove, sys_unlink: called on file we are removing 1799 * nfsrv_access: if VTEXT and we want WRITE access and we don't uncache 1800 * then return "text busy" 1801 * nfs_open: seems to uncache any file opened with nfs 1802 * vn_writechk: if VTEXT vnode and can't uncache return "text busy" 1803 */ 1804 1805 boolean_t 1806 uvm_vnp_uncache(vp) 1807 struct vnode *vp; 1808 { 1809 struct uvm_vnode *uvn = &vp->v_uvm; 1810 1811 /* 1812 * lock uvn part of the vnode and check to see if we need to do anything 1813 */ 1814 1815 simple_lock(&uvn->u_obj.vmobjlock); 1816 if ((uvn->u_flags & UVM_VNODE_VALID) == 0 || 1817 (uvn->u_flags & UVM_VNODE_BLOCKED) != 0) { 1818 simple_unlock(&uvn->u_obj.vmobjlock); 1819 return(TRUE); 1820 } 1821 1822 /* 1823 * we have a valid, non-blocked uvn. clear persist flag. 1824 * if uvn is currently active we can return now. 1825 */ 1826 1827 uvn->u_flags &= ~UVM_VNODE_CANPERSIST; 1828 if (uvn->u_obj.uo_refs) { 1829 simple_unlock(&uvn->u_obj.vmobjlock); 1830 return(FALSE); 1831 } 1832 1833 /* 1834 * uvn is currently persisting! we have to gain a reference to 1835 * it so that we can call uvn_detach to kill the uvn. 1836 */ 1837 1838 VREF(vp); /* seems ok, even with VOP_LOCK */ 1839 uvn->u_obj.uo_refs++; /* value is now 1 */ 1840 simple_unlock(&uvn->u_obj.vmobjlock); 1841 1842 1843 #ifdef DEBUG 1844 /* 1845 * carry over sanity check from old vnode pager: the vnode should 1846 * be VOP_LOCK'd, and we confirm it here. 1847 */ 1848 if (!VOP_ISLOCKED(vp)) { 1849 boolean_t is_ok_anyway = FALSE; 1850 #ifdef NFS 1851 extern int (**nfsv2_vnodeop_p) __P((void *)); 1852 extern int (**spec_nfsv2nodeop_p) __P((void *)); 1853 extern int (**fifo_nfsv2nodeop_p) __P((void *)); 1854 1855 /* vnode is NOT VOP_LOCKed: some vnode types _never_ lock */ 1856 if (vp->v_op == nfsv2_vnodeop_p || 1857 vp->v_op == spec_nfsv2nodeop_p) { 1858 is_ok_anyway = TRUE; 1859 } 1860 if (vp->v_op == fifo_nfsv2nodeop_p) { 1861 is_ok_anyway = TRUE; 1862 } 1863 #endif /* NFS */ 1864 if (!is_ok_anyway) 1865 panic("uvm_vnp_uncache: vnode not locked!"); 1866 } 1867 #endif /* DEBUG */ 1868 1869 /* 1870 * now drop our reference to the vnode. if we have the sole 1871 * reference to the vnode then this will cause it to die [as we 1872 * just cleared the persist flag]. we have to unlock the vnode 1873 * while we are doing this as it may trigger I/O. 1874 * 1875 * XXX: it might be possible for uvn to get reclaimed while we are 1876 * unlocked causing us to return TRUE when we should not. we ignore 1877 * this as a false-positive return value doesn't hurt us. 1878 */ 1879 VOP_UNLOCK(vp, 0); 1880 uvn_detach(&uvn->u_obj); 1881 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 1882 1883 /* 1884 * and return... 1885 */ 1886 1887 return(TRUE); 1888 } 1889 1890 /* 1891 * uvm_vnp_setsize: grow or shrink a vnode uvn 1892 * 1893 * grow => just update size value 1894 * shrink => toss un-needed pages 1895 * 1896 * => we assume that the caller has a reference of some sort to the 1897 * vnode in question so that it will not be yanked out from under 1898 * us. 1899 * 1900 * called from: 1901 * => truncate fns (ext2fs_truncate, ffs_truncate, detrunc[msdos]) 1902 * => "write" fns (ext2fs_write, WRITE [ufs/ufs], msdosfs_write, nfs_write) 1903 * => ffs_balloc [XXX: why? doesn't WRITE handle?] 1904 * => NFS: nfs_loadattrcache, nfs_getattrcache, nfs_setattr 1905 * => union fs: union_newsize 1906 */ 1907 1908 void 1909 uvm_vnp_setsize(vp, newsize) 1910 struct vnode *vp; 1911 voff_t newsize; 1912 { 1913 struct uvm_vnode *uvn = &vp->v_uvm; 1914 1915 /* 1916 * lock uvn and check for valid object, and if valid: do it! 1917 */ 1918 simple_lock(&uvn->u_obj.vmobjlock); 1919 if (uvn->u_flags & UVM_VNODE_VALID) { 1920 1921 /* 1922 * now check if the size has changed: if we shrink we had better 1923 * toss some pages... 1924 */ 1925 1926 if (uvn->u_size > newsize) { 1927 (void)uvn_flush(&uvn->u_obj, newsize, 1928 uvn->u_size, PGO_FREE); 1929 } 1930 uvn->u_size = newsize; 1931 } 1932 simple_unlock(&uvn->u_obj.vmobjlock); 1933 1934 /* 1935 * done 1936 */ 1937 return; 1938 } 1939 1940 /* 1941 * uvm_vnp_sync: flush all dirty VM pages back to their backing vnodes. 1942 * 1943 * => called from sys_sync with no VM structures locked 1944 * => only one process can do a sync at a time (because the uvn 1945 * structure only has one queue for sync'ing). we ensure this 1946 * by holding the uvn_sync_lock while the sync is in progress. 1947 * other processes attempting a sync will sleep on this lock 1948 * until we are done. 1949 */ 1950 1951 void 1952 uvm_vnp_sync(mp) 1953 struct mount *mp; 1954 { 1955 struct uvm_vnode *uvn; 1956 struct vnode *vp; 1957 boolean_t got_lock; 1958 1959 /* 1960 * step 1: ensure we are only ones using the uvn_sync_q by locking 1961 * our lock... 1962 */ 1963 lockmgr(&uvn_sync_lock, LK_EXCLUSIVE, (void *)0); 1964 1965 /* 1966 * step 2: build up a simpleq of uvns of interest based on the 1967 * write list. we gain a reference to uvns of interest. must 1968 * be careful about locking uvn's since we will be holding uvn_wl_lock 1969 * in the body of the loop. 1970 */ 1971 SIMPLEQ_INIT(&uvn_sync_q); 1972 simple_lock(&uvn_wl_lock); 1973 for (uvn = uvn_wlist.lh_first ; uvn != NULL ; 1974 uvn = uvn->u_wlist.le_next) { 1975 1976 vp = (struct vnode *) uvn; 1977 if (mp && vp->v_mount != mp) 1978 continue; 1979 1980 /* attempt to gain reference */ 1981 while ((got_lock = simple_lock_try(&uvn->u_obj.vmobjlock)) == 1982 FALSE && 1983 (uvn->u_flags & UVM_VNODE_BLOCKED) == 0) 1984 /* spin */ ; 1985 1986 /* 1987 * we will exit the loop if either if the following are true: 1988 * - we got the lock [always true if NCPU == 1] 1989 * - we failed to get the lock but noticed the vnode was 1990 * "blocked" -- in this case the vnode must be a dying 1991 * vnode, and since dying vnodes are in the process of 1992 * being flushed out, we can safely skip this one 1993 * 1994 * we want to skip over the vnode if we did not get the lock, 1995 * or if the vnode is already dying (due to the above logic). 1996 * 1997 * note that uvn must already be valid because we found it on 1998 * the wlist (this also means it can't be ALOCK'd). 1999 */ 2000 if (!got_lock || (uvn->u_flags & UVM_VNODE_BLOCKED) != 0) { 2001 if (got_lock) 2002 simple_unlock(&uvn->u_obj.vmobjlock); 2003 continue; /* skip it */ 2004 } 2005 2006 /* 2007 * gain reference. watch out for persisting uvns (need to 2008 * regain vnode REF). 2009 */ 2010 if (uvn->u_obj.uo_refs == 0) 2011 VREF(vp); 2012 uvn->u_obj.uo_refs++; 2013 simple_unlock(&uvn->u_obj.vmobjlock); 2014 2015 /* 2016 * got it! 2017 */ 2018 SIMPLEQ_INSERT_HEAD(&uvn_sync_q, uvn, u_syncq); 2019 } 2020 simple_unlock(&uvn_wl_lock); 2021 2022 /* 2023 * step 3: we now have a list of uvn's that may need cleaning. 2024 * we are holding the uvn_sync_lock, but have dropped the uvn_wl_lock 2025 * (so we can now safely lock uvn's again). 2026 */ 2027 2028 for (uvn = uvn_sync_q.sqh_first ; uvn ; uvn = uvn->u_syncq.sqe_next) { 2029 simple_lock(&uvn->u_obj.vmobjlock); 2030 #ifdef DEBUG 2031 if (uvn->u_flags & UVM_VNODE_DYING) { 2032 printf("uvm_vnp_sync: dying vnode on sync list\n"); 2033 } 2034 #endif 2035 uvn_flush(&uvn->u_obj, 0, 0, 2036 PGO_CLEANIT|PGO_ALLPAGES|PGO_DOACTCLUST); 2037 2038 /* 2039 * if we have the only reference and we just cleaned the uvn, 2040 * then we can pull it out of the UVM_VNODE_WRITEABLE state 2041 * thus allowing us to avoid thinking about flushing it again 2042 * on later sync ops. 2043 */ 2044 if (uvn->u_obj.uo_refs == 1 && 2045 (uvn->u_flags & UVM_VNODE_WRITEABLE)) { 2046 LIST_REMOVE(uvn, u_wlist); 2047 uvn->u_flags &= ~UVM_VNODE_WRITEABLE; 2048 } 2049 2050 simple_unlock(&uvn->u_obj.vmobjlock); 2051 2052 /* now drop our reference to the uvn */ 2053 uvn_detach(&uvn->u_obj); 2054 } 2055 2056 /* 2057 * done! release sync lock 2058 */ 2059 lockmgr(&uvn_sync_lock, LK_RELEASE, (void *)0); 2060 } 2061