1 /* $OpenBSD: uvm_vnode.c,v 1.20 2001/09/11 20:05:26 miod Exp $ */ 2 /* $NetBSD: uvm_vnode.c,v 1.33 2000/05/19 03:45:05 thorpej Exp $ */ 3 4 /* 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * Copyright (c) 1991, 1993 7 * The Regents of the University of California. 8 * Copyright (c) 1990 University of Utah. 9 * 10 * All rights reserved. 11 * 12 * This code is derived from software contributed to Berkeley by 13 * the Systems Programming Group of the University of Utah Computer 14 * Science Department. 15 * 16 * Redistribution and use in source and binary forms, with or without 17 * modification, are permitted provided that the following conditions 18 * are met: 19 * 1. Redistributions of source code must retain the above copyright 20 * notice, this list of conditions and the following disclaimer. 21 * 2. Redistributions in binary form must reproduce the above copyright 22 * notice, this list of conditions and the following disclaimer in the 23 * documentation and/or other materials provided with the distribution. 24 * 3. All advertising materials mentioning features or use of this software 25 * must display the following acknowledgement: 26 * This product includes software developed by Charles D. Cranor, 27 * Washington University, the University of California, Berkeley and 28 * its contributors. 29 * 4. Neither the name of the University nor the names of its contributors 30 * may be used to endorse or promote products derived from this software 31 * without specific prior written permission. 32 * 33 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 34 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 35 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 36 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 37 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 38 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 39 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 40 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 41 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 42 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 43 * SUCH DAMAGE. 44 * 45 * @(#)vnode_pager.c 8.8 (Berkeley) 2/13/94 46 * from: Id: uvm_vnode.c,v 1.1.2.26 1998/02/02 20:38:07 chuck Exp 47 */ 48 49 /* 50 * uvm_vnode.c: the vnode pager. 51 */ 52 53 #include <sys/param.h> 54 #include <sys/systm.h> 55 #include <sys/proc.h> 56 #include <sys/malloc.h> 57 #include <sys/vnode.h> 58 #include <sys/disklabel.h> 59 #include <sys/ioctl.h> 60 #include <sys/fcntl.h> 61 #include <sys/conf.h> 62 63 #include <miscfs/specfs/specdev.h> 64 65 #include <vm/vm.h> 66 #include <vm/vm_page.h> 67 68 #include <uvm/uvm.h> 69 #include <uvm/uvm_vnode.h> 70 71 /* 72 * private global data structure 73 * 74 * we keep a list of writeable active vnode-backed VM objects for sync op. 75 * we keep a simpleq of vnodes that are currently being sync'd. 76 */ 77 78 LIST_HEAD(uvn_list_struct, uvm_vnode); 79 static struct uvn_list_struct uvn_wlist; /* writeable uvns */ 80 static simple_lock_data_t uvn_wl_lock; /* locks uvn_wlist */ 81 82 SIMPLEQ_HEAD(uvn_sq_struct, uvm_vnode); 83 static struct uvn_sq_struct uvn_sync_q; /* sync'ing uvns */ 84 lock_data_t uvn_sync_lock; /* locks sync operation */ 85 86 /* 87 * functions 88 */ 89 90 static int uvn_asyncget __P((struct uvm_object *, voff_t, 91 int)); 92 static void uvn_cluster __P((struct uvm_object *, voff_t, 93 voff_t *, voff_t *)); 94 static void uvn_detach __P((struct uvm_object *)); 95 static boolean_t uvn_flush __P((struct uvm_object *, voff_t, 96 voff_t, int)); 97 static int uvn_get __P((struct uvm_object *, voff_t, 98 vm_page_t *, int *, int, 99 vm_prot_t, int, int)); 100 static void uvn_init __P((void)); 101 static int uvn_io __P((struct uvm_vnode *, vm_page_t *, 102 int, int, int)); 103 static int uvn_put __P((struct uvm_object *, vm_page_t *, 104 int, boolean_t)); 105 static void uvn_reference __P((struct uvm_object *)); 106 static boolean_t uvn_releasepg __P((struct vm_page *, 107 struct vm_page **)); 108 109 /* 110 * master pager structure 111 */ 112 113 struct uvm_pagerops uvm_vnodeops = { 114 uvn_init, 115 uvn_reference, 116 uvn_detach, 117 NULL, /* no specialized fault routine required */ 118 uvn_flush, 119 uvn_get, 120 uvn_asyncget, 121 uvn_put, 122 uvn_cluster, 123 uvm_mk_pcluster, /* use generic version of this: see uvm_pager.c */ 124 NULL, /* AIO-DONE function (not until we have asyncio) */ 125 uvn_releasepg, 126 }; 127 128 /* 129 * the ops! 130 */ 131 132 /* 133 * uvn_init 134 * 135 * init pager private data structures. 136 */ 137 138 static void 139 uvn_init() 140 { 141 142 LIST_INIT(&uvn_wlist); 143 simple_lock_init(&uvn_wl_lock); 144 /* note: uvn_sync_q init'd in uvm_vnp_sync() */ 145 lockinit(&uvn_sync_lock, PVM, "uvnsync", 0, 0); 146 } 147 148 /* 149 * uvn_attach 150 * 151 * attach a vnode structure to a VM object. if the vnode is already 152 * attached, then just bump the reference count by one and return the 153 * VM object. if not already attached, attach and return the new VM obj. 154 * the "accessprot" tells the max access the attaching thread wants to 155 * our pages. 156 * 157 * => caller must _not_ already be holding the lock on the uvm_object. 158 * => in fact, nothing should be locked so that we can sleep here. 159 * => note that uvm_object is first thing in vnode structure, so their 160 * pointers are equiv. 161 */ 162 163 struct uvm_object * 164 uvn_attach(arg, accessprot) 165 void *arg; 166 vm_prot_t accessprot; 167 { 168 struct vnode *vp = arg; 169 struct uvm_vnode *uvn = &vp->v_uvm; 170 struct vattr vattr; 171 int oldflags, result; 172 struct partinfo pi; 173 u_quad_t used_vnode_size; 174 UVMHIST_FUNC("uvn_attach"); UVMHIST_CALLED(maphist); 175 176 UVMHIST_LOG(maphist, "(vn=0x%x)", arg,0,0,0); 177 178 used_vnode_size = (u_quad_t)0; /* XXX gcc -Wuninitialized */ 179 180 /* 181 * first get a lock on the uvn. 182 */ 183 simple_lock(&uvn->u_obj.vmobjlock); 184 while (uvn->u_flags & UVM_VNODE_BLOCKED) { 185 printf("uvn_attach: blocked at 0x%p flags 0x%x\n", 186 uvn, uvn->u_flags); 187 uvn->u_flags |= UVM_VNODE_WANTED; 188 UVMHIST_LOG(maphist, " SLEEPING on blocked vn",0,0,0,0); 189 UVM_UNLOCK_AND_WAIT(uvn, &uvn->u_obj.vmobjlock, FALSE, 190 "uvn_attach", 0); 191 simple_lock(&uvn->u_obj.vmobjlock); 192 UVMHIST_LOG(maphist," WOKE UP",0,0,0,0); 193 } 194 195 /* 196 * if we're mapping a BLK device, make sure it is a disk. 197 */ 198 if (vp->v_type == VBLK && bdevsw[major(vp->v_rdev)].d_type != D_DISK) { 199 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock */ 200 UVMHIST_LOG(maphist,"<- done (VBLK not D_DISK!)", 0,0,0,0); 201 return(NULL); 202 } 203 204 /* 205 * now we have lock and uvn must not be in a blocked state. 206 * first check to see if it is already active, in which case 207 * we can bump the reference count, check to see if we need to 208 * add it to the writeable list, and then return. 209 */ 210 if (uvn->u_flags & UVM_VNODE_VALID) { /* already active? */ 211 212 /* regain VREF if we were persisting */ 213 if (uvn->u_obj.uo_refs == 0) { 214 VREF(vp); 215 UVMHIST_LOG(maphist," VREF (reclaim persisting vnode)", 216 0,0,0,0); 217 } 218 uvn->u_obj.uo_refs++; /* bump uvn ref! */ 219 220 /* check for new writeable uvn */ 221 if ((accessprot & VM_PROT_WRITE) != 0 && 222 (uvn->u_flags & UVM_VNODE_WRITEABLE) == 0) { 223 simple_lock(&uvn_wl_lock); 224 LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist); 225 simple_unlock(&uvn_wl_lock); 226 /* we are now on wlist! */ 227 uvn->u_flags |= UVM_VNODE_WRITEABLE; 228 } 229 230 /* unlock and return */ 231 simple_unlock(&uvn->u_obj.vmobjlock); 232 UVMHIST_LOG(maphist,"<- done, refcnt=%d", uvn->u_obj.uo_refs, 233 0, 0, 0); 234 return (&uvn->u_obj); 235 } 236 237 /* 238 * need to call VOP_GETATTR() to get the attributes, but that could 239 * block (due to I/O), so we want to unlock the object before calling. 240 * however, we want to keep anyone else from playing with the object 241 * while it is unlocked. to do this we set UVM_VNODE_ALOCK which 242 * prevents anyone from attaching to the vnode until we are done with 243 * it. 244 */ 245 uvn->u_flags = UVM_VNODE_ALOCK; 246 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock in case we sleep */ 247 /* XXX: curproc? */ 248 249 if (vp->v_type == VBLK) { 250 /* 251 * We could implement this as a specfs getattr call, but: 252 * 253 * (1) VOP_GETATTR() would get the file system 254 * vnode operation, not the specfs operation. 255 * 256 * (2) All we want is the size, anyhow. 257 */ 258 result = (*bdevsw[major(vp->v_rdev)].d_ioctl)(vp->v_rdev, 259 DIOCGPART, (caddr_t)&pi, FREAD, curproc); 260 if (result == 0) { 261 /* XXX should remember blocksize */ 262 used_vnode_size = (u_quad_t)pi.disklab->d_secsize * 263 (u_quad_t)pi.part->p_size; 264 } 265 } else { 266 result = VOP_GETATTR(vp, &vattr, curproc->p_ucred, curproc); 267 if (result == 0) 268 used_vnode_size = vattr.va_size; 269 } 270 271 /* relock object */ 272 simple_lock(&uvn->u_obj.vmobjlock); 273 274 if (result != 0) { 275 if (uvn->u_flags & UVM_VNODE_WANTED) 276 wakeup(uvn); 277 uvn->u_flags = 0; 278 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock */ 279 UVMHIST_LOG(maphist,"<- done (VOP_GETATTR FAILED!)", 0,0,0,0); 280 return(NULL); 281 } 282 283 /* 284 * make sure that the newsize fits within a vaddr_t 285 * XXX: need to revise addressing data types 286 */ 287 #ifdef DEBUG 288 if (vp->v_type == VBLK) 289 printf("used_vnode_size = %llu\n", (long long)used_vnode_size); 290 #endif 291 292 /* 293 * now set up the uvn. 294 */ 295 uvn->u_obj.pgops = &uvm_vnodeops; 296 TAILQ_INIT(&uvn->u_obj.memq); 297 uvn->u_obj.uo_npages = 0; 298 uvn->u_obj.uo_refs = 1; /* just us... */ 299 oldflags = uvn->u_flags; 300 uvn->u_flags = UVM_VNODE_VALID|UVM_VNODE_CANPERSIST; 301 uvn->u_nio = 0; 302 uvn->u_size = used_vnode_size; 303 304 /* if write access, we need to add it to the wlist */ 305 if (accessprot & VM_PROT_WRITE) { 306 simple_lock(&uvn_wl_lock); 307 LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist); 308 simple_unlock(&uvn_wl_lock); 309 uvn->u_flags |= UVM_VNODE_WRITEABLE; /* we are on wlist! */ 310 } 311 312 /* 313 * add a reference to the vnode. this reference will stay as long 314 * as there is a valid mapping of the vnode. dropped when the 315 * reference count goes to zero [and we either free or persist]. 316 */ 317 VREF(vp); 318 simple_unlock(&uvn->u_obj.vmobjlock); 319 if (oldflags & UVM_VNODE_WANTED) 320 wakeup(uvn); 321 322 UVMHIST_LOG(maphist,"<- done/VREF, ret 0x%x", &uvn->u_obj,0,0,0); 323 return(&uvn->u_obj); 324 } 325 326 327 /* 328 * uvn_reference 329 * 330 * duplicate a reference to a VM object. Note that the reference 331 * count must already be at least one (the passed in reference) so 332 * there is no chance of the uvn being killed or locked out here. 333 * 334 * => caller must call with object unlocked. 335 * => caller must be using the same accessprot as was used at attach time 336 */ 337 338 339 static void 340 uvn_reference(uobj) 341 struct uvm_object *uobj; 342 { 343 #ifdef DEBUG 344 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 345 #endif 346 UVMHIST_FUNC("uvn_reference"); UVMHIST_CALLED(maphist); 347 348 simple_lock(&uobj->vmobjlock); 349 #ifdef DEBUG 350 if ((uvn->u_flags & UVM_VNODE_VALID) == 0) { 351 printf("uvn_reference: ref=%d, flags=0x%x\n", uvn->u_flags, 352 uobj->uo_refs); 353 panic("uvn_reference: invalid state"); 354 } 355 #endif 356 uobj->uo_refs++; 357 UVMHIST_LOG(maphist, "<- done (uobj=0x%x, ref = %d)", 358 uobj, uobj->uo_refs,0,0); 359 simple_unlock(&uobj->vmobjlock); 360 } 361 362 /* 363 * uvn_detach 364 * 365 * remove a reference to a VM object. 366 * 367 * => caller must call with object unlocked and map locked. 368 * => this starts the detach process, but doesn't have to finish it 369 * (async i/o could still be pending). 370 */ 371 static void 372 uvn_detach(uobj) 373 struct uvm_object *uobj; 374 { 375 struct uvm_vnode *uvn; 376 struct vnode *vp; 377 int oldflags; 378 UVMHIST_FUNC("uvn_detach"); UVMHIST_CALLED(maphist); 379 380 simple_lock(&uobj->vmobjlock); 381 382 UVMHIST_LOG(maphist," (uobj=0x%x) ref=%d", uobj,uobj->uo_refs,0,0); 383 uobj->uo_refs--; /* drop ref! */ 384 if (uobj->uo_refs) { /* still more refs */ 385 simple_unlock(&uobj->vmobjlock); 386 UVMHIST_LOG(maphist, "<- done (rc>0)", 0,0,0,0); 387 return; 388 } 389 390 /* 391 * get other pointers ... 392 */ 393 394 uvn = (struct uvm_vnode *) uobj; 395 vp = (struct vnode *) uobj; 396 397 /* 398 * clear VTEXT flag now that there are no mappings left (VTEXT is used 399 * to keep an active text file from being overwritten). 400 */ 401 vp->v_flag &= ~VTEXT; 402 403 /* 404 * we just dropped the last reference to the uvn. see if we can 405 * let it "stick around". 406 */ 407 408 if (uvn->u_flags & UVM_VNODE_CANPERSIST) { 409 /* won't block */ 410 uvn_flush(uobj, 0, 0, PGO_DEACTIVATE|PGO_ALLPAGES); 411 simple_unlock(&uobj->vmobjlock); 412 vrele(vp); /* drop vnode reference */ 413 UVMHIST_LOG(maphist,"<- done/vrele! (persist)", 0,0,0,0); 414 return; 415 } 416 417 /* 418 * its a goner! 419 */ 420 421 UVMHIST_LOG(maphist," its a goner (flushing)!", 0,0,0,0); 422 423 uvn->u_flags |= UVM_VNODE_DYING; 424 425 /* 426 * even though we may unlock in flush, no one can gain a reference 427 * to us until we clear the "dying" flag [because it blocks 428 * attaches]. we will not do that until after we've disposed of all 429 * the pages with uvn_flush(). note that before the flush the only 430 * pages that could be marked PG_BUSY are ones that are in async 431 * pageout by the daemon. (there can't be any pending "get"'s 432 * because there are no references to the object). 433 */ 434 435 (void) uvn_flush(uobj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES); 436 437 UVMHIST_LOG(maphist," its a goner (done flush)!", 0,0,0,0); 438 439 /* 440 * given the structure of this pager, the above flush request will 441 * create the following state: all the pages that were in the object 442 * have either been free'd or they are marked PG_BUSY|PG_RELEASED. 443 * the PG_BUSY bit was set either by us or the daemon for async I/O. 444 * in either case, if we have pages left we can't kill the object 445 * yet because i/o is pending. in this case we set the "relkill" 446 * flag which will cause pgo_releasepg to kill the object once all 447 * the I/O's are done [pgo_releasepg will be called from the aiodone 448 * routine or from the page daemon]. 449 */ 450 451 if (uobj->uo_npages) { /* I/O pending. iodone will free */ 452 #ifdef DEBUG 453 /* 454 * XXXCDC: very unlikely to happen until we have async i/o 455 * so print a little info message in case it does. 456 */ 457 printf("uvn_detach: vn %p has pages left after flush - " 458 "relkill mode\n", uobj); 459 #endif 460 uvn->u_flags |= UVM_VNODE_RELKILL; 461 simple_unlock(&uobj->vmobjlock); 462 UVMHIST_LOG(maphist,"<- done! (releasepg will kill obj)", 0, 0, 463 0, 0); 464 return; 465 } 466 467 /* 468 * kill object now. note that we can't be on the sync q because 469 * all references are gone. 470 */ 471 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 472 simple_lock(&uvn_wl_lock); /* protect uvn_wlist */ 473 LIST_REMOVE(uvn, u_wlist); 474 simple_unlock(&uvn_wl_lock); 475 } 476 #ifdef DIAGNOSTIC 477 if (uobj->memq.tqh_first != NULL) 478 panic("uvn_deref: vnode VM object still has pages afer " 479 "syncio/free flush"); 480 #endif 481 oldflags = uvn->u_flags; 482 uvn->u_flags = 0; 483 simple_unlock(&uobj->vmobjlock); 484 485 /* wake up any sleepers */ 486 if (oldflags & UVM_VNODE_WANTED) 487 wakeup(uvn); 488 489 /* 490 * drop our reference to the vnode. 491 */ 492 vrele(vp); 493 UVMHIST_LOG(maphist,"<- done (vrele) final", 0,0,0,0); 494 495 return; 496 } 497 498 /* 499 * uvm_vnp_terminate: external hook to clear out a vnode's VM 500 * 501 * called in two cases: 502 * [1] when a persisting vnode vm object (i.e. one with a zero reference 503 * count) needs to be freed so that a vnode can be reused. this 504 * happens under "getnewvnode" in vfs_subr.c. if the vnode from 505 * the free list is still attached (i.e. not VBAD) then vgone is 506 * called. as part of the vgone trace this should get called to 507 * free the vm object. this is the common case. 508 * [2] when a filesystem is being unmounted by force (MNT_FORCE, 509 * "umount -f") the vgone() function is called on active vnodes 510 * on the mounted file systems to kill their data (the vnodes become 511 * "dead" ones [see src/sys/miscfs/deadfs/...]). that results in a 512 * call here (even if the uvn is still in use -- i.e. has a non-zero 513 * reference count). this case happens at "umount -f" and during a 514 * "reboot/halt" operation. 515 * 516 * => the caller must XLOCK and VOP_LOCK the vnode before calling us 517 * [protects us from getting a vnode that is already in the DYING 518 * state...] 519 * => unlike uvn_detach, this function must not return until all the 520 * uvn's pages are disposed of. 521 * => in case [2] the uvn is still alive after this call, but all I/O 522 * ops will fail (due to the backing vnode now being "dead"). this 523 * will prob. kill any process using the uvn due to pgo_get failing. 524 */ 525 526 void 527 uvm_vnp_terminate(vp) 528 struct vnode *vp; 529 { 530 struct uvm_vnode *uvn = &vp->v_uvm; 531 int oldflags; 532 UVMHIST_FUNC("uvm_vnp_terminate"); UVMHIST_CALLED(maphist); 533 534 /* 535 * lock object and check if it is valid 536 */ 537 simple_lock(&uvn->u_obj.vmobjlock); 538 UVMHIST_LOG(maphist, " vp=0x%x, ref=%d, flag=0x%x", vp, 539 uvn->u_obj.uo_refs, uvn->u_flags, 0); 540 if ((uvn->u_flags & UVM_VNODE_VALID) == 0) { 541 simple_unlock(&uvn->u_obj.vmobjlock); 542 UVMHIST_LOG(maphist, "<- done (not active)", 0, 0, 0, 0); 543 return; 544 } 545 546 /* 547 * must be a valid uvn that is not already dying (because XLOCK 548 * protects us from that). the uvn can't in the ALOCK state 549 * because it is valid, and uvn's that are in the ALOCK state haven't 550 * been marked valid yet. 551 */ 552 553 #ifdef DEBUG 554 /* 555 * debug check: are we yanking the vnode out from under our uvn? 556 */ 557 if (uvn->u_obj.uo_refs) { 558 printf("uvm_vnp_terminate(%p): terminating active vnode " 559 "(refs=%d)\n", uvn, uvn->u_obj.uo_refs); 560 } 561 #endif 562 563 /* 564 * it is possible that the uvn was detached and is in the relkill 565 * state [i.e. waiting for async i/o to finish so that releasepg can 566 * kill object]. we take over the vnode now and cancel the relkill. 567 * we want to know when the i/o is done so we can recycle right 568 * away. note that a uvn can only be in the RELKILL state if it 569 * has a zero reference count. 570 */ 571 572 if (uvn->u_flags & UVM_VNODE_RELKILL) 573 uvn->u_flags &= ~UVM_VNODE_RELKILL; /* cancel RELKILL */ 574 575 /* 576 * block the uvn by setting the dying flag, and then flush the 577 * pages. (note that flush may unlock object while doing I/O, but 578 * it will re-lock it before it returns control here). 579 * 580 * also, note that we tell I/O that we are already VOP_LOCK'd so 581 * that uvn_io doesn't attempt to VOP_LOCK again. 582 * 583 * XXXCDC: setting VNISLOCKED on an active uvn which is being terminated 584 * due to a forceful unmount might not be a good idea. maybe we 585 * need a way to pass in this info to uvn_flush through a 586 * pager-defined PGO_ constant [currently there are none]. 587 */ 588 uvn->u_flags |= UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED; 589 590 (void) uvn_flush(&uvn->u_obj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES); 591 592 /* 593 * as we just did a flush we expect all the pages to be gone or in 594 * the process of going. sleep to wait for the rest to go [via iosync]. 595 */ 596 597 while (uvn->u_obj.uo_npages) { 598 #ifdef DEBUG 599 struct vm_page *pp; 600 for (pp = uvn->u_obj.memq.tqh_first ; pp != NULL ; 601 pp = pp->listq.tqe_next) { 602 if ((pp->flags & PG_BUSY) == 0) 603 panic("uvm_vnp_terminate: detected unbusy pg"); 604 } 605 if (uvn->u_nio == 0) 606 panic("uvm_vnp_terminate: no I/O to wait for?"); 607 printf("uvm_vnp_terminate: waiting for I/O to fin.\n"); 608 /* 609 * XXXCDC: this is unlikely to happen without async i/o so we 610 * put a printf in just to keep an eye on it. 611 */ 612 #endif 613 uvn->u_flags |= UVM_VNODE_IOSYNC; 614 UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock, FALSE, 615 "uvn_term",0); 616 simple_lock(&uvn->u_obj.vmobjlock); 617 } 618 619 /* 620 * done. now we free the uvn if its reference count is zero 621 * (true if we are zapping a persisting uvn). however, if we are 622 * terminating a uvn with active mappings we let it live ... future 623 * calls down to the vnode layer will fail. 624 */ 625 626 oldflags = uvn->u_flags; 627 if (uvn->u_obj.uo_refs) { 628 629 /* 630 * uvn must live on it is dead-vnode state until all references 631 * are gone. restore flags. clear CANPERSIST state. 632 */ 633 634 uvn->u_flags &= ~(UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED| 635 UVM_VNODE_WANTED|UVM_VNODE_CANPERSIST); 636 637 } else { 638 639 /* 640 * free the uvn now. note that the VREF reference is already 641 * gone [it is dropped when we enter the persist state]. 642 */ 643 if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED) 644 panic("uvm_vnp_terminate: io sync wanted bit set"); 645 646 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 647 simple_lock(&uvn_wl_lock); 648 LIST_REMOVE(uvn, u_wlist); 649 simple_unlock(&uvn_wl_lock); 650 } 651 uvn->u_flags = 0; /* uvn is history, clear all bits */ 652 } 653 654 if (oldflags & UVM_VNODE_WANTED) 655 wakeup(uvn); /* object lock still held */ 656 657 simple_unlock(&uvn->u_obj.vmobjlock); 658 UVMHIST_LOG(maphist, "<- done", 0, 0, 0, 0); 659 660 } 661 662 /* 663 * uvn_releasepg: handled a released page in a uvn 664 * 665 * => "pg" is a PG_BUSY [caller owns it], PG_RELEASED page that we need 666 * to dispose of. 667 * => caller must handled PG_WANTED case 668 * => called with page's object locked, pageq's unlocked 669 * => returns TRUE if page's object is still alive, FALSE if we 670 * killed the page's object. if we return TRUE, then we 671 * return with the object locked. 672 * => if (nextpgp != NULL) => we return pageq.tqe_next here, and return 673 * with the page queues locked [for pagedaemon] 674 * => if (nextpgp == NULL) => we return with page queues unlocked [normal case] 675 * => we kill the uvn if it is not referenced and we are suppose to 676 * kill it ("relkill"). 677 */ 678 679 boolean_t 680 uvn_releasepg(pg, nextpgp) 681 struct vm_page *pg; 682 struct vm_page **nextpgp; /* OUT */ 683 { 684 struct uvm_vnode *uvn = (struct uvm_vnode *) pg->uobject; 685 #ifdef DIAGNOSTIC 686 if ((pg->flags & PG_RELEASED) == 0) 687 panic("uvn_releasepg: page not released!"); 688 #endif 689 690 /* 691 * dispose of the page [caller handles PG_WANTED] 692 */ 693 pmap_page_protect(pg, VM_PROT_NONE); 694 uvm_lock_pageq(); 695 if (nextpgp) 696 *nextpgp = pg->pageq.tqe_next; /* next page for daemon */ 697 uvm_pagefree(pg); 698 if (!nextpgp) 699 uvm_unlock_pageq(); 700 701 /* 702 * now see if we need to kill the object 703 */ 704 if (uvn->u_flags & UVM_VNODE_RELKILL) { 705 if (uvn->u_obj.uo_refs) 706 panic("uvn_releasepg: kill flag set on referenced " 707 "object!"); 708 if (uvn->u_obj.uo_npages == 0) { 709 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 710 simple_lock(&uvn_wl_lock); 711 LIST_REMOVE(uvn, u_wlist); 712 simple_unlock(&uvn_wl_lock); 713 } 714 #ifdef DIAGNOSTIC 715 if (uvn->u_obj.memq.tqh_first) 716 panic("uvn_releasepg: pages in object with npages == 0"); 717 #endif 718 if (uvn->u_flags & UVM_VNODE_WANTED) 719 /* still holding object lock */ 720 wakeup(uvn); 721 722 uvn->u_flags = 0; /* DEAD! */ 723 simple_unlock(&uvn->u_obj.vmobjlock); 724 return (FALSE); 725 } 726 } 727 return (TRUE); 728 } 729 730 /* 731 * NOTE: currently we have to use VOP_READ/VOP_WRITE because they go 732 * through the buffer cache and allow I/O in any size. These VOPs use 733 * synchronous i/o. [vs. VOP_STRATEGY which can be async, but doesn't 734 * go through the buffer cache or allow I/O sizes larger than a 735 * block]. we will eventually want to change this. 736 * 737 * issues to consider: 738 * uvm provides the uvm_aiodesc structure for async i/o management. 739 * there are two tailq's in the uvm. structure... one for pending async 740 * i/o and one for "done" async i/o. to do an async i/o one puts 741 * an aiodesc on the "pending" list (protected by splbio()), starts the 742 * i/o and returns VM_PAGER_PEND. when the i/o is done, we expect 743 * some sort of "i/o done" function to be called (at splbio(), interrupt 744 * time). this function should remove the aiodesc from the pending list 745 * and place it on the "done" list and wakeup the daemon. the daemon 746 * will run at normal spl() and will remove all items from the "done" 747 * list and call the "aiodone" hook for each done request (see uvm_pager.c). 748 * [in the old vm code, this was done by calling the "put" routine with 749 * null arguments which made the code harder to read and understand because 750 * you had one function ("put") doing two things.] 751 * 752 * so the current pager needs: 753 * int uvn_aiodone(struct uvm_aiodesc *) 754 * 755 * => return KERN_SUCCESS (aio finished, free it). otherwise requeue for 756 * later collection. 757 * => called with pageq's locked by the daemon. 758 * 759 * general outline: 760 * - "try" to lock object. if fail, just return (will try again later) 761 * - drop "u_nio" (this req is done!) 762 * - if (object->iosync && u_naio == 0) { wakeup &uvn->u_naio } 763 * - get "page" structures (atop?). 764 * - handle "wanted" pages 765 * - handle "released" pages [using pgo_releasepg] 766 * >>> pgo_releasepg may kill the object 767 * dont forget to look at "object" wanted flag in all cases. 768 */ 769 770 771 /* 772 * uvn_flush: flush pages out of a uvm object. 773 * 774 * => object should be locked by caller. we may _unlock_ the object 775 * if (and only if) we need to clean a page (PGO_CLEANIT). 776 * we return with the object locked. 777 * => if PGO_CLEANIT is set, we may block (due to I/O). thus, a caller 778 * might want to unlock higher level resources (e.g. vm_map) 779 * before calling flush. 780 * => if PGO_CLEANIT is not set, then we will neither unlock the object 781 * or block. 782 * => if PGO_ALLPAGE is set, then all pages in the object are valid targets 783 * for flushing. 784 * => NOTE: we rely on the fact that the object's memq is a TAILQ and 785 * that new pages are inserted on the tail end of the list. thus, 786 * we can make a complete pass through the object in one go by starting 787 * at the head and working towards the tail (new pages are put in 788 * front of us). 789 * => NOTE: we are allowed to lock the page queues, so the caller 790 * must not be holding the lock on them [e.g. pagedaemon had 791 * better not call us with the queues locked] 792 * => we return TRUE unless we encountered some sort of I/O error 793 * 794 * comment on "cleaning" object and PG_BUSY pages: 795 * this routine is holding the lock on the object. the only time 796 * that it can run into a PG_BUSY page that it does not own is if 797 * some other process has started I/O on the page (e.g. either 798 * a pagein, or a pageout). if the PG_BUSY page is being paged 799 * in, then it can not be dirty (!PG_CLEAN) because no one has 800 * had a chance to modify it yet. if the PG_BUSY page is being 801 * paged out then it means that someone else has already started 802 * cleaning the page for us (how nice!). in this case, if we 803 * have syncio specified, then after we make our pass through the 804 * object we need to wait for the other PG_BUSY pages to clear 805 * off (i.e. we need to do an iosync). also note that once a 806 * page is PG_BUSY it must stay in its object until it is un-busyed. 807 * 808 * note on page traversal: 809 * we can traverse the pages in an object either by going down the 810 * linked list in "uobj->memq", or we can go over the address range 811 * by page doing hash table lookups for each address. depending 812 * on how many pages are in the object it may be cheaper to do one 813 * or the other. we set "by_list" to true if we are using memq. 814 * if the cost of a hash lookup was equal to the cost of the list 815 * traversal we could compare the number of pages in the start->stop 816 * range to the total number of pages in the object. however, it 817 * seems that a hash table lookup is more expensive than the linked 818 * list traversal, so we multiply the number of pages in the 819 * start->stop range by a penalty which we define below. 820 */ 821 822 #define UVN_HASH_PENALTY 4 /* XXX: a guess */ 823 824 static boolean_t 825 uvn_flush(uobj, start, stop, flags) 826 struct uvm_object *uobj; 827 voff_t start, stop; 828 int flags; 829 { 830 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 831 struct vm_page *pp, *ppnext, *ptmp; 832 struct vm_page *pps[MAXBSIZE >> PAGE_SHIFT], **ppsp; 833 int npages, result, lcv; 834 boolean_t retval, need_iosync, by_list, needs_clean, all; 835 voff_t curoff; 836 u_short pp_version; 837 UVMHIST_FUNC("uvn_flush"); UVMHIST_CALLED(maphist); 838 839 curoff = 0; /* XXX: shut up gcc */ 840 /* 841 * get init vals and determine how we are going to traverse object 842 */ 843 844 need_iosync = FALSE; 845 retval = TRUE; /* return value */ 846 if (flags & PGO_ALLPAGES) { 847 all = TRUE; 848 by_list = TRUE; /* always go by the list */ 849 } else { 850 start = trunc_page(start); 851 stop = round_page(stop); 852 #ifdef DEBUG 853 if (stop > round_page(uvn->u_size)) 854 printf("uvn_flush: strange, got an out of range " 855 "flush (fixed)\n"); 856 #endif 857 all = FALSE; 858 by_list = (uobj->uo_npages <= 859 ((stop - start) >> PAGE_SHIFT) * UVN_HASH_PENALTY); 860 } 861 862 UVMHIST_LOG(maphist, 863 " flush start=0x%x, stop=0x%x, by_list=%d, flags=0x%x", 864 start, stop, by_list, flags); 865 866 /* 867 * PG_CLEANCHK: this bit is used by the pgo_mk_pcluster function as 868 * a _hint_ as to how up to date the PG_CLEAN bit is. if the hint 869 * is wrong it will only prevent us from clustering... it won't break 870 * anything. we clear all PG_CLEANCHK bits here, and pgo_mk_pcluster 871 * will set them as it syncs PG_CLEAN. This is only an issue if we 872 * are looking at non-inactive pages (because inactive page's PG_CLEAN 873 * bit is always up to date since there are no mappings). 874 * [borrowed PG_CLEANCHK idea from FreeBSD VM] 875 */ 876 877 if ((flags & PGO_CLEANIT) != 0 && 878 uobj->pgops->pgo_mk_pcluster != NULL) { 879 if (by_list) { 880 for (pp = uobj->memq.tqh_first ; pp != NULL ; 881 pp = pp->listq.tqe_next) { 882 if (!all && 883 (pp->offset < start || pp->offset >= stop)) 884 continue; 885 pp->flags &= ~PG_CLEANCHK; 886 } 887 888 } else { /* by hash */ 889 for (curoff = start ; curoff < stop; 890 curoff += PAGE_SIZE) { 891 pp = uvm_pagelookup(uobj, curoff); 892 if (pp) 893 pp->flags &= ~PG_CLEANCHK; 894 } 895 } 896 } 897 898 /* 899 * now do it. note: we must update ppnext in body of loop or we 900 * will get stuck. we need to use ppnext because we may free "pp" 901 * before doing the next loop. 902 */ 903 904 if (by_list) { 905 pp = uobj->memq.tqh_first; 906 } else { 907 curoff = start; 908 pp = uvm_pagelookup(uobj, curoff); 909 } 910 911 ppnext = NULL; /* XXX: shut up gcc */ 912 ppsp = NULL; /* XXX: shut up gcc */ 913 uvm_lock_pageq(); /* page queues locked */ 914 915 /* locked: both page queues and uobj */ 916 for ( ; (by_list && pp != NULL) || 917 (!by_list && curoff < stop) ; pp = ppnext) { 918 919 if (by_list) { 920 921 /* 922 * range check 923 */ 924 925 if (!all && 926 (pp->offset < start || pp->offset >= stop)) { 927 ppnext = pp->listq.tqe_next; 928 continue; 929 } 930 931 } else { 932 933 /* 934 * null check 935 */ 936 937 curoff += PAGE_SIZE; 938 if (pp == NULL) { 939 if (curoff < stop) 940 ppnext = uvm_pagelookup(uobj, curoff); 941 continue; 942 } 943 944 } 945 946 /* 947 * handle case where we do not need to clean page (either 948 * because we are not clean or because page is not dirty or 949 * is busy): 950 * 951 * NOTE: we are allowed to deactivate a non-wired active 952 * PG_BUSY page, but once a PG_BUSY page is on the inactive 953 * queue it must stay put until it is !PG_BUSY (so as not to 954 * confuse pagedaemon). 955 */ 956 957 if ((flags & PGO_CLEANIT) == 0 || (pp->flags & PG_BUSY) != 0) { 958 needs_clean = FALSE; 959 if ((pp->flags & PG_BUSY) != 0 && 960 (flags & (PGO_CLEANIT|PGO_SYNCIO)) == 961 (PGO_CLEANIT|PGO_SYNCIO)) 962 need_iosync = TRUE; 963 } else { 964 /* 965 * freeing: nuke all mappings so we can sync 966 * PG_CLEAN bit with no race 967 */ 968 if ((pp->flags & PG_CLEAN) != 0 && 969 (flags & PGO_FREE) != 0 && 970 (pp->pqflags & PQ_ACTIVE) != 0) 971 pmap_page_protect(pp, VM_PROT_NONE); 972 if ((pp->flags & PG_CLEAN) != 0 && 973 pmap_is_modified(pp)) 974 pp->flags &= ~(PG_CLEAN); 975 pp->flags |= PG_CLEANCHK; /* update "hint" */ 976 977 needs_clean = ((pp->flags & PG_CLEAN) == 0); 978 } 979 980 /* 981 * if we don't need a clean... load ppnext and dispose of pp 982 */ 983 if (!needs_clean) { 984 /* load ppnext */ 985 if (by_list) 986 ppnext = pp->listq.tqe_next; 987 else { 988 if (curoff < stop) 989 ppnext = uvm_pagelookup(uobj, curoff); 990 } 991 992 /* now dispose of pp */ 993 if (flags & PGO_DEACTIVATE) { 994 if ((pp->pqflags & PQ_INACTIVE) == 0 && 995 pp->wire_count == 0) { 996 pmap_page_protect(pp, VM_PROT_NONE); 997 uvm_pagedeactivate(pp); 998 } 999 1000 } else if (flags & PGO_FREE) { 1001 if (pp->flags & PG_BUSY) { 1002 /* release busy pages */ 1003 pp->flags |= PG_RELEASED; 1004 } else { 1005 pmap_page_protect(pp, VM_PROT_NONE); 1006 /* removed page from object */ 1007 uvm_pagefree(pp); 1008 } 1009 } 1010 /* ppnext is valid so we can continue... */ 1011 continue; 1012 } 1013 1014 /* 1015 * pp points to a page in the locked object that we are 1016 * working on. if it is !PG_CLEAN,!PG_BUSY and we asked 1017 * for cleaning (PGO_CLEANIT). we clean it now. 1018 * 1019 * let uvm_pager_put attempted a clustered page out. 1020 * note: locked: uobj and page queues. 1021 */ 1022 1023 pp->flags |= PG_BUSY; /* we 'own' page now */ 1024 UVM_PAGE_OWN(pp, "uvn_flush"); 1025 pmap_page_protect(pp, VM_PROT_READ); 1026 pp_version = pp->version; 1027 ReTry: 1028 ppsp = pps; 1029 npages = sizeof(pps) / sizeof(struct vm_page *); 1030 1031 /* locked: page queues, uobj */ 1032 result = uvm_pager_put(uobj, pp, &ppsp, &npages, 1033 flags | PGO_DOACTCLUST, start, stop); 1034 /* unlocked: page queues, uobj */ 1035 1036 /* 1037 * at this point nothing is locked. if we did an async I/O 1038 * it is remotely possible for the async i/o to complete and 1039 * the page "pp" be freed or what not before we get a chance 1040 * to relock the object. in order to detect this, we have 1041 * saved the version number of the page in "pp_version". 1042 */ 1043 1044 /* relock! */ 1045 simple_lock(&uobj->vmobjlock); 1046 uvm_lock_pageq(); 1047 1048 /* 1049 * VM_PAGER_AGAIN: given the structure of this pager, this 1050 * can only happen when we are doing async I/O and can't 1051 * map the pages into kernel memory (pager_map) due to lack 1052 * of vm space. if this happens we drop back to sync I/O. 1053 */ 1054 1055 if (result == VM_PAGER_AGAIN) { 1056 /* 1057 * it is unlikely, but page could have been released 1058 * while we had the object lock dropped. we ignore 1059 * this now and retry the I/O. we will detect and 1060 * handle the released page after the syncio I/O 1061 * completes. 1062 */ 1063 #ifdef DIAGNOSTIC 1064 if (flags & PGO_SYNCIO) 1065 panic("uvn_flush: PGO_SYNCIO return 'try again' error (impossible)"); 1066 #endif 1067 flags |= PGO_SYNCIO; 1068 goto ReTry; 1069 } 1070 1071 /* 1072 * the cleaning operation is now done. finish up. note that 1073 * on error (!OK, !PEND) uvm_pager_put drops the cluster for us. 1074 * if success (OK, PEND) then uvm_pager_put returns the cluster 1075 * to us in ppsp/npages. 1076 */ 1077 1078 /* 1079 * for pending async i/o if we are not deactivating/freeing 1080 * we can move on to the next page. 1081 */ 1082 1083 if (result == VM_PAGER_PEND) { 1084 1085 if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) { 1086 /* 1087 * no per-page ops: refresh ppnext and continue 1088 */ 1089 if (by_list) { 1090 if (pp->version == pp_version) 1091 ppnext = pp->listq.tqe_next; 1092 else 1093 /* reset */ 1094 ppnext = uobj->memq.tqh_first; 1095 } else { 1096 if (curoff < stop) 1097 ppnext = uvm_pagelookup(uobj, 1098 curoff); 1099 } 1100 continue; 1101 } 1102 1103 /* need to do anything here? */ 1104 } 1105 1106 /* 1107 * need to look at each page of the I/O operation. we defer 1108 * processing "pp" until the last trip through this "for" loop 1109 * so that we can load "ppnext" for the main loop after we 1110 * play with the cluster pages [thus the "npages + 1" in the 1111 * loop below]. 1112 */ 1113 1114 for (lcv = 0 ; lcv < npages + 1 ; lcv++) { 1115 1116 /* 1117 * handle ppnext for outside loop, and saving pp 1118 * until the end. 1119 */ 1120 if (lcv < npages) { 1121 if (ppsp[lcv] == pp) 1122 continue; /* skip pp until the end */ 1123 ptmp = ppsp[lcv]; 1124 } else { 1125 ptmp = pp; 1126 1127 /* set up next page for outer loop */ 1128 if (by_list) { 1129 if (pp->version == pp_version) 1130 ppnext = pp->listq.tqe_next; 1131 else 1132 /* reset */ 1133 ppnext = uobj->memq.tqh_first; 1134 } else { 1135 if (curoff < stop) 1136 ppnext = uvm_pagelookup(uobj, curoff); 1137 } 1138 } 1139 1140 /* 1141 * verify the page didn't get moved while obj was 1142 * unlocked 1143 */ 1144 if (result == VM_PAGER_PEND && ptmp->uobject != uobj) 1145 continue; 1146 1147 /* 1148 * unbusy the page if I/O is done. note that for 1149 * pending I/O it is possible that the I/O op 1150 * finished before we relocked the object (in 1151 * which case the page is no longer busy). 1152 */ 1153 1154 if (result != VM_PAGER_PEND) { 1155 if (ptmp->flags & PG_WANTED) 1156 /* still holding object lock */ 1157 wakeup(ptmp); 1158 1159 ptmp->flags &= ~(PG_WANTED|PG_BUSY); 1160 UVM_PAGE_OWN(ptmp, NULL); 1161 if (ptmp->flags & PG_RELEASED) { 1162 1163 /* pgo_releasepg wants this */ 1164 uvm_unlock_pageq(); 1165 if (!uvn_releasepg(ptmp, NULL)) 1166 return (TRUE); 1167 1168 uvm_lock_pageq(); /* relock */ 1169 continue; /* next page */ 1170 1171 } else { 1172 ptmp->flags |= (PG_CLEAN|PG_CLEANCHK); 1173 if ((flags & PGO_FREE) == 0) 1174 pmap_clear_modify(ptmp); 1175 } 1176 } 1177 1178 /* 1179 * dispose of page 1180 */ 1181 1182 if (flags & PGO_DEACTIVATE) { 1183 if ((pp->pqflags & PQ_INACTIVE) == 0 && 1184 pp->wire_count == 0) { 1185 pmap_page_protect(ptmp, VM_PROT_NONE); 1186 uvm_pagedeactivate(ptmp); 1187 } 1188 1189 } else if (flags & PGO_FREE) { 1190 if (result == VM_PAGER_PEND) { 1191 if ((ptmp->flags & PG_BUSY) != 0) 1192 /* signal for i/o done */ 1193 ptmp->flags |= PG_RELEASED; 1194 } else { 1195 if (result != VM_PAGER_OK) { 1196 printf("uvn_flush: obj=%p, " 1197 "offset=0x%llx. error " 1198 "during pageout.\n", 1199 pp->uobject, 1200 (long long)pp->offset); 1201 printf("uvn_flush: WARNING: " 1202 "changes to page may be " 1203 "lost!\n"); 1204 retval = FALSE; 1205 } 1206 pmap_page_protect(ptmp, VM_PROT_NONE); 1207 uvm_pagefree(ptmp); 1208 } 1209 } 1210 1211 } /* end of "lcv" for loop */ 1212 1213 } /* end of "pp" for loop */ 1214 1215 /* 1216 * done with pagequeues: unlock 1217 */ 1218 uvm_unlock_pageq(); 1219 1220 /* 1221 * now wait for all I/O if required. 1222 */ 1223 if (need_iosync) { 1224 1225 UVMHIST_LOG(maphist," <<DOING IOSYNC>>",0,0,0,0); 1226 while (uvn->u_nio != 0) { 1227 uvn->u_flags |= UVM_VNODE_IOSYNC; 1228 UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock, 1229 FALSE, "uvn_flush",0); 1230 simple_lock(&uvn->u_obj.vmobjlock); 1231 } 1232 if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED) 1233 wakeup(&uvn->u_flags); 1234 uvn->u_flags &= ~(UVM_VNODE_IOSYNC|UVM_VNODE_IOSYNCWANTED); 1235 } 1236 1237 /* return, with object locked! */ 1238 UVMHIST_LOG(maphist,"<- done (retval=0x%x)",retval,0,0,0); 1239 return(retval); 1240 } 1241 1242 /* 1243 * uvn_cluster 1244 * 1245 * we are about to do I/O in an object at offset. this function is called 1246 * to establish a range of offsets around "offset" in which we can cluster 1247 * I/O. 1248 * 1249 * - currently doesn't matter if obj locked or not. 1250 */ 1251 1252 static void 1253 uvn_cluster(uobj, offset, loffset, hoffset) 1254 struct uvm_object *uobj; 1255 voff_t offset; 1256 voff_t *loffset, *hoffset; /* OUT */ 1257 { 1258 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 1259 *loffset = offset; 1260 1261 if (*loffset >= uvn->u_size) 1262 panic("uvn_cluster: offset out of range"); 1263 1264 /* 1265 * XXX: old pager claims we could use VOP_BMAP to get maxcontig value. 1266 */ 1267 *hoffset = *loffset + MAXBSIZE; 1268 if (*hoffset > round_page(uvn->u_size)) /* past end? */ 1269 *hoffset = round_page(uvn->u_size); 1270 1271 return; 1272 } 1273 1274 /* 1275 * uvn_put: flush page data to backing store. 1276 * 1277 * => prefer map unlocked (not required) 1278 * => object must be locked! we will _unlock_ it before starting I/O. 1279 * => flags: PGO_SYNCIO -- use sync. I/O 1280 * => note: caller must set PG_CLEAN and pmap_clear_modify (if needed) 1281 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync. 1282 * [thus we never do async i/o! see iodone comment] 1283 */ 1284 1285 static int 1286 uvn_put(uobj, pps, npages, flags) 1287 struct uvm_object *uobj; 1288 struct vm_page **pps; 1289 int npages, flags; 1290 { 1291 int retval; 1292 1293 /* note: object locked */ 1294 retval = uvn_io((struct uvm_vnode*)uobj, pps, npages, flags, UIO_WRITE); 1295 /* note: object unlocked */ 1296 1297 return(retval); 1298 } 1299 1300 1301 /* 1302 * uvn_get: get pages (synchronously) from backing store 1303 * 1304 * => prefer map unlocked (not required) 1305 * => object must be locked! we will _unlock_ it before starting any I/O. 1306 * => flags: PGO_ALLPAGES: get all of the pages 1307 * PGO_LOCKED: fault data structures are locked 1308 * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx] 1309 * => NOTE: caller must check for released pages!! 1310 */ 1311 1312 static int 1313 uvn_get(uobj, offset, pps, npagesp, centeridx, access_type, advice, flags) 1314 struct uvm_object *uobj; 1315 voff_t offset; 1316 struct vm_page **pps; /* IN/OUT */ 1317 int *npagesp; /* IN (OUT if PGO_LOCKED) */ 1318 int centeridx, advice, flags; 1319 vm_prot_t access_type; 1320 { 1321 voff_t current_offset; 1322 struct vm_page *ptmp; 1323 int lcv, result, gotpages; 1324 boolean_t done; 1325 UVMHIST_FUNC("uvn_get"); UVMHIST_CALLED(maphist); 1326 UVMHIST_LOG(maphist, "flags=%d", flags,0,0,0); 1327 1328 /* 1329 * step 1: handled the case where fault data structures are locked. 1330 */ 1331 1332 if (flags & PGO_LOCKED) { 1333 1334 /* 1335 * gotpages is the current number of pages we've gotten (which 1336 * we pass back up to caller via *npagesp. 1337 */ 1338 1339 gotpages = 0; 1340 1341 /* 1342 * step 1a: get pages that are already resident. only do this 1343 * if the data structures are locked (i.e. the first time 1344 * through). 1345 */ 1346 1347 done = TRUE; /* be optimistic */ 1348 1349 for (lcv = 0, current_offset = offset ; lcv < *npagesp ; 1350 lcv++, current_offset += PAGE_SIZE) { 1351 1352 /* do we care about this page? if not, skip it */ 1353 if (pps[lcv] == PGO_DONTCARE) 1354 continue; 1355 1356 /* lookup page */ 1357 ptmp = uvm_pagelookup(uobj, current_offset); 1358 1359 /* to be useful must get a non-busy, non-released pg */ 1360 if (ptmp == NULL || 1361 (ptmp->flags & (PG_BUSY|PG_RELEASED)) != 0) { 1362 if (lcv == centeridx || (flags & PGO_ALLPAGES) 1363 != 0) 1364 done = FALSE; /* need to do a wait or I/O! */ 1365 continue; 1366 } 1367 1368 /* 1369 * useful page: busy/lock it and plug it in our 1370 * result array 1371 */ 1372 ptmp->flags |= PG_BUSY; /* loan up to caller */ 1373 UVM_PAGE_OWN(ptmp, "uvn_get1"); 1374 pps[lcv] = ptmp; 1375 gotpages++; 1376 1377 } /* "for" lcv loop */ 1378 1379 /* 1380 * XXX: given the "advice", should we consider async read-ahead? 1381 * XXX: fault current does deactive of pages behind us. is 1382 * this good (other callers might now). 1383 */ 1384 /* 1385 * XXX: read-ahead currently handled by buffer cache (bread) 1386 * level. 1387 * XXX: no async i/o available. 1388 * XXX: so we don't do anything now. 1389 */ 1390 1391 /* 1392 * step 1c: now we've either done everything needed or we to 1393 * unlock and do some waiting or I/O. 1394 */ 1395 1396 *npagesp = gotpages; /* let caller know */ 1397 if (done) 1398 return(VM_PAGER_OK); /* bingo! */ 1399 else 1400 /* EEK! Need to unlock and I/O */ 1401 return(VM_PAGER_UNLOCK); 1402 } 1403 1404 /* 1405 * step 2: get non-resident or busy pages. 1406 * object is locked. data structures are unlocked. 1407 * 1408 * XXX: because we can't do async I/O at this level we get things 1409 * page at a time (otherwise we'd chunk). the VOP_READ() will do 1410 * async-read-ahead for us at a lower level. 1411 */ 1412 1413 for (lcv = 0, current_offset = offset ; 1414 lcv < *npagesp ; lcv++, current_offset += PAGE_SIZE) { 1415 1416 /* skip over pages we've already gotten or don't want */ 1417 /* skip over pages we don't _have_ to get */ 1418 if (pps[lcv] != NULL || (lcv != centeridx && 1419 (flags & PGO_ALLPAGES) == 0)) 1420 continue; 1421 1422 /* 1423 * we have yet to locate the current page (pps[lcv]). we first 1424 * look for a page that is already at the current offset. if 1425 * we fine a page, we check to see if it is busy or released. 1426 * if that is the case, then we sleep on the page until it is 1427 * no longer busy or released and repeat the lookup. if the 1428 * page we found is neither busy nor released, then we busy it 1429 * (so we own it) and plug it into pps[lcv]. this breaks the 1430 * following while loop and indicates we are ready to move on 1431 * to the next page in the "lcv" loop above. 1432 * 1433 * if we exit the while loop with pps[lcv] still set to NULL, 1434 * then it means that we allocated a new busy/fake/clean page 1435 * ptmp in the object and we need to do I/O to fill in the data. 1436 */ 1437 1438 while (pps[lcv] == NULL) { /* top of "pps" while loop */ 1439 1440 /* look for a current page */ 1441 ptmp = uvm_pagelookup(uobj, current_offset); 1442 1443 /* nope? allocate one now (if we can) */ 1444 if (ptmp == NULL) { 1445 1446 ptmp = uvm_pagealloc(uobj, current_offset, 1447 NULL, 0); 1448 1449 /* out of RAM? */ 1450 if (ptmp == NULL) { 1451 simple_unlock(&uobj->vmobjlock); 1452 uvm_wait("uvn_getpage"); 1453 simple_lock(&uobj->vmobjlock); 1454 1455 /* goto top of pps while loop */ 1456 continue; 1457 } 1458 1459 /* 1460 * got new page ready for I/O. break pps 1461 * while loop. pps[lcv] is still NULL. 1462 */ 1463 break; 1464 } 1465 1466 /* page is there, see if we need to wait on it */ 1467 if ((ptmp->flags & (PG_BUSY|PG_RELEASED)) != 0) { 1468 ptmp->flags |= PG_WANTED; 1469 UVM_UNLOCK_AND_WAIT(ptmp, 1470 &uobj->vmobjlock, FALSE, "uvn_get",0); 1471 simple_lock(&uobj->vmobjlock); 1472 continue; /* goto top of pps while loop */ 1473 } 1474 1475 /* 1476 * if we get here then the page has become resident 1477 * and unbusy between steps 1 and 2. we busy it 1478 * now (so we own it) and set pps[lcv] (so that we 1479 * exit the while loop). 1480 */ 1481 ptmp->flags |= PG_BUSY; 1482 UVM_PAGE_OWN(ptmp, "uvn_get2"); 1483 pps[lcv] = ptmp; 1484 } 1485 1486 /* 1487 * if we own the a valid page at the correct offset, pps[lcv] 1488 * will point to it. nothing more to do except go to the 1489 * next page. 1490 */ 1491 1492 if (pps[lcv]) 1493 continue; /* next lcv */ 1494 1495 /* 1496 * we have a "fake/busy/clean" page that we just allocated. do 1497 * I/O to fill it with valid data. note that object must be 1498 * locked going into uvn_io, but will be unlocked afterwards. 1499 */ 1500 1501 result = uvn_io((struct uvm_vnode *) uobj, &ptmp, 1, 1502 PGO_SYNCIO, UIO_READ); 1503 1504 /* 1505 * I/O done. object is unlocked (by uvn_io). because we used 1506 * syncio the result can not be PEND or AGAIN. we must relock 1507 * and check for errors. 1508 */ 1509 1510 /* lock object. check for errors. */ 1511 simple_lock(&uobj->vmobjlock); 1512 if (result != VM_PAGER_OK) { 1513 if (ptmp->flags & PG_WANTED) 1514 /* object lock still held */ 1515 wakeup(ptmp); 1516 1517 ptmp->flags &= ~(PG_WANTED|PG_BUSY); 1518 UVM_PAGE_OWN(ptmp, NULL); 1519 uvm_lock_pageq(); 1520 uvm_pagefree(ptmp); 1521 uvm_unlock_pageq(); 1522 simple_unlock(&uobj->vmobjlock); 1523 return(result); 1524 } 1525 1526 /* 1527 * we got the page! clear the fake flag (indicates valid 1528 * data now in page) and plug into our result array. note 1529 * that page is still busy. 1530 * 1531 * it is the callers job to: 1532 * => check if the page is released 1533 * => unbusy the page 1534 * => activate the page 1535 */ 1536 1537 ptmp->flags &= ~PG_FAKE; /* data is valid ... */ 1538 pmap_clear_modify(ptmp); /* ... and clean */ 1539 pps[lcv] = ptmp; 1540 1541 } /* lcv loop */ 1542 1543 /* 1544 * finally, unlock object and return. 1545 */ 1546 1547 simple_unlock(&uobj->vmobjlock); 1548 return (VM_PAGER_OK); 1549 } 1550 1551 /* 1552 * uvn_asyncget: start async I/O to bring pages into ram 1553 * 1554 * => caller must lock object(???XXX: see if this is best) 1555 * => could be called from uvn_get or a madvise() fault-ahead. 1556 * => if it fails, it doesn't matter. 1557 */ 1558 1559 static int 1560 uvn_asyncget(uobj, offset, npages) 1561 struct uvm_object *uobj; 1562 voff_t offset; 1563 int npages; 1564 { 1565 1566 /* 1567 * XXXCDC: we can't do async I/O yet 1568 */ 1569 printf("uvn_asyncget called\n"); 1570 return (KERN_SUCCESS); 1571 } 1572 1573 /* 1574 * uvn_io: do I/O to a vnode 1575 * 1576 * => prefer map unlocked (not required) 1577 * => object must be locked! we will _unlock_ it before starting I/O. 1578 * => flags: PGO_SYNCIO -- use sync. I/O 1579 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync. 1580 * [thus we never do async i/o! see iodone comment] 1581 */ 1582 1583 static int 1584 uvn_io(uvn, pps, npages, flags, rw) 1585 struct uvm_vnode *uvn; 1586 vm_page_t *pps; 1587 int npages, flags, rw; 1588 { 1589 struct vnode *vn; 1590 struct uio uio; 1591 struct iovec iov; 1592 vaddr_t kva; 1593 off_t file_offset; 1594 int waitf, result, mapinflags; 1595 size_t got, wanted; 1596 UVMHIST_FUNC("uvn_io"); UVMHIST_CALLED(maphist); 1597 1598 UVMHIST_LOG(maphist, "rw=%d", rw,0,0,0); 1599 1600 /* 1601 * init values 1602 */ 1603 1604 waitf = (flags & PGO_SYNCIO) ? M_WAITOK : M_NOWAIT; 1605 vn = (struct vnode *) uvn; 1606 file_offset = pps[0]->offset; 1607 1608 /* 1609 * check for sync'ing I/O. 1610 */ 1611 1612 while (uvn->u_flags & UVM_VNODE_IOSYNC) { 1613 if (waitf == M_NOWAIT) { 1614 simple_unlock(&uvn->u_obj.vmobjlock); 1615 UVMHIST_LOG(maphist,"<- try again (iosync)",0,0,0,0); 1616 return(VM_PAGER_AGAIN); 1617 } 1618 uvn->u_flags |= UVM_VNODE_IOSYNCWANTED; 1619 UVM_UNLOCK_AND_WAIT(&uvn->u_flags, &uvn->u_obj.vmobjlock, 1620 FALSE, "uvn_iosync",0); 1621 simple_lock(&uvn->u_obj.vmobjlock); 1622 } 1623 1624 /* 1625 * check size 1626 */ 1627 1628 if (file_offset >= uvn->u_size) { 1629 simple_unlock(&uvn->u_obj.vmobjlock); 1630 UVMHIST_LOG(maphist,"<- BAD (size check)",0,0,0,0); 1631 return(VM_PAGER_BAD); 1632 } 1633 1634 /* 1635 * first try and map the pages in (without waiting) 1636 */ 1637 1638 mapinflags = (rw == UIO_READ) ? 1639 UVMPAGER_MAPIN_READ : UVMPAGER_MAPIN_WRITE; 1640 1641 kva = uvm_pagermapin(pps, npages, NULL, mapinflags); 1642 if (kva == 0 && waitf == M_NOWAIT) { 1643 simple_unlock(&uvn->u_obj.vmobjlock); 1644 UVMHIST_LOG(maphist,"<- mapin failed (try again)",0,0,0,0); 1645 return(VM_PAGER_AGAIN); 1646 } 1647 1648 /* 1649 * ok, now bump u_nio up. at this point we are done with uvn 1650 * and can unlock it. if we still don't have a kva, try again 1651 * (this time with sleep ok). 1652 */ 1653 1654 uvn->u_nio++; /* we have an I/O in progress! */ 1655 simple_unlock(&uvn->u_obj.vmobjlock); 1656 /* NOTE: object now unlocked */ 1657 if (kva == 0) 1658 kva = uvm_pagermapin(pps, npages, NULL, 1659 mapinflags | UVMPAGER_MAPIN_WAITOK); 1660 1661 /* 1662 * ok, mapped in. our pages are PG_BUSY so they are not going to 1663 * get touched (so we can look at "offset" without having to lock 1664 * the object). set up for I/O. 1665 */ 1666 1667 /* 1668 * fill out uio/iov 1669 */ 1670 1671 iov.iov_base = (caddr_t) kva; 1672 wanted = npages << PAGE_SHIFT; 1673 if (file_offset + wanted > uvn->u_size) 1674 wanted = uvn->u_size - file_offset; /* XXX: needed? */ 1675 iov.iov_len = wanted; 1676 uio.uio_iov = &iov; 1677 uio.uio_iovcnt = 1; 1678 uio.uio_offset = file_offset; 1679 uio.uio_segflg = UIO_SYSSPACE; 1680 uio.uio_rw = rw; 1681 uio.uio_resid = wanted; 1682 uio.uio_procp = curproc; 1683 1684 /* 1685 * do the I/O! (XXX: curproc?) 1686 */ 1687 1688 UVMHIST_LOG(maphist, "calling VOP",0,0,0,0); 1689 1690 /* 1691 * This process may already have this vnode locked, if we faulted in 1692 * copyin() or copyout() on a region backed by this vnode 1693 * while doing I/O to the vnode. If this is the case, don't 1694 * panic.. instead, return the error to the user. 1695 * 1696 * XXX this is a stopgap to prevent a panic. 1697 * Ideally, this kind of operation *should* work. 1698 */ 1699 result = 0; 1700 if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0) 1701 result = vn_lock(vn, LK_EXCLUSIVE | LK_RETRY, curproc /*XXX*/); 1702 1703 if (result == 0) { 1704 /* NOTE: vnode now locked! */ 1705 1706 if (rw == UIO_READ) 1707 result = VOP_READ(vn, &uio, 0, curproc->p_ucred); 1708 else 1709 result = VOP_WRITE(vn, &uio, 0, curproc->p_ucred); 1710 1711 if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0) 1712 VOP_UNLOCK(vn, 0, curproc /*XXX*/); 1713 } 1714 1715 /* NOTE: vnode now unlocked (unless vnislocked) */ 1716 1717 UVMHIST_LOG(maphist, "done calling VOP",0,0,0,0); 1718 1719 /* 1720 * result == unix style errno (0 == OK!) 1721 * 1722 * zero out rest of buffer (if needed) 1723 */ 1724 1725 if (result == 0) { 1726 got = wanted - uio.uio_resid; 1727 1728 if (wanted && got == 0) { 1729 result = EIO; /* XXX: error? */ 1730 } else if (got < PAGE_SIZE * npages && rw == UIO_READ) { 1731 memset((void *) (kva + got), 0, 1732 (npages << PAGE_SHIFT) - got); 1733 } 1734 } 1735 1736 /* 1737 * now remove pager mapping 1738 */ 1739 uvm_pagermapout(kva, npages); 1740 1741 /* 1742 * now clean up the object (i.e. drop I/O count) 1743 */ 1744 1745 simple_lock(&uvn->u_obj.vmobjlock); 1746 /* NOTE: object now locked! */ 1747 1748 uvn->u_nio--; /* I/O DONE! */ 1749 if ((uvn->u_flags & UVM_VNODE_IOSYNC) != 0 && uvn->u_nio == 0) { 1750 wakeup(&uvn->u_nio); 1751 } 1752 simple_unlock(&uvn->u_obj.vmobjlock); 1753 /* NOTE: object now unlocked! */ 1754 1755 /* 1756 * done! 1757 */ 1758 1759 UVMHIST_LOG(maphist, "<- done (result %d)", result,0,0,0); 1760 if (result == 0) 1761 return(VM_PAGER_OK); 1762 else 1763 return(VM_PAGER_ERROR); 1764 } 1765 1766 /* 1767 * uvm_vnp_uncache: disable "persisting" in a vnode... when last reference 1768 * is gone we will kill the object (flushing dirty pages back to the vnode 1769 * if needed). 1770 * 1771 * => returns TRUE if there was no uvm_object attached or if there was 1772 * one and we killed it [i.e. if there is no active uvn] 1773 * => called with the vnode VOP_LOCK'd [we will unlock it for I/O, if 1774 * needed] 1775 * 1776 * => XXX: given that we now kill uvn's when a vnode is recycled (without 1777 * having to hold a reference on the vnode) and given a working 1778 * uvm_vnp_sync(), how does that effect the need for this function? 1779 * [XXXCDC: seems like it can die?] 1780 * 1781 * => XXX: this function should DIE once we merge the VM and buffer 1782 * cache. 1783 * 1784 * research shows that this is called in the following places: 1785 * ext2fs_truncate, ffs_truncate, detrunc[msdosfs]: called when vnode 1786 * changes sizes 1787 * ext2fs_write, WRITE [ufs_readwrite], msdosfs_write: called when we 1788 * are written to 1789 * ex2fs_chmod, ufs_chmod: called if VTEXT vnode and the sticky bit 1790 * is off 1791 * ffs_realloccg: when we can't extend the current block and have 1792 * to allocate a new one we call this [XXX: why?] 1793 * nfsrv_rename, rename_files: called when the target filename is there 1794 * and we want to remove it 1795 * nfsrv_remove, sys_unlink: called on file we are removing 1796 * nfsrv_access: if VTEXT and we want WRITE access and we don't uncache 1797 * then return "text busy" 1798 * nfs_open: seems to uncache any file opened with nfs 1799 * vn_writechk: if VTEXT vnode and can't uncache return "text busy" 1800 */ 1801 1802 boolean_t 1803 uvm_vnp_uncache(vp) 1804 struct vnode *vp; 1805 { 1806 struct uvm_vnode *uvn = &vp->v_uvm; 1807 1808 /* 1809 * lock uvn part of the vnode and check to see if we need to do anything 1810 */ 1811 1812 simple_lock(&uvn->u_obj.vmobjlock); 1813 if ((uvn->u_flags & UVM_VNODE_VALID) == 0 || 1814 (uvn->u_flags & UVM_VNODE_BLOCKED) != 0) { 1815 simple_unlock(&uvn->u_obj.vmobjlock); 1816 return(TRUE); 1817 } 1818 1819 /* 1820 * we have a valid, non-blocked uvn. clear persist flag. 1821 * if uvn is currently active we can return now. 1822 */ 1823 1824 uvn->u_flags &= ~UVM_VNODE_CANPERSIST; 1825 if (uvn->u_obj.uo_refs) { 1826 simple_unlock(&uvn->u_obj.vmobjlock); 1827 return(FALSE); 1828 } 1829 1830 /* 1831 * uvn is currently persisting! we have to gain a reference to 1832 * it so that we can call uvn_detach to kill the uvn. 1833 */ 1834 1835 VREF(vp); /* seems ok, even with VOP_LOCK */ 1836 uvn->u_obj.uo_refs++; /* value is now 1 */ 1837 simple_unlock(&uvn->u_obj.vmobjlock); 1838 1839 1840 #ifdef DEBUG 1841 /* 1842 * carry over sanity check from old vnode pager: the vnode should 1843 * be VOP_LOCK'd, and we confirm it here. 1844 */ 1845 if (!VOP_ISLOCKED(vp)) { 1846 boolean_t is_ok_anyway = FALSE; 1847 #if defined(NFSCLIENT) 1848 extern int (**nfsv2_vnodeop_p) __P((void *)); 1849 extern int (**spec_nfsv2nodeop_p) __P((void *)); 1850 extern int (**fifo_nfsv2nodeop_p) __P((void *)); 1851 1852 /* vnode is NOT VOP_LOCKed: some vnode types _never_ lock */ 1853 if (vp->v_op == nfsv2_vnodeop_p || 1854 vp->v_op == spec_nfsv2nodeop_p) { 1855 is_ok_anyway = TRUE; 1856 } 1857 if (vp->v_op == fifo_nfsv2nodeop_p) { 1858 is_ok_anyway = TRUE; 1859 } 1860 #endif /* defined(NFSSERVER) || defined(NFSCLIENT) */ 1861 if (!is_ok_anyway) 1862 panic("uvm_vnp_uncache: vnode not locked!"); 1863 } 1864 #endif /* DEBUG */ 1865 1866 /* 1867 * now drop our reference to the vnode. if we have the sole 1868 * reference to the vnode then this will cause it to die [as we 1869 * just cleared the persist flag]. we have to unlock the vnode 1870 * while we are doing this as it may trigger I/O. 1871 * 1872 * XXX: it might be possible for uvn to get reclaimed while we are 1873 * unlocked causing us to return TRUE when we should not. we ignore 1874 * this as a false-positive return value doesn't hurt us. 1875 */ 1876 VOP_UNLOCK(vp, 0, curproc /*XXX*/); 1877 uvn_detach(&uvn->u_obj); 1878 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, curproc/*XXX*/); 1879 1880 /* 1881 * and return... 1882 */ 1883 1884 return(TRUE); 1885 } 1886 1887 /* 1888 * uvm_vnp_setsize: grow or shrink a vnode uvn 1889 * 1890 * grow => just update size value 1891 * shrink => toss un-needed pages 1892 * 1893 * => we assume that the caller has a reference of some sort to the 1894 * vnode in question so that it will not be yanked out from under 1895 * us. 1896 * 1897 * called from: 1898 * => truncate fns (ext2fs_truncate, ffs_truncate, detrunc[msdos]) 1899 * => "write" fns (ext2fs_write, WRITE [ufs/ufs], msdosfs_write, nfs_write) 1900 * => ffs_balloc [XXX: why? doesn't WRITE handle?] 1901 * => NFS: nfs_loadattrcache, nfs_getattrcache, nfs_setattr 1902 * => union fs: union_newsize 1903 */ 1904 1905 void 1906 uvm_vnp_setsize(vp, newsize) 1907 struct vnode *vp; 1908 voff_t newsize; 1909 { 1910 struct uvm_vnode *uvn = &vp->v_uvm; 1911 1912 /* 1913 * lock uvn and check for valid object, and if valid: do it! 1914 */ 1915 simple_lock(&uvn->u_obj.vmobjlock); 1916 if (uvn->u_flags & UVM_VNODE_VALID) { 1917 1918 /* 1919 * now check if the size has changed: if we shrink we had better 1920 * toss some pages... 1921 */ 1922 1923 if (uvn->u_size > newsize) { 1924 (void)uvn_flush(&uvn->u_obj, newsize, 1925 uvn->u_size, PGO_FREE); 1926 } 1927 uvn->u_size = newsize; 1928 } 1929 simple_unlock(&uvn->u_obj.vmobjlock); 1930 1931 /* 1932 * done 1933 */ 1934 return; 1935 } 1936 1937 /* 1938 * uvm_vnp_sync: flush all dirty VM pages back to their backing vnodes. 1939 * 1940 * => called from sys_sync with no VM structures locked 1941 * => only one process can do a sync at a time (because the uvn 1942 * structure only has one queue for sync'ing). we ensure this 1943 * by holding the uvn_sync_lock while the sync is in progress. 1944 * other processes attempting a sync will sleep on this lock 1945 * until we are done. 1946 */ 1947 1948 void 1949 uvm_vnp_sync(mp) 1950 struct mount *mp; 1951 { 1952 struct uvm_vnode *uvn; 1953 struct vnode *vp; 1954 boolean_t got_lock; 1955 1956 /* 1957 * step 1: ensure we are only ones using the uvn_sync_q by locking 1958 * our lock... 1959 */ 1960 lockmgr(&uvn_sync_lock, LK_EXCLUSIVE, (void *)0, curproc /*XXX*/); 1961 1962 /* 1963 * step 2: build up a simpleq of uvns of interest based on the 1964 * write list. we gain a reference to uvns of interest. must 1965 * be careful about locking uvn's since we will be holding uvn_wl_lock 1966 * in the body of the loop. 1967 */ 1968 SIMPLEQ_INIT(&uvn_sync_q); 1969 simple_lock(&uvn_wl_lock); 1970 for (uvn = uvn_wlist.lh_first ; uvn != NULL ; 1971 uvn = uvn->u_wlist.le_next) { 1972 1973 vp = (struct vnode *) uvn; 1974 if (mp && vp->v_mount != mp) 1975 continue; 1976 1977 /* attempt to gain reference */ 1978 while ((got_lock = simple_lock_try(&uvn->u_obj.vmobjlock)) == 1979 FALSE && 1980 (uvn->u_flags & UVM_VNODE_BLOCKED) == 0) 1981 /* spin */ ; 1982 1983 /* 1984 * we will exit the loop if either if the following are true: 1985 * - we got the lock [always true if NCPU == 1] 1986 * - we failed to get the lock but noticed the vnode was 1987 * "blocked" -- in this case the vnode must be a dying 1988 * vnode, and since dying vnodes are in the process of 1989 * being flushed out, we can safely skip this one 1990 * 1991 * we want to skip over the vnode if we did not get the lock, 1992 * or if the vnode is already dying (due to the above logic). 1993 * 1994 * note that uvn must already be valid because we found it on 1995 * the wlist (this also means it can't be ALOCK'd). 1996 */ 1997 if (!got_lock || (uvn->u_flags & UVM_VNODE_BLOCKED) != 0) { 1998 if (got_lock) 1999 simple_unlock(&uvn->u_obj.vmobjlock); 2000 continue; /* skip it */ 2001 } 2002 2003 /* 2004 * gain reference. watch out for persisting uvns (need to 2005 * regain vnode REF). 2006 */ 2007 if (uvn->u_obj.uo_refs == 0) 2008 VREF(vp); 2009 uvn->u_obj.uo_refs++; 2010 simple_unlock(&uvn->u_obj.vmobjlock); 2011 2012 /* 2013 * got it! 2014 */ 2015 SIMPLEQ_INSERT_HEAD(&uvn_sync_q, uvn, u_syncq); 2016 } 2017 simple_unlock(&uvn_wl_lock); 2018 2019 /* 2020 * step 3: we now have a list of uvn's that may need cleaning. 2021 * we are holding the uvn_sync_lock, but have dropped the uvn_wl_lock 2022 * (so we can now safely lock uvn's again). 2023 */ 2024 2025 for (uvn = uvn_sync_q.sqh_first ; uvn ; uvn = uvn->u_syncq.sqe_next) { 2026 simple_lock(&uvn->u_obj.vmobjlock); 2027 #ifdef DEBUG 2028 if (uvn->u_flags & UVM_VNODE_DYING) { 2029 printf("uvm_vnp_sync: dying vnode on sync list\n"); 2030 } 2031 #endif 2032 uvn_flush(&uvn->u_obj, 0, 0, 2033 PGO_CLEANIT|PGO_ALLPAGES|PGO_DOACTCLUST); 2034 2035 /* 2036 * if we have the only reference and we just cleaned the uvn, 2037 * then we can pull it out of the UVM_VNODE_WRITEABLE state 2038 * thus allowing us to avoid thinking about flushing it again 2039 * on later sync ops. 2040 */ 2041 if (uvn->u_obj.uo_refs == 1 && 2042 (uvn->u_flags & UVM_VNODE_WRITEABLE)) { 2043 LIST_REMOVE(uvn, u_wlist); 2044 uvn->u_flags &= ~UVM_VNODE_WRITEABLE; 2045 } 2046 2047 simple_unlock(&uvn->u_obj.vmobjlock); 2048 2049 /* now drop our reference to the uvn */ 2050 uvn_detach(&uvn->u_obj); 2051 } 2052 2053 /* 2054 * done! release sync lock 2055 */ 2056 lockmgr(&uvn_sync_lock, LK_RELEASE, (void *)0, curproc /*XXX*/); 2057 } 2058