1 /* $OpenBSD: uvm_vnode.c,v 1.18 2001/08/11 10:57:22 art Exp $ */ 2 /* $NetBSD: uvm_vnode.c,v 1.33 2000/05/19 03:45:05 thorpej Exp $ */ 3 4 /* 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * Copyright (c) 1991, 1993 7 * The Regents of the University of California. 8 * Copyright (c) 1990 University of Utah. 9 * 10 * All rights reserved. 11 * 12 * This code is derived from software contributed to Berkeley by 13 * the Systems Programming Group of the University of Utah Computer 14 * Science Department. 15 * 16 * Redistribution and use in source and binary forms, with or without 17 * modification, are permitted provided that the following conditions 18 * are met: 19 * 1. Redistributions of source code must retain the above copyright 20 * notice, this list of conditions and the following disclaimer. 21 * 2. Redistributions in binary form must reproduce the above copyright 22 * notice, this list of conditions and the following disclaimer in the 23 * documentation and/or other materials provided with the distribution. 24 * 3. All advertising materials mentioning features or use of this software 25 * must display the following acknowledgement: 26 * This product includes software developed by Charles D. Cranor, 27 * Washington University, the University of California, Berkeley and 28 * its contributors. 29 * 4. Neither the name of the University nor the names of its contributors 30 * may be used to endorse or promote products derived from this software 31 * without specific prior written permission. 32 * 33 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 34 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 35 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 36 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 37 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 38 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 39 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 40 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 41 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 42 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 43 * SUCH DAMAGE. 44 * 45 * @(#)vnode_pager.c 8.8 (Berkeley) 2/13/94 46 * from: Id: uvm_vnode.c,v 1.1.2.26 1998/02/02 20:38:07 chuck Exp 47 */ 48 49 /* 50 * uvm_vnode.c: the vnode pager. 51 */ 52 53 #include <sys/param.h> 54 #include <sys/systm.h> 55 #include <sys/proc.h> 56 #include <sys/malloc.h> 57 #include <sys/vnode.h> 58 #include <sys/disklabel.h> 59 #include <sys/ioctl.h> 60 #include <sys/fcntl.h> 61 #include <sys/conf.h> 62 63 #include <miscfs/specfs/specdev.h> 64 65 #include <vm/vm.h> 66 #include <vm/vm_page.h> 67 #include <vm/vm_kern.h> 68 69 #include <uvm/uvm.h> 70 #include <uvm/uvm_vnode.h> 71 72 /* 73 * private global data structure 74 * 75 * we keep a list of writeable active vnode-backed VM objects for sync op. 76 * we keep a simpleq of vnodes that are currently being sync'd. 77 */ 78 79 LIST_HEAD(uvn_list_struct, uvm_vnode); 80 static struct uvn_list_struct uvn_wlist; /* writeable uvns */ 81 static simple_lock_data_t uvn_wl_lock; /* locks uvn_wlist */ 82 83 SIMPLEQ_HEAD(uvn_sq_struct, uvm_vnode); 84 static struct uvn_sq_struct uvn_sync_q; /* sync'ing uvns */ 85 lock_data_t uvn_sync_lock; /* locks sync operation */ 86 87 /* 88 * functions 89 */ 90 91 static int uvn_asyncget __P((struct uvm_object *, voff_t, 92 int)); 93 static void uvn_cluster __P((struct uvm_object *, voff_t, 94 voff_t *, voff_t *)); 95 static void uvn_detach __P((struct uvm_object *)); 96 static boolean_t uvn_flush __P((struct uvm_object *, voff_t, 97 voff_t, int)); 98 static int uvn_get __P((struct uvm_object *, voff_t, 99 vm_page_t *, int *, int, 100 vm_prot_t, int, int)); 101 static void uvn_init __P((void)); 102 static int uvn_io __P((struct uvm_vnode *, vm_page_t *, 103 int, int, int)); 104 static int uvn_put __P((struct uvm_object *, vm_page_t *, 105 int, boolean_t)); 106 static void uvn_reference __P((struct uvm_object *)); 107 static boolean_t uvn_releasepg __P((struct vm_page *, 108 struct vm_page **)); 109 110 /* 111 * master pager structure 112 */ 113 114 struct uvm_pagerops uvm_vnodeops = { 115 uvn_init, 116 uvn_reference, 117 uvn_detach, 118 NULL, /* no specialized fault routine required */ 119 uvn_flush, 120 uvn_get, 121 uvn_asyncget, 122 uvn_put, 123 uvn_cluster, 124 uvm_mk_pcluster, /* use generic version of this: see uvm_pager.c */ 125 NULL, /* AIO-DONE function (not until we have asyncio) */ 126 uvn_releasepg, 127 }; 128 129 /* 130 * the ops! 131 */ 132 133 /* 134 * uvn_init 135 * 136 * init pager private data structures. 137 */ 138 139 static void 140 uvn_init() 141 { 142 143 LIST_INIT(&uvn_wlist); 144 simple_lock_init(&uvn_wl_lock); 145 /* note: uvn_sync_q init'd in uvm_vnp_sync() */ 146 lockinit(&uvn_sync_lock, PVM, "uvnsync", 0, 0); 147 } 148 149 /* 150 * uvn_attach 151 * 152 * attach a vnode structure to a VM object. if the vnode is already 153 * attached, then just bump the reference count by one and return the 154 * VM object. if not already attached, attach and return the new VM obj. 155 * the "accessprot" tells the max access the attaching thread wants to 156 * our pages. 157 * 158 * => caller must _not_ already be holding the lock on the uvm_object. 159 * => in fact, nothing should be locked so that we can sleep here. 160 * => note that uvm_object is first thing in vnode structure, so their 161 * pointers are equiv. 162 */ 163 164 struct uvm_object * 165 uvn_attach(arg, accessprot) 166 void *arg; 167 vm_prot_t accessprot; 168 { 169 struct vnode *vp = arg; 170 struct uvm_vnode *uvn = &vp->v_uvm; 171 struct vattr vattr; 172 int oldflags, result; 173 struct partinfo pi; 174 u_quad_t used_vnode_size; 175 UVMHIST_FUNC("uvn_attach"); UVMHIST_CALLED(maphist); 176 177 UVMHIST_LOG(maphist, "(vn=0x%x)", arg,0,0,0); 178 179 used_vnode_size = (u_quad_t)0; /* XXX gcc -Wuninitialized */ 180 181 /* 182 * first get a lock on the uvn. 183 */ 184 simple_lock(&uvn->u_obj.vmobjlock); 185 while (uvn->u_flags & UVM_VNODE_BLOCKED) { 186 printf("uvn_attach: blocked at 0x%p flags 0x%x\n", 187 uvn, uvn->u_flags); 188 uvn->u_flags |= UVM_VNODE_WANTED; 189 UVMHIST_LOG(maphist, " SLEEPING on blocked vn",0,0,0,0); 190 UVM_UNLOCK_AND_WAIT(uvn, &uvn->u_obj.vmobjlock, FALSE, 191 "uvn_attach", 0); 192 simple_lock(&uvn->u_obj.vmobjlock); 193 UVMHIST_LOG(maphist," WOKE UP",0,0,0,0); 194 } 195 196 /* 197 * if we're mapping a BLK device, make sure it is a disk. 198 */ 199 if (vp->v_type == VBLK && bdevsw[major(vp->v_rdev)].d_type != D_DISK) { 200 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock */ 201 UVMHIST_LOG(maphist,"<- done (VBLK not D_DISK!)", 0,0,0,0); 202 return(NULL); 203 } 204 205 /* 206 * now we have lock and uvn must not be in a blocked state. 207 * first check to see if it is already active, in which case 208 * we can bump the reference count, check to see if we need to 209 * add it to the writeable list, and then return. 210 */ 211 if (uvn->u_flags & UVM_VNODE_VALID) { /* already active? */ 212 213 /* regain VREF if we were persisting */ 214 if (uvn->u_obj.uo_refs == 0) { 215 VREF(vp); 216 UVMHIST_LOG(maphist," VREF (reclaim persisting vnode)", 217 0,0,0,0); 218 } 219 uvn->u_obj.uo_refs++; /* bump uvn ref! */ 220 221 /* check for new writeable uvn */ 222 if ((accessprot & VM_PROT_WRITE) != 0 && 223 (uvn->u_flags & UVM_VNODE_WRITEABLE) == 0) { 224 simple_lock(&uvn_wl_lock); 225 LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist); 226 simple_unlock(&uvn_wl_lock); 227 /* we are now on wlist! */ 228 uvn->u_flags |= UVM_VNODE_WRITEABLE; 229 } 230 231 /* unlock and return */ 232 simple_unlock(&uvn->u_obj.vmobjlock); 233 UVMHIST_LOG(maphist,"<- done, refcnt=%d", uvn->u_obj.uo_refs, 234 0, 0, 0); 235 return (&uvn->u_obj); 236 } 237 238 /* 239 * need to call VOP_GETATTR() to get the attributes, but that could 240 * block (due to I/O), so we want to unlock the object before calling. 241 * however, we want to keep anyone else from playing with the object 242 * while it is unlocked. to do this we set UVM_VNODE_ALOCK which 243 * prevents anyone from attaching to the vnode until we are done with 244 * it. 245 */ 246 uvn->u_flags = UVM_VNODE_ALOCK; 247 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock in case we sleep */ 248 /* XXX: curproc? */ 249 250 if (vp->v_type == VBLK) { 251 /* 252 * We could implement this as a specfs getattr call, but: 253 * 254 * (1) VOP_GETATTR() would get the file system 255 * vnode operation, not the specfs operation. 256 * 257 * (2) All we want is the size, anyhow. 258 */ 259 result = (*bdevsw[major(vp->v_rdev)].d_ioctl)(vp->v_rdev, 260 DIOCGPART, (caddr_t)&pi, FREAD, curproc); 261 if (result == 0) { 262 /* XXX should remember blocksize */ 263 used_vnode_size = (u_quad_t)pi.disklab->d_secsize * 264 (u_quad_t)pi.part->p_size; 265 } 266 } else { 267 result = VOP_GETATTR(vp, &vattr, curproc->p_ucred, curproc); 268 if (result == 0) 269 used_vnode_size = vattr.va_size; 270 } 271 272 /* relock object */ 273 simple_lock(&uvn->u_obj.vmobjlock); 274 275 if (result != 0) { 276 if (uvn->u_flags & UVM_VNODE_WANTED) 277 wakeup(uvn); 278 uvn->u_flags = 0; 279 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock */ 280 UVMHIST_LOG(maphist,"<- done (VOP_GETATTR FAILED!)", 0,0,0,0); 281 return(NULL); 282 } 283 284 /* 285 * make sure that the newsize fits within a vaddr_t 286 * XXX: need to revise addressing data types 287 */ 288 #ifdef DEBUG 289 if (vp->v_type == VBLK) 290 printf("used_vnode_size = %qu\n", (long long)used_vnode_size); 291 #endif 292 293 /* 294 * now set up the uvn. 295 */ 296 uvn->u_obj.pgops = &uvm_vnodeops; 297 TAILQ_INIT(&uvn->u_obj.memq); 298 uvn->u_obj.uo_npages = 0; 299 uvn->u_obj.uo_refs = 1; /* just us... */ 300 oldflags = uvn->u_flags; 301 uvn->u_flags = UVM_VNODE_VALID|UVM_VNODE_CANPERSIST; 302 uvn->u_nio = 0; 303 uvn->u_size = used_vnode_size; 304 305 /* if write access, we need to add it to the wlist */ 306 if (accessprot & VM_PROT_WRITE) { 307 simple_lock(&uvn_wl_lock); 308 LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist); 309 simple_unlock(&uvn_wl_lock); 310 uvn->u_flags |= UVM_VNODE_WRITEABLE; /* we are on wlist! */ 311 } 312 313 /* 314 * add a reference to the vnode. this reference will stay as long 315 * as there is a valid mapping of the vnode. dropped when the 316 * reference count goes to zero [and we either free or persist]. 317 */ 318 VREF(vp); 319 simple_unlock(&uvn->u_obj.vmobjlock); 320 if (oldflags & UVM_VNODE_WANTED) 321 wakeup(uvn); 322 323 UVMHIST_LOG(maphist,"<- done/VREF, ret 0x%x", &uvn->u_obj,0,0,0); 324 return(&uvn->u_obj); 325 } 326 327 328 /* 329 * uvn_reference 330 * 331 * duplicate a reference to a VM object. Note that the reference 332 * count must already be at least one (the passed in reference) so 333 * there is no chance of the uvn being killed or locked out here. 334 * 335 * => caller must call with object unlocked. 336 * => caller must be using the same accessprot as was used at attach time 337 */ 338 339 340 static void 341 uvn_reference(uobj) 342 struct uvm_object *uobj; 343 { 344 #ifdef DEBUG 345 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 346 #endif 347 UVMHIST_FUNC("uvn_reference"); UVMHIST_CALLED(maphist); 348 349 simple_lock(&uobj->vmobjlock); 350 #ifdef DEBUG 351 if ((uvn->u_flags & UVM_VNODE_VALID) == 0) { 352 printf("uvn_reference: ref=%d, flags=0x%x\n", uvn->u_flags, 353 uobj->uo_refs); 354 panic("uvn_reference: invalid state"); 355 } 356 #endif 357 uobj->uo_refs++; 358 UVMHIST_LOG(maphist, "<- done (uobj=0x%x, ref = %d)", 359 uobj, uobj->uo_refs,0,0); 360 simple_unlock(&uobj->vmobjlock); 361 } 362 363 /* 364 * uvn_detach 365 * 366 * remove a reference to a VM object. 367 * 368 * => caller must call with object unlocked and map locked. 369 * => this starts the detach process, but doesn't have to finish it 370 * (async i/o could still be pending). 371 */ 372 static void 373 uvn_detach(uobj) 374 struct uvm_object *uobj; 375 { 376 struct uvm_vnode *uvn; 377 struct vnode *vp; 378 int oldflags; 379 UVMHIST_FUNC("uvn_detach"); UVMHIST_CALLED(maphist); 380 381 simple_lock(&uobj->vmobjlock); 382 383 UVMHIST_LOG(maphist," (uobj=0x%x) ref=%d", uobj,uobj->uo_refs,0,0); 384 uobj->uo_refs--; /* drop ref! */ 385 if (uobj->uo_refs) { /* still more refs */ 386 simple_unlock(&uobj->vmobjlock); 387 UVMHIST_LOG(maphist, "<- done (rc>0)", 0,0,0,0); 388 return; 389 } 390 391 /* 392 * get other pointers ... 393 */ 394 395 uvn = (struct uvm_vnode *) uobj; 396 vp = (struct vnode *) uobj; 397 398 /* 399 * clear VTEXT flag now that there are no mappings left (VTEXT is used 400 * to keep an active text file from being overwritten). 401 */ 402 vp->v_flag &= ~VTEXT; 403 404 /* 405 * we just dropped the last reference to the uvn. see if we can 406 * let it "stick around". 407 */ 408 409 if (uvn->u_flags & UVM_VNODE_CANPERSIST) { 410 /* won't block */ 411 uvn_flush(uobj, 0, 0, PGO_DEACTIVATE|PGO_ALLPAGES); 412 simple_unlock(&uobj->vmobjlock); 413 vrele(vp); /* drop vnode reference */ 414 UVMHIST_LOG(maphist,"<- done/vrele! (persist)", 0,0,0,0); 415 return; 416 } 417 418 /* 419 * its a goner! 420 */ 421 422 UVMHIST_LOG(maphist," its a goner (flushing)!", 0,0,0,0); 423 424 uvn->u_flags |= UVM_VNODE_DYING; 425 426 /* 427 * even though we may unlock in flush, no one can gain a reference 428 * to us until we clear the "dying" flag [because it blocks 429 * attaches]. we will not do that until after we've disposed of all 430 * the pages with uvn_flush(). note that before the flush the only 431 * pages that could be marked PG_BUSY are ones that are in async 432 * pageout by the daemon. (there can't be any pending "get"'s 433 * because there are no references to the object). 434 */ 435 436 (void) uvn_flush(uobj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES); 437 438 UVMHIST_LOG(maphist," its a goner (done flush)!", 0,0,0,0); 439 440 /* 441 * given the structure of this pager, the above flush request will 442 * create the following state: all the pages that were in the object 443 * have either been free'd or they are marked PG_BUSY|PG_RELEASED. 444 * the PG_BUSY bit was set either by us or the daemon for async I/O. 445 * in either case, if we have pages left we can't kill the object 446 * yet because i/o is pending. in this case we set the "relkill" 447 * flag which will cause pgo_releasepg to kill the object once all 448 * the I/O's are done [pgo_releasepg will be called from the aiodone 449 * routine or from the page daemon]. 450 */ 451 452 if (uobj->uo_npages) { /* I/O pending. iodone will free */ 453 #ifdef DEBUG 454 /* 455 * XXXCDC: very unlikely to happen until we have async i/o 456 * so print a little info message in case it does. 457 */ 458 printf("uvn_detach: vn %p has pages left after flush - " 459 "relkill mode\n", uobj); 460 #endif 461 uvn->u_flags |= UVM_VNODE_RELKILL; 462 simple_unlock(&uobj->vmobjlock); 463 UVMHIST_LOG(maphist,"<- done! (releasepg will kill obj)", 0, 0, 464 0, 0); 465 return; 466 } 467 468 /* 469 * kill object now. note that we can't be on the sync q because 470 * all references are gone. 471 */ 472 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 473 simple_lock(&uvn_wl_lock); /* protect uvn_wlist */ 474 LIST_REMOVE(uvn, u_wlist); 475 simple_unlock(&uvn_wl_lock); 476 } 477 #ifdef DIAGNOSTIC 478 if (uobj->memq.tqh_first != NULL) 479 panic("uvn_deref: vnode VM object still has pages afer " 480 "syncio/free flush"); 481 #endif 482 oldflags = uvn->u_flags; 483 uvn->u_flags = 0; 484 simple_unlock(&uobj->vmobjlock); 485 486 /* wake up any sleepers */ 487 if (oldflags & UVM_VNODE_WANTED) 488 wakeup(uvn); 489 490 /* 491 * drop our reference to the vnode. 492 */ 493 vrele(vp); 494 UVMHIST_LOG(maphist,"<- done (vrele) final", 0,0,0,0); 495 496 return; 497 } 498 499 /* 500 * uvm_vnp_terminate: external hook to clear out a vnode's VM 501 * 502 * called in two cases: 503 * [1] when a persisting vnode vm object (i.e. one with a zero reference 504 * count) needs to be freed so that a vnode can be reused. this 505 * happens under "getnewvnode" in vfs_subr.c. if the vnode from 506 * the free list is still attached (i.e. not VBAD) then vgone is 507 * called. as part of the vgone trace this should get called to 508 * free the vm object. this is the common case. 509 * [2] when a filesystem is being unmounted by force (MNT_FORCE, 510 * "umount -f") the vgone() function is called on active vnodes 511 * on the mounted file systems to kill their data (the vnodes become 512 * "dead" ones [see src/sys/miscfs/deadfs/...]). that results in a 513 * call here (even if the uvn is still in use -- i.e. has a non-zero 514 * reference count). this case happens at "umount -f" and during a 515 * "reboot/halt" operation. 516 * 517 * => the caller must XLOCK and VOP_LOCK the vnode before calling us 518 * [protects us from getting a vnode that is already in the DYING 519 * state...] 520 * => unlike uvn_detach, this function must not return until all the 521 * uvn's pages are disposed of. 522 * => in case [2] the uvn is still alive after this call, but all I/O 523 * ops will fail (due to the backing vnode now being "dead"). this 524 * will prob. kill any process using the uvn due to pgo_get failing. 525 */ 526 527 void 528 uvm_vnp_terminate(vp) 529 struct vnode *vp; 530 { 531 struct uvm_vnode *uvn = &vp->v_uvm; 532 int oldflags; 533 UVMHIST_FUNC("uvm_vnp_terminate"); UVMHIST_CALLED(maphist); 534 535 /* 536 * lock object and check if it is valid 537 */ 538 simple_lock(&uvn->u_obj.vmobjlock); 539 UVMHIST_LOG(maphist, " vp=0x%x, ref=%d, flag=0x%x", vp, 540 uvn->u_obj.uo_refs, uvn->u_flags, 0); 541 if ((uvn->u_flags & UVM_VNODE_VALID) == 0) { 542 simple_unlock(&uvn->u_obj.vmobjlock); 543 UVMHIST_LOG(maphist, "<- done (not active)", 0, 0, 0, 0); 544 return; 545 } 546 547 /* 548 * must be a valid uvn that is not already dying (because XLOCK 549 * protects us from that). the uvn can't in the ALOCK state 550 * because it is valid, and uvn's that are in the ALOCK state haven't 551 * been marked valid yet. 552 */ 553 554 #ifdef DEBUG 555 /* 556 * debug check: are we yanking the vnode out from under our uvn? 557 */ 558 if (uvn->u_obj.uo_refs) { 559 printf("uvm_vnp_terminate(%p): terminating active vnode " 560 "(refs=%d)\n", uvn, uvn->u_obj.uo_refs); 561 } 562 #endif 563 564 /* 565 * it is possible that the uvn was detached and is in the relkill 566 * state [i.e. waiting for async i/o to finish so that releasepg can 567 * kill object]. we take over the vnode now and cancel the relkill. 568 * we want to know when the i/o is done so we can recycle right 569 * away. note that a uvn can only be in the RELKILL state if it 570 * has a zero reference count. 571 */ 572 573 if (uvn->u_flags & UVM_VNODE_RELKILL) 574 uvn->u_flags &= ~UVM_VNODE_RELKILL; /* cancel RELKILL */ 575 576 /* 577 * block the uvn by setting the dying flag, and then flush the 578 * pages. (note that flush may unlock object while doing I/O, but 579 * it will re-lock it before it returns control here). 580 * 581 * also, note that we tell I/O that we are already VOP_LOCK'd so 582 * that uvn_io doesn't attempt to VOP_LOCK again. 583 * 584 * XXXCDC: setting VNISLOCKED on an active uvn which is being terminated 585 * due to a forceful unmount might not be a good idea. maybe we 586 * need a way to pass in this info to uvn_flush through a 587 * pager-defined PGO_ constant [currently there are none]. 588 */ 589 uvn->u_flags |= UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED; 590 591 (void) uvn_flush(&uvn->u_obj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES); 592 593 /* 594 * as we just did a flush we expect all the pages to be gone or in 595 * the process of going. sleep to wait for the rest to go [via iosync]. 596 */ 597 598 while (uvn->u_obj.uo_npages) { 599 #ifdef DEBUG 600 struct vm_page *pp; 601 for (pp = uvn->u_obj.memq.tqh_first ; pp != NULL ; 602 pp = pp->listq.tqe_next) { 603 if ((pp->flags & PG_BUSY) == 0) 604 panic("uvm_vnp_terminate: detected unbusy pg"); 605 } 606 if (uvn->u_nio == 0) 607 panic("uvm_vnp_terminate: no I/O to wait for?"); 608 printf("uvm_vnp_terminate: waiting for I/O to fin.\n"); 609 /* 610 * XXXCDC: this is unlikely to happen without async i/o so we 611 * put a printf in just to keep an eye on it. 612 */ 613 #endif 614 uvn->u_flags |= UVM_VNODE_IOSYNC; 615 UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock, FALSE, 616 "uvn_term",0); 617 simple_lock(&uvn->u_obj.vmobjlock); 618 } 619 620 /* 621 * done. now we free the uvn if its reference count is zero 622 * (true if we are zapping a persisting uvn). however, if we are 623 * terminating a uvn with active mappings we let it live ... future 624 * calls down to the vnode layer will fail. 625 */ 626 627 oldflags = uvn->u_flags; 628 if (uvn->u_obj.uo_refs) { 629 630 /* 631 * uvn must live on it is dead-vnode state until all references 632 * are gone. restore flags. clear CANPERSIST state. 633 */ 634 635 uvn->u_flags &= ~(UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED| 636 UVM_VNODE_WANTED|UVM_VNODE_CANPERSIST); 637 638 } else { 639 640 /* 641 * free the uvn now. note that the VREF reference is already 642 * gone [it is dropped when we enter the persist state]. 643 */ 644 if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED) 645 panic("uvm_vnp_terminate: io sync wanted bit set"); 646 647 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 648 simple_lock(&uvn_wl_lock); 649 LIST_REMOVE(uvn, u_wlist); 650 simple_unlock(&uvn_wl_lock); 651 } 652 uvn->u_flags = 0; /* uvn is history, clear all bits */ 653 } 654 655 if (oldflags & UVM_VNODE_WANTED) 656 wakeup(uvn); /* object lock still held */ 657 658 simple_unlock(&uvn->u_obj.vmobjlock); 659 UVMHIST_LOG(maphist, "<- done", 0, 0, 0, 0); 660 661 } 662 663 /* 664 * uvn_releasepg: handled a released page in a uvn 665 * 666 * => "pg" is a PG_BUSY [caller owns it], PG_RELEASED page that we need 667 * to dispose of. 668 * => caller must handled PG_WANTED case 669 * => called with page's object locked, pageq's unlocked 670 * => returns TRUE if page's object is still alive, FALSE if we 671 * killed the page's object. if we return TRUE, then we 672 * return with the object locked. 673 * => if (nextpgp != NULL) => we return pageq.tqe_next here, and return 674 * with the page queues locked [for pagedaemon] 675 * => if (nextpgp == NULL) => we return with page queues unlocked [normal case] 676 * => we kill the uvn if it is not referenced and we are suppose to 677 * kill it ("relkill"). 678 */ 679 680 boolean_t 681 uvn_releasepg(pg, nextpgp) 682 struct vm_page *pg; 683 struct vm_page **nextpgp; /* OUT */ 684 { 685 struct uvm_vnode *uvn = (struct uvm_vnode *) pg->uobject; 686 #ifdef DIAGNOSTIC 687 if ((pg->flags & PG_RELEASED) == 0) 688 panic("uvn_releasepg: page not released!"); 689 #endif 690 691 /* 692 * dispose of the page [caller handles PG_WANTED] 693 */ 694 pmap_page_protect(pg, VM_PROT_NONE); 695 uvm_lock_pageq(); 696 if (nextpgp) 697 *nextpgp = pg->pageq.tqe_next; /* next page for daemon */ 698 uvm_pagefree(pg); 699 if (!nextpgp) 700 uvm_unlock_pageq(); 701 702 /* 703 * now see if we need to kill the object 704 */ 705 if (uvn->u_flags & UVM_VNODE_RELKILL) { 706 if (uvn->u_obj.uo_refs) 707 panic("uvn_releasepg: kill flag set on referenced " 708 "object!"); 709 if (uvn->u_obj.uo_npages == 0) { 710 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 711 simple_lock(&uvn_wl_lock); 712 LIST_REMOVE(uvn, u_wlist); 713 simple_unlock(&uvn_wl_lock); 714 } 715 #ifdef DIAGNOSTIC 716 if (uvn->u_obj.memq.tqh_first) 717 panic("uvn_releasepg: pages in object with npages == 0"); 718 #endif 719 if (uvn->u_flags & UVM_VNODE_WANTED) 720 /* still holding object lock */ 721 wakeup(uvn); 722 723 uvn->u_flags = 0; /* DEAD! */ 724 simple_unlock(&uvn->u_obj.vmobjlock); 725 return (FALSE); 726 } 727 } 728 return (TRUE); 729 } 730 731 /* 732 * NOTE: currently we have to use VOP_READ/VOP_WRITE because they go 733 * through the buffer cache and allow I/O in any size. These VOPs use 734 * synchronous i/o. [vs. VOP_STRATEGY which can be async, but doesn't 735 * go through the buffer cache or allow I/O sizes larger than a 736 * block]. we will eventually want to change this. 737 * 738 * issues to consider: 739 * uvm provides the uvm_aiodesc structure for async i/o management. 740 * there are two tailq's in the uvm. structure... one for pending async 741 * i/o and one for "done" async i/o. to do an async i/o one puts 742 * an aiodesc on the "pending" list (protected by splbio()), starts the 743 * i/o and returns VM_PAGER_PEND. when the i/o is done, we expect 744 * some sort of "i/o done" function to be called (at splbio(), interrupt 745 * time). this function should remove the aiodesc from the pending list 746 * and place it on the "done" list and wakeup the daemon. the daemon 747 * will run at normal spl() and will remove all items from the "done" 748 * list and call the "aiodone" hook for each done request (see uvm_pager.c). 749 * [in the old vm code, this was done by calling the "put" routine with 750 * null arguments which made the code harder to read and understand because 751 * you had one function ("put") doing two things.] 752 * 753 * so the current pager needs: 754 * int uvn_aiodone(struct uvm_aiodesc *) 755 * 756 * => return KERN_SUCCESS (aio finished, free it). otherwise requeue for 757 * later collection. 758 * => called with pageq's locked by the daemon. 759 * 760 * general outline: 761 * - "try" to lock object. if fail, just return (will try again later) 762 * - drop "u_nio" (this req is done!) 763 * - if (object->iosync && u_naio == 0) { wakeup &uvn->u_naio } 764 * - get "page" structures (atop?). 765 * - handle "wanted" pages 766 * - handle "released" pages [using pgo_releasepg] 767 * >>> pgo_releasepg may kill the object 768 * dont forget to look at "object" wanted flag in all cases. 769 */ 770 771 772 /* 773 * uvn_flush: flush pages out of a uvm object. 774 * 775 * => object should be locked by caller. we may _unlock_ the object 776 * if (and only if) we need to clean a page (PGO_CLEANIT). 777 * we return with the object locked. 778 * => if PGO_CLEANIT is set, we may block (due to I/O). thus, a caller 779 * might want to unlock higher level resources (e.g. vm_map) 780 * before calling flush. 781 * => if PGO_CLEANIT is not set, then we will neither unlock the object 782 * or block. 783 * => if PGO_ALLPAGE is set, then all pages in the object are valid targets 784 * for flushing. 785 * => NOTE: we rely on the fact that the object's memq is a TAILQ and 786 * that new pages are inserted on the tail end of the list. thus, 787 * we can make a complete pass through the object in one go by starting 788 * at the head and working towards the tail (new pages are put in 789 * front of us). 790 * => NOTE: we are allowed to lock the page queues, so the caller 791 * must not be holding the lock on them [e.g. pagedaemon had 792 * better not call us with the queues locked] 793 * => we return TRUE unless we encountered some sort of I/O error 794 * 795 * comment on "cleaning" object and PG_BUSY pages: 796 * this routine is holding the lock on the object. the only time 797 * that it can run into a PG_BUSY page that it does not own is if 798 * some other process has started I/O on the page (e.g. either 799 * a pagein, or a pageout). if the PG_BUSY page is being paged 800 * in, then it can not be dirty (!PG_CLEAN) because no one has 801 * had a chance to modify it yet. if the PG_BUSY page is being 802 * paged out then it means that someone else has already started 803 * cleaning the page for us (how nice!). in this case, if we 804 * have syncio specified, then after we make our pass through the 805 * object we need to wait for the other PG_BUSY pages to clear 806 * off (i.e. we need to do an iosync). also note that once a 807 * page is PG_BUSY it must stay in its object until it is un-busyed. 808 * 809 * note on page traversal: 810 * we can traverse the pages in an object either by going down the 811 * linked list in "uobj->memq", or we can go over the address range 812 * by page doing hash table lookups for each address. depending 813 * on how many pages are in the object it may be cheaper to do one 814 * or the other. we set "by_list" to true if we are using memq. 815 * if the cost of a hash lookup was equal to the cost of the list 816 * traversal we could compare the number of pages in the start->stop 817 * range to the total number of pages in the object. however, it 818 * seems that a hash table lookup is more expensive than the linked 819 * list traversal, so we multiply the number of pages in the 820 * start->stop range by a penalty which we define below. 821 */ 822 823 #define UVN_HASH_PENALTY 4 /* XXX: a guess */ 824 825 static boolean_t 826 uvn_flush(uobj, start, stop, flags) 827 struct uvm_object *uobj; 828 voff_t start, stop; 829 int flags; 830 { 831 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 832 struct vm_page *pp, *ppnext, *ptmp; 833 struct vm_page *pps[MAXBSIZE >> PAGE_SHIFT], **ppsp; 834 int npages, result, lcv; 835 boolean_t retval, need_iosync, by_list, needs_clean, all; 836 voff_t curoff; 837 u_short pp_version; 838 UVMHIST_FUNC("uvn_flush"); UVMHIST_CALLED(maphist); 839 840 curoff = 0; /* XXX: shut up gcc */ 841 /* 842 * get init vals and determine how we are going to traverse object 843 */ 844 845 need_iosync = FALSE; 846 retval = TRUE; /* return value */ 847 if (flags & PGO_ALLPAGES) { 848 all = TRUE; 849 by_list = TRUE; /* always go by the list */ 850 } else { 851 start = trunc_page(start); 852 stop = round_page(stop); 853 #ifdef DEBUG 854 if (stop > round_page(uvn->u_size)) 855 printf("uvn_flush: strange, got an out of range " 856 "flush (fixed)\n"); 857 #endif 858 all = FALSE; 859 by_list = (uobj->uo_npages <= 860 ((stop - start) >> PAGE_SHIFT) * UVN_HASH_PENALTY); 861 } 862 863 UVMHIST_LOG(maphist, 864 " flush start=0x%x, stop=0x%x, by_list=%d, flags=0x%x", 865 start, stop, by_list, flags); 866 867 /* 868 * PG_CLEANCHK: this bit is used by the pgo_mk_pcluster function as 869 * a _hint_ as to how up to date the PG_CLEAN bit is. if the hint 870 * is wrong it will only prevent us from clustering... it won't break 871 * anything. we clear all PG_CLEANCHK bits here, and pgo_mk_pcluster 872 * will set them as it syncs PG_CLEAN. This is only an issue if we 873 * are looking at non-inactive pages (because inactive page's PG_CLEAN 874 * bit is always up to date since there are no mappings). 875 * [borrowed PG_CLEANCHK idea from FreeBSD VM] 876 */ 877 878 if ((flags & PGO_CLEANIT) != 0 && 879 uobj->pgops->pgo_mk_pcluster != NULL) { 880 if (by_list) { 881 for (pp = uobj->memq.tqh_first ; pp != NULL ; 882 pp = pp->listq.tqe_next) { 883 if (!all && 884 (pp->offset < start || pp->offset >= stop)) 885 continue; 886 pp->flags &= ~PG_CLEANCHK; 887 } 888 889 } else { /* by hash */ 890 for (curoff = start ; curoff < stop; 891 curoff += PAGE_SIZE) { 892 pp = uvm_pagelookup(uobj, curoff); 893 if (pp) 894 pp->flags &= ~PG_CLEANCHK; 895 } 896 } 897 } 898 899 /* 900 * now do it. note: we must update ppnext in body of loop or we 901 * will get stuck. we need to use ppnext because we may free "pp" 902 * before doing the next loop. 903 */ 904 905 if (by_list) { 906 pp = uobj->memq.tqh_first; 907 } else { 908 curoff = start; 909 pp = uvm_pagelookup(uobj, curoff); 910 } 911 912 ppnext = NULL; /* XXX: shut up gcc */ 913 ppsp = NULL; /* XXX: shut up gcc */ 914 uvm_lock_pageq(); /* page queues locked */ 915 916 /* locked: both page queues and uobj */ 917 for ( ; (by_list && pp != NULL) || 918 (!by_list && curoff < stop) ; pp = ppnext) { 919 920 if (by_list) { 921 922 /* 923 * range check 924 */ 925 926 if (!all && 927 (pp->offset < start || pp->offset >= stop)) { 928 ppnext = pp->listq.tqe_next; 929 continue; 930 } 931 932 } else { 933 934 /* 935 * null check 936 */ 937 938 curoff += PAGE_SIZE; 939 if (pp == NULL) { 940 if (curoff < stop) 941 ppnext = uvm_pagelookup(uobj, curoff); 942 continue; 943 } 944 945 } 946 947 /* 948 * handle case where we do not need to clean page (either 949 * because we are not clean or because page is not dirty or 950 * is busy): 951 * 952 * NOTE: we are allowed to deactivate a non-wired active 953 * PG_BUSY page, but once a PG_BUSY page is on the inactive 954 * queue it must stay put until it is !PG_BUSY (so as not to 955 * confuse pagedaemon). 956 */ 957 958 if ((flags & PGO_CLEANIT) == 0 || (pp->flags & PG_BUSY) != 0) { 959 needs_clean = FALSE; 960 if ((pp->flags & PG_BUSY) != 0 && 961 (flags & (PGO_CLEANIT|PGO_SYNCIO)) == 962 (PGO_CLEANIT|PGO_SYNCIO)) 963 need_iosync = TRUE; 964 } else { 965 /* 966 * freeing: nuke all mappings so we can sync 967 * PG_CLEAN bit with no race 968 */ 969 if ((pp->flags & PG_CLEAN) != 0 && 970 (flags & PGO_FREE) != 0 && 971 (pp->pqflags & PQ_ACTIVE) != 0) 972 pmap_page_protect(pp, VM_PROT_NONE); 973 if ((pp->flags & PG_CLEAN) != 0 && 974 pmap_is_modified(pp)) 975 pp->flags &= ~(PG_CLEAN); 976 pp->flags |= PG_CLEANCHK; /* update "hint" */ 977 978 needs_clean = ((pp->flags & PG_CLEAN) == 0); 979 } 980 981 /* 982 * if we don't need a clean... load ppnext and dispose of pp 983 */ 984 if (!needs_clean) { 985 /* load ppnext */ 986 if (by_list) 987 ppnext = pp->listq.tqe_next; 988 else { 989 if (curoff < stop) 990 ppnext = uvm_pagelookup(uobj, curoff); 991 } 992 993 /* now dispose of pp */ 994 if (flags & PGO_DEACTIVATE) { 995 if ((pp->pqflags & PQ_INACTIVE) == 0 && 996 pp->wire_count == 0) { 997 pmap_page_protect(pp, VM_PROT_NONE); 998 uvm_pagedeactivate(pp); 999 } 1000 1001 } else if (flags & PGO_FREE) { 1002 if (pp->flags & PG_BUSY) { 1003 /* release busy pages */ 1004 pp->flags |= PG_RELEASED; 1005 } else { 1006 pmap_page_protect(pp, VM_PROT_NONE); 1007 /* removed page from object */ 1008 uvm_pagefree(pp); 1009 } 1010 } 1011 /* ppnext is valid so we can continue... */ 1012 continue; 1013 } 1014 1015 /* 1016 * pp points to a page in the locked object that we are 1017 * working on. if it is !PG_CLEAN,!PG_BUSY and we asked 1018 * for cleaning (PGO_CLEANIT). we clean it now. 1019 * 1020 * let uvm_pager_put attempted a clustered page out. 1021 * note: locked: uobj and page queues. 1022 */ 1023 1024 pp->flags |= PG_BUSY; /* we 'own' page now */ 1025 UVM_PAGE_OWN(pp, "uvn_flush"); 1026 pmap_page_protect(pp, VM_PROT_READ); 1027 pp_version = pp->version; 1028 ReTry: 1029 ppsp = pps; 1030 npages = sizeof(pps) / sizeof(struct vm_page *); 1031 1032 /* locked: page queues, uobj */ 1033 result = uvm_pager_put(uobj, pp, &ppsp, &npages, 1034 flags | PGO_DOACTCLUST, start, stop); 1035 /* unlocked: page queues, uobj */ 1036 1037 /* 1038 * at this point nothing is locked. if we did an async I/O 1039 * it is remotely possible for the async i/o to complete and 1040 * the page "pp" be freed or what not before we get a chance 1041 * to relock the object. in order to detect this, we have 1042 * saved the version number of the page in "pp_version". 1043 */ 1044 1045 /* relock! */ 1046 simple_lock(&uobj->vmobjlock); 1047 uvm_lock_pageq(); 1048 1049 /* 1050 * VM_PAGER_AGAIN: given the structure of this pager, this 1051 * can only happen when we are doing async I/O and can't 1052 * map the pages into kernel memory (pager_map) due to lack 1053 * of vm space. if this happens we drop back to sync I/O. 1054 */ 1055 1056 if (result == VM_PAGER_AGAIN) { 1057 /* 1058 * it is unlikely, but page could have been released 1059 * while we had the object lock dropped. we ignore 1060 * this now and retry the I/O. we will detect and 1061 * handle the released page after the syncio I/O 1062 * completes. 1063 */ 1064 #ifdef DIAGNOSTIC 1065 if (flags & PGO_SYNCIO) 1066 panic("uvn_flush: PGO_SYNCIO return 'try again' error (impossible)"); 1067 #endif 1068 flags |= PGO_SYNCIO; 1069 goto ReTry; 1070 } 1071 1072 /* 1073 * the cleaning operation is now done. finish up. note that 1074 * on error (!OK, !PEND) uvm_pager_put drops the cluster for us. 1075 * if success (OK, PEND) then uvm_pager_put returns the cluster 1076 * to us in ppsp/npages. 1077 */ 1078 1079 /* 1080 * for pending async i/o if we are not deactivating/freeing 1081 * we can move on to the next page. 1082 */ 1083 1084 if (result == VM_PAGER_PEND) { 1085 1086 if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) { 1087 /* 1088 * no per-page ops: refresh ppnext and continue 1089 */ 1090 if (by_list) { 1091 if (pp->version == pp_version) 1092 ppnext = pp->listq.tqe_next; 1093 else 1094 /* reset */ 1095 ppnext = uobj->memq.tqh_first; 1096 } else { 1097 if (curoff < stop) 1098 ppnext = uvm_pagelookup(uobj, 1099 curoff); 1100 } 1101 continue; 1102 } 1103 1104 /* need to do anything here? */ 1105 } 1106 1107 /* 1108 * need to look at each page of the I/O operation. we defer 1109 * processing "pp" until the last trip through this "for" loop 1110 * so that we can load "ppnext" for the main loop after we 1111 * play with the cluster pages [thus the "npages + 1" in the 1112 * loop below]. 1113 */ 1114 1115 for (lcv = 0 ; lcv < npages + 1 ; lcv++) { 1116 1117 /* 1118 * handle ppnext for outside loop, and saving pp 1119 * until the end. 1120 */ 1121 if (lcv < npages) { 1122 if (ppsp[lcv] == pp) 1123 continue; /* skip pp until the end */ 1124 ptmp = ppsp[lcv]; 1125 } else { 1126 ptmp = pp; 1127 1128 /* set up next page for outer loop */ 1129 if (by_list) { 1130 if (pp->version == pp_version) 1131 ppnext = pp->listq.tqe_next; 1132 else 1133 /* reset */ 1134 ppnext = uobj->memq.tqh_first; 1135 } else { 1136 if (curoff < stop) 1137 ppnext = uvm_pagelookup(uobj, curoff); 1138 } 1139 } 1140 1141 /* 1142 * verify the page didn't get moved while obj was 1143 * unlocked 1144 */ 1145 if (result == VM_PAGER_PEND && ptmp->uobject != uobj) 1146 continue; 1147 1148 /* 1149 * unbusy the page if I/O is done. note that for 1150 * pending I/O it is possible that the I/O op 1151 * finished before we relocked the object (in 1152 * which case the page is no longer busy). 1153 */ 1154 1155 if (result != VM_PAGER_PEND) { 1156 if (ptmp->flags & PG_WANTED) 1157 /* still holding object lock */ 1158 wakeup(ptmp); 1159 1160 ptmp->flags &= ~(PG_WANTED|PG_BUSY); 1161 UVM_PAGE_OWN(ptmp, NULL); 1162 if (ptmp->flags & PG_RELEASED) { 1163 1164 /* pgo_releasepg wants this */ 1165 uvm_unlock_pageq(); 1166 if (!uvn_releasepg(ptmp, NULL)) 1167 return (TRUE); 1168 1169 uvm_lock_pageq(); /* relock */ 1170 continue; /* next page */ 1171 1172 } else { 1173 ptmp->flags |= (PG_CLEAN|PG_CLEANCHK); 1174 if ((flags & PGO_FREE) == 0) 1175 pmap_clear_modify(ptmp); 1176 } 1177 } 1178 1179 /* 1180 * dispose of page 1181 */ 1182 1183 if (flags & PGO_DEACTIVATE) { 1184 if ((pp->pqflags & PQ_INACTIVE) == 0 && 1185 pp->wire_count == 0) { 1186 pmap_page_protect(ptmp, VM_PROT_NONE); 1187 uvm_pagedeactivate(ptmp); 1188 } 1189 1190 } else if (flags & PGO_FREE) { 1191 if (result == VM_PAGER_PEND) { 1192 if ((ptmp->flags & PG_BUSY) != 0) 1193 /* signal for i/o done */ 1194 ptmp->flags |= PG_RELEASED; 1195 } else { 1196 if (result != VM_PAGER_OK) { 1197 printf("uvn_flush: obj=%p, " 1198 "offset=0x%llx. error " 1199 "during pageout.\n", 1200 pp->uobject, 1201 (long long)pp->offset); 1202 printf("uvn_flush: WARNING: " 1203 "changes to page may be " 1204 "lost!\n"); 1205 retval = FALSE; 1206 } 1207 pmap_page_protect(ptmp, VM_PROT_NONE); 1208 uvm_pagefree(ptmp); 1209 } 1210 } 1211 1212 } /* end of "lcv" for loop */ 1213 1214 } /* end of "pp" for loop */ 1215 1216 /* 1217 * done with pagequeues: unlock 1218 */ 1219 uvm_unlock_pageq(); 1220 1221 /* 1222 * now wait for all I/O if required. 1223 */ 1224 if (need_iosync) { 1225 1226 UVMHIST_LOG(maphist," <<DOING IOSYNC>>",0,0,0,0); 1227 while (uvn->u_nio != 0) { 1228 uvn->u_flags |= UVM_VNODE_IOSYNC; 1229 UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock, 1230 FALSE, "uvn_flush",0); 1231 simple_lock(&uvn->u_obj.vmobjlock); 1232 } 1233 if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED) 1234 wakeup(&uvn->u_flags); 1235 uvn->u_flags &= ~(UVM_VNODE_IOSYNC|UVM_VNODE_IOSYNCWANTED); 1236 } 1237 1238 /* return, with object locked! */ 1239 UVMHIST_LOG(maphist,"<- done (retval=0x%x)",retval,0,0,0); 1240 return(retval); 1241 } 1242 1243 /* 1244 * uvn_cluster 1245 * 1246 * we are about to do I/O in an object at offset. this function is called 1247 * to establish a range of offsets around "offset" in which we can cluster 1248 * I/O. 1249 * 1250 * - currently doesn't matter if obj locked or not. 1251 */ 1252 1253 static void 1254 uvn_cluster(uobj, offset, loffset, hoffset) 1255 struct uvm_object *uobj; 1256 voff_t offset; 1257 voff_t *loffset, *hoffset; /* OUT */ 1258 { 1259 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 1260 *loffset = offset; 1261 1262 if (*loffset >= uvn->u_size) 1263 panic("uvn_cluster: offset out of range"); 1264 1265 /* 1266 * XXX: old pager claims we could use VOP_BMAP to get maxcontig value. 1267 */ 1268 *hoffset = *loffset + MAXBSIZE; 1269 if (*hoffset > round_page(uvn->u_size)) /* past end? */ 1270 *hoffset = round_page(uvn->u_size); 1271 1272 return; 1273 } 1274 1275 /* 1276 * uvn_put: flush page data to backing store. 1277 * 1278 * => prefer map unlocked (not required) 1279 * => object must be locked! we will _unlock_ it before starting I/O. 1280 * => flags: PGO_SYNCIO -- use sync. I/O 1281 * => note: caller must set PG_CLEAN and pmap_clear_modify (if needed) 1282 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync. 1283 * [thus we never do async i/o! see iodone comment] 1284 */ 1285 1286 static int 1287 uvn_put(uobj, pps, npages, flags) 1288 struct uvm_object *uobj; 1289 struct vm_page **pps; 1290 int npages, flags; 1291 { 1292 int retval; 1293 1294 /* note: object locked */ 1295 retval = uvn_io((struct uvm_vnode*)uobj, pps, npages, flags, UIO_WRITE); 1296 /* note: object unlocked */ 1297 1298 return(retval); 1299 } 1300 1301 1302 /* 1303 * uvn_get: get pages (synchronously) from backing store 1304 * 1305 * => prefer map unlocked (not required) 1306 * => object must be locked! we will _unlock_ it before starting any I/O. 1307 * => flags: PGO_ALLPAGES: get all of the pages 1308 * PGO_LOCKED: fault data structures are locked 1309 * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx] 1310 * => NOTE: caller must check for released pages!! 1311 */ 1312 1313 static int 1314 uvn_get(uobj, offset, pps, npagesp, centeridx, access_type, advice, flags) 1315 struct uvm_object *uobj; 1316 voff_t offset; 1317 struct vm_page **pps; /* IN/OUT */ 1318 int *npagesp; /* IN (OUT if PGO_LOCKED) */ 1319 int centeridx, advice, flags; 1320 vm_prot_t access_type; 1321 { 1322 voff_t current_offset; 1323 struct vm_page *ptmp; 1324 int lcv, result, gotpages; 1325 boolean_t done; 1326 UVMHIST_FUNC("uvn_get"); UVMHIST_CALLED(maphist); 1327 UVMHIST_LOG(maphist, "flags=%d", flags,0,0,0); 1328 1329 /* 1330 * step 1: handled the case where fault data structures are locked. 1331 */ 1332 1333 if (flags & PGO_LOCKED) { 1334 1335 /* 1336 * gotpages is the current number of pages we've gotten (which 1337 * we pass back up to caller via *npagesp. 1338 */ 1339 1340 gotpages = 0; 1341 1342 /* 1343 * step 1a: get pages that are already resident. only do this 1344 * if the data structures are locked (i.e. the first time 1345 * through). 1346 */ 1347 1348 done = TRUE; /* be optimistic */ 1349 1350 for (lcv = 0, current_offset = offset ; lcv < *npagesp ; 1351 lcv++, current_offset += PAGE_SIZE) { 1352 1353 /* do we care about this page? if not, skip it */ 1354 if (pps[lcv] == PGO_DONTCARE) 1355 continue; 1356 1357 /* lookup page */ 1358 ptmp = uvm_pagelookup(uobj, current_offset); 1359 1360 /* to be useful must get a non-busy, non-released pg */ 1361 if (ptmp == NULL || 1362 (ptmp->flags & (PG_BUSY|PG_RELEASED)) != 0) { 1363 if (lcv == centeridx || (flags & PGO_ALLPAGES) 1364 != 0) 1365 done = FALSE; /* need to do a wait or I/O! */ 1366 continue; 1367 } 1368 1369 /* 1370 * useful page: busy/lock it and plug it in our 1371 * result array 1372 */ 1373 ptmp->flags |= PG_BUSY; /* loan up to caller */ 1374 UVM_PAGE_OWN(ptmp, "uvn_get1"); 1375 pps[lcv] = ptmp; 1376 gotpages++; 1377 1378 } /* "for" lcv loop */ 1379 1380 /* 1381 * XXX: given the "advice", should we consider async read-ahead? 1382 * XXX: fault current does deactive of pages behind us. is 1383 * this good (other callers might now). 1384 */ 1385 /* 1386 * XXX: read-ahead currently handled by buffer cache (bread) 1387 * level. 1388 * XXX: no async i/o available. 1389 * XXX: so we don't do anything now. 1390 */ 1391 1392 /* 1393 * step 1c: now we've either done everything needed or we to 1394 * unlock and do some waiting or I/O. 1395 */ 1396 1397 *npagesp = gotpages; /* let caller know */ 1398 if (done) 1399 return(VM_PAGER_OK); /* bingo! */ 1400 else 1401 /* EEK! Need to unlock and I/O */ 1402 return(VM_PAGER_UNLOCK); 1403 } 1404 1405 /* 1406 * step 2: get non-resident or busy pages. 1407 * object is locked. data structures are unlocked. 1408 * 1409 * XXX: because we can't do async I/O at this level we get things 1410 * page at a time (otherwise we'd chunk). the VOP_READ() will do 1411 * async-read-ahead for us at a lower level. 1412 */ 1413 1414 for (lcv = 0, current_offset = offset ; 1415 lcv < *npagesp ; lcv++, current_offset += PAGE_SIZE) { 1416 1417 /* skip over pages we've already gotten or don't want */ 1418 /* skip over pages we don't _have_ to get */ 1419 if (pps[lcv] != NULL || (lcv != centeridx && 1420 (flags & PGO_ALLPAGES) == 0)) 1421 continue; 1422 1423 /* 1424 * we have yet to locate the current page (pps[lcv]). we first 1425 * look for a page that is already at the current offset. if 1426 * we fine a page, we check to see if it is busy or released. 1427 * if that is the case, then we sleep on the page until it is 1428 * no longer busy or released and repeat the lookup. if the 1429 * page we found is neither busy nor released, then we busy it 1430 * (so we own it) and plug it into pps[lcv]. this breaks the 1431 * following while loop and indicates we are ready to move on 1432 * to the next page in the "lcv" loop above. 1433 * 1434 * if we exit the while loop with pps[lcv] still set to NULL, 1435 * then it means that we allocated a new busy/fake/clean page 1436 * ptmp in the object and we need to do I/O to fill in the data. 1437 */ 1438 1439 while (pps[lcv] == NULL) { /* top of "pps" while loop */ 1440 1441 /* look for a current page */ 1442 ptmp = uvm_pagelookup(uobj, current_offset); 1443 1444 /* nope? allocate one now (if we can) */ 1445 if (ptmp == NULL) { 1446 1447 ptmp = uvm_pagealloc(uobj, current_offset, 1448 NULL, 0); 1449 1450 /* out of RAM? */ 1451 if (ptmp == NULL) { 1452 simple_unlock(&uobj->vmobjlock); 1453 uvm_wait("uvn_getpage"); 1454 simple_lock(&uobj->vmobjlock); 1455 1456 /* goto top of pps while loop */ 1457 continue; 1458 } 1459 1460 /* 1461 * got new page ready for I/O. break pps 1462 * while loop. pps[lcv] is still NULL. 1463 */ 1464 break; 1465 } 1466 1467 /* page is there, see if we need to wait on it */ 1468 if ((ptmp->flags & (PG_BUSY|PG_RELEASED)) != 0) { 1469 ptmp->flags |= PG_WANTED; 1470 UVM_UNLOCK_AND_WAIT(ptmp, 1471 &uobj->vmobjlock, FALSE, "uvn_get",0); 1472 simple_lock(&uobj->vmobjlock); 1473 continue; /* goto top of pps while loop */ 1474 } 1475 1476 /* 1477 * if we get here then the page has become resident 1478 * and unbusy between steps 1 and 2. we busy it 1479 * now (so we own it) and set pps[lcv] (so that we 1480 * exit the while loop). 1481 */ 1482 ptmp->flags |= PG_BUSY; 1483 UVM_PAGE_OWN(ptmp, "uvn_get2"); 1484 pps[lcv] = ptmp; 1485 } 1486 1487 /* 1488 * if we own the a valid page at the correct offset, pps[lcv] 1489 * will point to it. nothing more to do except go to the 1490 * next page. 1491 */ 1492 1493 if (pps[lcv]) 1494 continue; /* next lcv */ 1495 1496 /* 1497 * we have a "fake/busy/clean" page that we just allocated. do 1498 * I/O to fill it with valid data. note that object must be 1499 * locked going into uvn_io, but will be unlocked afterwards. 1500 */ 1501 1502 result = uvn_io((struct uvm_vnode *) uobj, &ptmp, 1, 1503 PGO_SYNCIO, UIO_READ); 1504 1505 /* 1506 * I/O done. object is unlocked (by uvn_io). because we used 1507 * syncio the result can not be PEND or AGAIN. we must relock 1508 * and check for errors. 1509 */ 1510 1511 /* lock object. check for errors. */ 1512 simple_lock(&uobj->vmobjlock); 1513 if (result != VM_PAGER_OK) { 1514 if (ptmp->flags & PG_WANTED) 1515 /* object lock still held */ 1516 wakeup(ptmp); 1517 1518 ptmp->flags &= ~(PG_WANTED|PG_BUSY); 1519 UVM_PAGE_OWN(ptmp, NULL); 1520 uvm_lock_pageq(); 1521 uvm_pagefree(ptmp); 1522 uvm_unlock_pageq(); 1523 simple_unlock(&uobj->vmobjlock); 1524 return(result); 1525 } 1526 1527 /* 1528 * we got the page! clear the fake flag (indicates valid 1529 * data now in page) and plug into our result array. note 1530 * that page is still busy. 1531 * 1532 * it is the callers job to: 1533 * => check if the page is released 1534 * => unbusy the page 1535 * => activate the page 1536 */ 1537 1538 ptmp->flags &= ~PG_FAKE; /* data is valid ... */ 1539 pmap_clear_modify(ptmp); /* ... and clean */ 1540 pps[lcv] = ptmp; 1541 1542 } /* lcv loop */ 1543 1544 /* 1545 * finally, unlock object and return. 1546 */ 1547 1548 simple_unlock(&uobj->vmobjlock); 1549 return (VM_PAGER_OK); 1550 } 1551 1552 /* 1553 * uvn_asyncget: start async I/O to bring pages into ram 1554 * 1555 * => caller must lock object(???XXX: see if this is best) 1556 * => could be called from uvn_get or a madvise() fault-ahead. 1557 * => if it fails, it doesn't matter. 1558 */ 1559 1560 static int 1561 uvn_asyncget(uobj, offset, npages) 1562 struct uvm_object *uobj; 1563 voff_t offset; 1564 int npages; 1565 { 1566 1567 /* 1568 * XXXCDC: we can't do async I/O yet 1569 */ 1570 printf("uvn_asyncget called\n"); 1571 return (KERN_SUCCESS); 1572 } 1573 1574 /* 1575 * uvn_io: do I/O to a vnode 1576 * 1577 * => prefer map unlocked (not required) 1578 * => object must be locked! we will _unlock_ it before starting I/O. 1579 * => flags: PGO_SYNCIO -- use sync. I/O 1580 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync. 1581 * [thus we never do async i/o! see iodone comment] 1582 */ 1583 1584 static int 1585 uvn_io(uvn, pps, npages, flags, rw) 1586 struct uvm_vnode *uvn; 1587 vm_page_t *pps; 1588 int npages, flags, rw; 1589 { 1590 struct vnode *vn; 1591 struct uio uio; 1592 struct iovec iov; 1593 vaddr_t kva; 1594 off_t file_offset; 1595 int waitf, result, mapinflags; 1596 size_t got, wanted; 1597 UVMHIST_FUNC("uvn_io"); UVMHIST_CALLED(maphist); 1598 1599 UVMHIST_LOG(maphist, "rw=%d", rw,0,0,0); 1600 1601 /* 1602 * init values 1603 */ 1604 1605 waitf = (flags & PGO_SYNCIO) ? M_WAITOK : M_NOWAIT; 1606 vn = (struct vnode *) uvn; 1607 file_offset = pps[0]->offset; 1608 1609 /* 1610 * check for sync'ing I/O. 1611 */ 1612 1613 while (uvn->u_flags & UVM_VNODE_IOSYNC) { 1614 if (waitf == M_NOWAIT) { 1615 simple_unlock(&uvn->u_obj.vmobjlock); 1616 UVMHIST_LOG(maphist,"<- try again (iosync)",0,0,0,0); 1617 return(VM_PAGER_AGAIN); 1618 } 1619 uvn->u_flags |= UVM_VNODE_IOSYNCWANTED; 1620 UVM_UNLOCK_AND_WAIT(&uvn->u_flags, &uvn->u_obj.vmobjlock, 1621 FALSE, "uvn_iosync",0); 1622 simple_lock(&uvn->u_obj.vmobjlock); 1623 } 1624 1625 /* 1626 * check size 1627 */ 1628 1629 if (file_offset >= uvn->u_size) { 1630 simple_unlock(&uvn->u_obj.vmobjlock); 1631 UVMHIST_LOG(maphist,"<- BAD (size check)",0,0,0,0); 1632 return(VM_PAGER_BAD); 1633 } 1634 1635 /* 1636 * first try and map the pages in (without waiting) 1637 */ 1638 1639 mapinflags = (rw == UIO_READ) ? 1640 UVMPAGER_MAPIN_READ : UVMPAGER_MAPIN_WRITE; 1641 1642 kva = uvm_pagermapin(pps, npages, NULL, mapinflags); 1643 if (kva == 0 && waitf == M_NOWAIT) { 1644 simple_unlock(&uvn->u_obj.vmobjlock); 1645 UVMHIST_LOG(maphist,"<- mapin failed (try again)",0,0,0,0); 1646 return(VM_PAGER_AGAIN); 1647 } 1648 1649 /* 1650 * ok, now bump u_nio up. at this point we are done with uvn 1651 * and can unlock it. if we still don't have a kva, try again 1652 * (this time with sleep ok). 1653 */ 1654 1655 uvn->u_nio++; /* we have an I/O in progress! */ 1656 simple_unlock(&uvn->u_obj.vmobjlock); 1657 /* NOTE: object now unlocked */ 1658 if (kva == 0) 1659 kva = uvm_pagermapin(pps, npages, NULL, 1660 mapinflags | UVMPAGER_MAPIN_WAITOK); 1661 1662 /* 1663 * ok, mapped in. our pages are PG_BUSY so they are not going to 1664 * get touched (so we can look at "offset" without having to lock 1665 * the object). set up for I/O. 1666 */ 1667 1668 /* 1669 * fill out uio/iov 1670 */ 1671 1672 iov.iov_base = (caddr_t) kva; 1673 wanted = npages << PAGE_SHIFT; 1674 if (file_offset + wanted > uvn->u_size) 1675 wanted = uvn->u_size - file_offset; /* XXX: needed? */ 1676 iov.iov_len = wanted; 1677 uio.uio_iov = &iov; 1678 uio.uio_iovcnt = 1; 1679 uio.uio_offset = file_offset; 1680 uio.uio_segflg = UIO_SYSSPACE; 1681 uio.uio_rw = rw; 1682 uio.uio_resid = wanted; 1683 uio.uio_procp = curproc; 1684 1685 /* 1686 * do the I/O! (XXX: curproc?) 1687 */ 1688 1689 UVMHIST_LOG(maphist, "calling VOP",0,0,0,0); 1690 1691 /* 1692 * This process may already have this vnode locked, if we faulted in 1693 * copyin() or copyout() on a region backed by this vnode 1694 * while doing I/O to the vnode. If this is the case, don't 1695 * panic.. instead, return the error to the user. 1696 * 1697 * XXX this is a stopgap to prevent a panic. 1698 * Ideally, this kind of operation *should* work. 1699 */ 1700 result = 0; 1701 if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0) 1702 result = vn_lock(vn, LK_EXCLUSIVE | LK_RETRY, curproc /*XXX*/); 1703 1704 if (result == 0) { 1705 /* NOTE: vnode now locked! */ 1706 1707 if (rw == UIO_READ) 1708 result = VOP_READ(vn, &uio, 0, curproc->p_ucred); 1709 else 1710 result = VOP_WRITE(vn, &uio, 0, curproc->p_ucred); 1711 1712 if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0) 1713 VOP_UNLOCK(vn, 0, curproc /*XXX*/); 1714 } 1715 1716 /* NOTE: vnode now unlocked (unless vnislocked) */ 1717 1718 UVMHIST_LOG(maphist, "done calling VOP",0,0,0,0); 1719 1720 /* 1721 * result == unix style errno (0 == OK!) 1722 * 1723 * zero out rest of buffer (if needed) 1724 */ 1725 1726 if (result == 0) { 1727 got = wanted - uio.uio_resid; 1728 1729 if (wanted && got == 0) { 1730 result = EIO; /* XXX: error? */ 1731 } else if (got < PAGE_SIZE * npages && rw == UIO_READ) { 1732 memset((void *) (kva + got), 0, 1733 (npages << PAGE_SHIFT) - got); 1734 } 1735 } 1736 1737 /* 1738 * now remove pager mapping 1739 */ 1740 uvm_pagermapout(kva, npages); 1741 1742 /* 1743 * now clean up the object (i.e. drop I/O count) 1744 */ 1745 1746 simple_lock(&uvn->u_obj.vmobjlock); 1747 /* NOTE: object now locked! */ 1748 1749 uvn->u_nio--; /* I/O DONE! */ 1750 if ((uvn->u_flags & UVM_VNODE_IOSYNC) != 0 && uvn->u_nio == 0) { 1751 wakeup(&uvn->u_nio); 1752 } 1753 simple_unlock(&uvn->u_obj.vmobjlock); 1754 /* NOTE: object now unlocked! */ 1755 1756 /* 1757 * done! 1758 */ 1759 1760 UVMHIST_LOG(maphist, "<- done (result %d)", result,0,0,0); 1761 if (result == 0) 1762 return(VM_PAGER_OK); 1763 else 1764 return(VM_PAGER_ERROR); 1765 } 1766 1767 /* 1768 * uvm_vnp_uncache: disable "persisting" in a vnode... when last reference 1769 * is gone we will kill the object (flushing dirty pages back to the vnode 1770 * if needed). 1771 * 1772 * => returns TRUE if there was no uvm_object attached or if there was 1773 * one and we killed it [i.e. if there is no active uvn] 1774 * => called with the vnode VOP_LOCK'd [we will unlock it for I/O, if 1775 * needed] 1776 * 1777 * => XXX: given that we now kill uvn's when a vnode is recycled (without 1778 * having to hold a reference on the vnode) and given a working 1779 * uvm_vnp_sync(), how does that effect the need for this function? 1780 * [XXXCDC: seems like it can die?] 1781 * 1782 * => XXX: this function should DIE once we merge the VM and buffer 1783 * cache. 1784 * 1785 * research shows that this is called in the following places: 1786 * ext2fs_truncate, ffs_truncate, detrunc[msdosfs]: called when vnode 1787 * changes sizes 1788 * ext2fs_write, WRITE [ufs_readwrite], msdosfs_write: called when we 1789 * are written to 1790 * ex2fs_chmod, ufs_chmod: called if VTEXT vnode and the sticky bit 1791 * is off 1792 * ffs_realloccg: when we can't extend the current block and have 1793 * to allocate a new one we call this [XXX: why?] 1794 * nfsrv_rename, rename_files: called when the target filename is there 1795 * and we want to remove it 1796 * nfsrv_remove, sys_unlink: called on file we are removing 1797 * nfsrv_access: if VTEXT and we want WRITE access and we don't uncache 1798 * then return "text busy" 1799 * nfs_open: seems to uncache any file opened with nfs 1800 * vn_writechk: if VTEXT vnode and can't uncache return "text busy" 1801 */ 1802 1803 boolean_t 1804 uvm_vnp_uncache(vp) 1805 struct vnode *vp; 1806 { 1807 struct uvm_vnode *uvn = &vp->v_uvm; 1808 1809 /* 1810 * lock uvn part of the vnode and check to see if we need to do anything 1811 */ 1812 1813 simple_lock(&uvn->u_obj.vmobjlock); 1814 if ((uvn->u_flags & UVM_VNODE_VALID) == 0 || 1815 (uvn->u_flags & UVM_VNODE_BLOCKED) != 0) { 1816 simple_unlock(&uvn->u_obj.vmobjlock); 1817 return(TRUE); 1818 } 1819 1820 /* 1821 * we have a valid, non-blocked uvn. clear persist flag. 1822 * if uvn is currently active we can return now. 1823 */ 1824 1825 uvn->u_flags &= ~UVM_VNODE_CANPERSIST; 1826 if (uvn->u_obj.uo_refs) { 1827 simple_unlock(&uvn->u_obj.vmobjlock); 1828 return(FALSE); 1829 } 1830 1831 /* 1832 * uvn is currently persisting! we have to gain a reference to 1833 * it so that we can call uvn_detach to kill the uvn. 1834 */ 1835 1836 VREF(vp); /* seems ok, even with VOP_LOCK */ 1837 uvn->u_obj.uo_refs++; /* value is now 1 */ 1838 simple_unlock(&uvn->u_obj.vmobjlock); 1839 1840 1841 #ifdef DEBUG 1842 /* 1843 * carry over sanity check from old vnode pager: the vnode should 1844 * be VOP_LOCK'd, and we confirm it here. 1845 */ 1846 if (!VOP_ISLOCKED(vp)) { 1847 boolean_t is_ok_anyway = FALSE; 1848 #if defined(NFSCLIENT) 1849 extern int (**nfsv2_vnodeop_p) __P((void *)); 1850 extern int (**spec_nfsv2nodeop_p) __P((void *)); 1851 extern int (**fifo_nfsv2nodeop_p) __P((void *)); 1852 1853 /* vnode is NOT VOP_LOCKed: some vnode types _never_ lock */ 1854 if (vp->v_op == nfsv2_vnodeop_p || 1855 vp->v_op == spec_nfsv2nodeop_p) { 1856 is_ok_anyway = TRUE; 1857 } 1858 if (vp->v_op == fifo_nfsv2nodeop_p) { 1859 is_ok_anyway = TRUE; 1860 } 1861 #endif /* defined(NFSSERVER) || defined(NFSCLIENT) */ 1862 if (!is_ok_anyway) 1863 panic("uvm_vnp_uncache: vnode not locked!"); 1864 } 1865 #endif /* DEBUG */ 1866 1867 /* 1868 * now drop our reference to the vnode. if we have the sole 1869 * reference to the vnode then this will cause it to die [as we 1870 * just cleared the persist flag]. we have to unlock the vnode 1871 * while we are doing this as it may trigger I/O. 1872 * 1873 * XXX: it might be possible for uvn to get reclaimed while we are 1874 * unlocked causing us to return TRUE when we should not. we ignore 1875 * this as a false-positive return value doesn't hurt us. 1876 */ 1877 VOP_UNLOCK(vp, 0, curproc /*XXX*/); 1878 uvn_detach(&uvn->u_obj); 1879 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, curproc/*XXX*/); 1880 1881 /* 1882 * and return... 1883 */ 1884 1885 return(TRUE); 1886 } 1887 1888 /* 1889 * uvm_vnp_setsize: grow or shrink a vnode uvn 1890 * 1891 * grow => just update size value 1892 * shrink => toss un-needed pages 1893 * 1894 * => we assume that the caller has a reference of some sort to the 1895 * vnode in question so that it will not be yanked out from under 1896 * us. 1897 * 1898 * called from: 1899 * => truncate fns (ext2fs_truncate, ffs_truncate, detrunc[msdos]) 1900 * => "write" fns (ext2fs_write, WRITE [ufs/ufs], msdosfs_write, nfs_write) 1901 * => ffs_balloc [XXX: why? doesn't WRITE handle?] 1902 * => NFS: nfs_loadattrcache, nfs_getattrcache, nfs_setattr 1903 * => union fs: union_newsize 1904 */ 1905 1906 void 1907 uvm_vnp_setsize(vp, newsize) 1908 struct vnode *vp; 1909 voff_t newsize; 1910 { 1911 struct uvm_vnode *uvn = &vp->v_uvm; 1912 1913 /* 1914 * lock uvn and check for valid object, and if valid: do it! 1915 */ 1916 simple_lock(&uvn->u_obj.vmobjlock); 1917 if (uvn->u_flags & UVM_VNODE_VALID) { 1918 1919 /* 1920 * now check if the size has changed: if we shrink we had better 1921 * toss some pages... 1922 */ 1923 1924 if (uvn->u_size > newsize) { 1925 (void)uvn_flush(&uvn->u_obj, newsize, 1926 uvn->u_size, PGO_FREE); 1927 } 1928 uvn->u_size = newsize; 1929 } 1930 simple_unlock(&uvn->u_obj.vmobjlock); 1931 1932 /* 1933 * done 1934 */ 1935 return; 1936 } 1937 1938 /* 1939 * uvm_vnp_sync: flush all dirty VM pages back to their backing vnodes. 1940 * 1941 * => called from sys_sync with no VM structures locked 1942 * => only one process can do a sync at a time (because the uvn 1943 * structure only has one queue for sync'ing). we ensure this 1944 * by holding the uvn_sync_lock while the sync is in progress. 1945 * other processes attempting a sync will sleep on this lock 1946 * until we are done. 1947 */ 1948 1949 void 1950 uvm_vnp_sync(mp) 1951 struct mount *mp; 1952 { 1953 struct uvm_vnode *uvn; 1954 struct vnode *vp; 1955 boolean_t got_lock; 1956 1957 /* 1958 * step 1: ensure we are only ones using the uvn_sync_q by locking 1959 * our lock... 1960 */ 1961 lockmgr(&uvn_sync_lock, LK_EXCLUSIVE, (void *)0, curproc /*XXX*/); 1962 1963 /* 1964 * step 2: build up a simpleq of uvns of interest based on the 1965 * write list. we gain a reference to uvns of interest. must 1966 * be careful about locking uvn's since we will be holding uvn_wl_lock 1967 * in the body of the loop. 1968 */ 1969 SIMPLEQ_INIT(&uvn_sync_q); 1970 simple_lock(&uvn_wl_lock); 1971 for (uvn = uvn_wlist.lh_first ; uvn != NULL ; 1972 uvn = uvn->u_wlist.le_next) { 1973 1974 vp = (struct vnode *) uvn; 1975 if (mp && vp->v_mount != mp) 1976 continue; 1977 1978 /* attempt to gain reference */ 1979 while ((got_lock = simple_lock_try(&uvn->u_obj.vmobjlock)) == 1980 FALSE && 1981 (uvn->u_flags & UVM_VNODE_BLOCKED) == 0) 1982 /* spin */ ; 1983 1984 /* 1985 * we will exit the loop if either if the following are true: 1986 * - we got the lock [always true if NCPU == 1] 1987 * - we failed to get the lock but noticed the vnode was 1988 * "blocked" -- in this case the vnode must be a dying 1989 * vnode, and since dying vnodes are in the process of 1990 * being flushed out, we can safely skip this one 1991 * 1992 * we want to skip over the vnode if we did not get the lock, 1993 * or if the vnode is already dying (due to the above logic). 1994 * 1995 * note that uvn must already be valid because we found it on 1996 * the wlist (this also means it can't be ALOCK'd). 1997 */ 1998 if (!got_lock || (uvn->u_flags & UVM_VNODE_BLOCKED) != 0) { 1999 if (got_lock) 2000 simple_unlock(&uvn->u_obj.vmobjlock); 2001 continue; /* skip it */ 2002 } 2003 2004 /* 2005 * gain reference. watch out for persisting uvns (need to 2006 * regain vnode REF). 2007 */ 2008 if (uvn->u_obj.uo_refs == 0) 2009 VREF(vp); 2010 uvn->u_obj.uo_refs++; 2011 simple_unlock(&uvn->u_obj.vmobjlock); 2012 2013 /* 2014 * got it! 2015 */ 2016 SIMPLEQ_INSERT_HEAD(&uvn_sync_q, uvn, u_syncq); 2017 } 2018 simple_unlock(&uvn_wl_lock); 2019 2020 /* 2021 * step 3: we now have a list of uvn's that may need cleaning. 2022 * we are holding the uvn_sync_lock, but have dropped the uvn_wl_lock 2023 * (so we can now safely lock uvn's again). 2024 */ 2025 2026 for (uvn = uvn_sync_q.sqh_first ; uvn ; uvn = uvn->u_syncq.sqe_next) { 2027 simple_lock(&uvn->u_obj.vmobjlock); 2028 #ifdef DEBUG 2029 if (uvn->u_flags & UVM_VNODE_DYING) { 2030 printf("uvm_vnp_sync: dying vnode on sync list\n"); 2031 } 2032 #endif 2033 uvn_flush(&uvn->u_obj, 0, 0, 2034 PGO_CLEANIT|PGO_ALLPAGES|PGO_DOACTCLUST); 2035 2036 /* 2037 * if we have the only reference and we just cleaned the uvn, 2038 * then we can pull it out of the UVM_VNODE_WRITEABLE state 2039 * thus allowing us to avoid thinking about flushing it again 2040 * on later sync ops. 2041 */ 2042 if (uvn->u_obj.uo_refs == 1 && 2043 (uvn->u_flags & UVM_VNODE_WRITEABLE)) { 2044 LIST_REMOVE(uvn, u_wlist); 2045 uvn->u_flags &= ~UVM_VNODE_WRITEABLE; 2046 } 2047 2048 simple_unlock(&uvn->u_obj.vmobjlock); 2049 2050 /* now drop our reference to the uvn */ 2051 uvn_detach(&uvn->u_obj); 2052 } 2053 2054 /* 2055 * done! release sync lock 2056 */ 2057 lockmgr(&uvn_sync_lock, LK_RELEASE, (void *)0, curproc /*XXX*/); 2058 } 2059