1 /* $NetBSD: uvm_vnode.c,v 1.35 2000/06/27 17:29:37 mrg Exp $ */ 2 3 /* 4 * Copyright (c) 1997 Charles D. Cranor and Washington University. 5 * Copyright (c) 1991, 1993 6 * The Regents of the University of California. 7 * Copyright (c) 1990 University of Utah. 8 * 9 * All rights reserved. 10 * 11 * This code is derived from software contributed to Berkeley by 12 * the Systems Programming Group of the University of Utah Computer 13 * Science Department. 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 3. All advertising materials mentioning features or use of this software 24 * must display the following acknowledgement: 25 * This product includes software developed by Charles D. Cranor, 26 * Washington University, the University of California, Berkeley and 27 * its contributors. 28 * 4. Neither the name of the University nor the names of its contributors 29 * may be used to endorse or promote products derived from this software 30 * without specific prior written permission. 31 * 32 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 33 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 34 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 35 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 36 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 37 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 38 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 39 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 40 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 41 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 42 * SUCH DAMAGE. 43 * 44 * @(#)vnode_pager.c 8.8 (Berkeley) 2/13/94 45 * from: Id: uvm_vnode.c,v 1.1.2.26 1998/02/02 20:38:07 chuck Exp 46 */ 47 48 #include "fs_nfs.h" 49 #include "opt_uvmhist.h" 50 51 /* 52 * uvm_vnode.c: the vnode pager. 53 */ 54 55 #include <sys/param.h> 56 #include <sys/systm.h> 57 #include <sys/proc.h> 58 #include <sys/malloc.h> 59 #include <sys/vnode.h> 60 #include <sys/disklabel.h> 61 #include <sys/ioctl.h> 62 #include <sys/fcntl.h> 63 #include <sys/conf.h> 64 65 #include <miscfs/specfs/specdev.h> 66 67 #include <uvm/uvm.h> 68 #include <uvm/uvm_vnode.h> 69 70 /* 71 * private global data structure 72 * 73 * we keep a list of writeable active vnode-backed VM objects for sync op. 74 * we keep a simpleq of vnodes that are currently being sync'd. 75 */ 76 77 LIST_HEAD(uvn_list_struct, uvm_vnode); 78 static struct uvn_list_struct uvn_wlist; /* writeable uvns */ 79 static simple_lock_data_t uvn_wl_lock; /* locks uvn_wlist */ 80 81 SIMPLEQ_HEAD(uvn_sq_struct, uvm_vnode); 82 static struct uvn_sq_struct uvn_sync_q; /* sync'ing uvns */ 83 lock_data_t uvn_sync_lock; /* locks sync operation */ 84 85 /* 86 * functions 87 */ 88 89 static int uvn_asyncget __P((struct uvm_object *, voff_t, 90 int)); 91 static void uvn_cluster __P((struct uvm_object *, voff_t, 92 voff_t *, voff_t *)); 93 static void uvn_detach __P((struct uvm_object *)); 94 static boolean_t uvn_flush __P((struct uvm_object *, voff_t, 95 voff_t, int)); 96 static int uvn_get __P((struct uvm_object *, voff_t, 97 vm_page_t *, int *, int, 98 vm_prot_t, int, int)); 99 static void uvn_init __P((void)); 100 static int uvn_io __P((struct uvm_vnode *, vm_page_t *, 101 int, int, int)); 102 static int uvn_put __P((struct uvm_object *, vm_page_t *, 103 int, boolean_t)); 104 static void uvn_reference __P((struct uvm_object *)); 105 static boolean_t uvn_releasepg __P((struct vm_page *, 106 struct vm_page **)); 107 108 /* 109 * master pager structure 110 */ 111 112 struct uvm_pagerops uvm_vnodeops = { 113 uvn_init, 114 uvn_reference, 115 uvn_detach, 116 NULL, /* no specialized fault routine required */ 117 uvn_flush, 118 uvn_get, 119 uvn_asyncget, 120 uvn_put, 121 uvn_cluster, 122 uvm_mk_pcluster, /* use generic version of this: see uvm_pager.c */ 123 NULL, /* AIO-DONE function (not until we have asyncio) */ 124 uvn_releasepg, 125 }; 126 127 /* 128 * the ops! 129 */ 130 131 /* 132 * uvn_init 133 * 134 * init pager private data structures. 135 */ 136 137 static void 138 uvn_init() 139 { 140 141 LIST_INIT(&uvn_wlist); 142 simple_lock_init(&uvn_wl_lock); 143 /* note: uvn_sync_q init'd in uvm_vnp_sync() */ 144 lockinit(&uvn_sync_lock, PVM, "uvnsync", 0, 0); 145 } 146 147 /* 148 * uvn_attach 149 * 150 * attach a vnode structure to a VM object. if the vnode is already 151 * attached, then just bump the reference count by one and return the 152 * VM object. if not already attached, attach and return the new VM obj. 153 * the "accessprot" tells the max access the attaching thread wants to 154 * our pages. 155 * 156 * => caller must _not_ already be holding the lock on the uvm_object. 157 * => in fact, nothing should be locked so that we can sleep here. 158 * => note that uvm_object is first thing in vnode structure, so their 159 * pointers are equiv. 160 */ 161 162 struct uvm_object * 163 uvn_attach(arg, accessprot) 164 void *arg; 165 vm_prot_t accessprot; 166 { 167 struct vnode *vp = arg; 168 struct uvm_vnode *uvn = &vp->v_uvm; 169 struct vattr vattr; 170 int oldflags, result; 171 struct partinfo pi; 172 u_quad_t used_vnode_size; 173 UVMHIST_FUNC("uvn_attach"); UVMHIST_CALLED(maphist); 174 175 UVMHIST_LOG(maphist, "(vn=0x%x)", arg,0,0,0); 176 177 used_vnode_size = (u_quad_t)0; /* XXX gcc -Wuninitialized */ 178 179 /* 180 * first get a lock on the uvn. 181 */ 182 simple_lock(&uvn->u_obj.vmobjlock); 183 while (uvn->u_flags & UVM_VNODE_BLOCKED) { 184 printf("uvn_attach: blocked at 0x%p flags 0x%x\n", 185 uvn, uvn->u_flags); 186 uvn->u_flags |= UVM_VNODE_WANTED; 187 UVMHIST_LOG(maphist, " SLEEPING on blocked vn",0,0,0,0); 188 UVM_UNLOCK_AND_WAIT(uvn, &uvn->u_obj.vmobjlock, FALSE, 189 "uvn_attach", 0); 190 simple_lock(&uvn->u_obj.vmobjlock); 191 UVMHIST_LOG(maphist," WOKE UP",0,0,0,0); 192 } 193 194 /* 195 * if we're mapping a BLK device, make sure it is a disk. 196 */ 197 if (vp->v_type == VBLK && bdevsw[major(vp->v_rdev)].d_type != D_DISK) { 198 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock */ 199 UVMHIST_LOG(maphist,"<- done (VBLK not D_DISK!)", 0,0,0,0); 200 return(NULL); 201 } 202 203 /* 204 * now we have lock and uvn must not be in a blocked state. 205 * first check to see if it is already active, in which case 206 * we can bump the reference count, check to see if we need to 207 * add it to the writeable list, and then return. 208 */ 209 if (uvn->u_flags & UVM_VNODE_VALID) { /* already active? */ 210 211 /* regain VREF if we were persisting */ 212 if (uvn->u_obj.uo_refs == 0) { 213 VREF(vp); 214 UVMHIST_LOG(maphist," VREF (reclaim persisting vnode)", 215 0,0,0,0); 216 } 217 uvn->u_obj.uo_refs++; /* bump uvn ref! */ 218 219 /* check for new writeable uvn */ 220 if ((accessprot & VM_PROT_WRITE) != 0 && 221 (uvn->u_flags & UVM_VNODE_WRITEABLE) == 0) { 222 simple_lock(&uvn_wl_lock); 223 LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist); 224 simple_unlock(&uvn_wl_lock); 225 /* we are now on wlist! */ 226 uvn->u_flags |= UVM_VNODE_WRITEABLE; 227 } 228 229 /* unlock and return */ 230 simple_unlock(&uvn->u_obj.vmobjlock); 231 UVMHIST_LOG(maphist,"<- done, refcnt=%d", uvn->u_obj.uo_refs, 232 0, 0, 0); 233 return (&uvn->u_obj); 234 } 235 236 /* 237 * need to call VOP_GETATTR() to get the attributes, but that could 238 * block (due to I/O), so we want to unlock the object before calling. 239 * however, we want to keep anyone else from playing with the object 240 * while it is unlocked. to do this we set UVM_VNODE_ALOCK which 241 * prevents anyone from attaching to the vnode until we are done with 242 * it. 243 */ 244 uvn->u_flags = UVM_VNODE_ALOCK; 245 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock in case we sleep */ 246 /* XXX: curproc? */ 247 248 if (vp->v_type == VBLK) { 249 /* 250 * We could implement this as a specfs getattr call, but: 251 * 252 * (1) VOP_GETATTR() would get the file system 253 * vnode operation, not the specfs operation. 254 * 255 * (2) All we want is the size, anyhow. 256 */ 257 result = (*bdevsw[major(vp->v_rdev)].d_ioctl)(vp->v_rdev, 258 DIOCGPART, (caddr_t)&pi, FREAD, curproc); 259 if (result == 0) { 260 /* XXX should remember blocksize */ 261 used_vnode_size = (u_quad_t)pi.disklab->d_secsize * 262 (u_quad_t)pi.part->p_size; 263 } 264 } else { 265 result = VOP_GETATTR(vp, &vattr, curproc->p_ucred, curproc); 266 if (result == 0) 267 used_vnode_size = vattr.va_size; 268 } 269 270 /* relock object */ 271 simple_lock(&uvn->u_obj.vmobjlock); 272 273 if (result != 0) { 274 if (uvn->u_flags & UVM_VNODE_WANTED) 275 wakeup(uvn); 276 uvn->u_flags = 0; 277 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock */ 278 UVMHIST_LOG(maphist,"<- done (VOP_GETATTR FAILED!)", 0,0,0,0); 279 return(NULL); 280 } 281 282 /* 283 * make sure that the newsize fits within a vaddr_t 284 * XXX: need to revise addressing data types 285 */ 286 #ifdef DEBUG 287 if (vp->v_type == VBLK) 288 printf("used_vnode_size = %qu\n", (long long)used_vnode_size); 289 #endif 290 291 /* 292 * now set up the uvn. 293 */ 294 uvn->u_obj.pgops = &uvm_vnodeops; 295 TAILQ_INIT(&uvn->u_obj.memq); 296 uvn->u_obj.uo_npages = 0; 297 uvn->u_obj.uo_refs = 1; /* just us... */ 298 oldflags = uvn->u_flags; 299 uvn->u_flags = UVM_VNODE_VALID|UVM_VNODE_CANPERSIST; 300 uvn->u_nio = 0; 301 uvn->u_size = used_vnode_size; 302 303 /* if write access, we need to add it to the wlist */ 304 if (accessprot & VM_PROT_WRITE) { 305 simple_lock(&uvn_wl_lock); 306 LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist); 307 simple_unlock(&uvn_wl_lock); 308 uvn->u_flags |= UVM_VNODE_WRITEABLE; /* we are on wlist! */ 309 } 310 311 /* 312 * add a reference to the vnode. this reference will stay as long 313 * as there is a valid mapping of the vnode. dropped when the 314 * reference count goes to zero [and we either free or persist]. 315 */ 316 VREF(vp); 317 simple_unlock(&uvn->u_obj.vmobjlock); 318 if (oldflags & UVM_VNODE_WANTED) 319 wakeup(uvn); 320 321 UVMHIST_LOG(maphist,"<- done/VREF, ret 0x%x", &uvn->u_obj,0,0,0); 322 return(&uvn->u_obj); 323 } 324 325 326 /* 327 * uvn_reference 328 * 329 * duplicate a reference to a VM object. Note that the reference 330 * count must already be at least one (the passed in reference) so 331 * there is no chance of the uvn being killed or locked out here. 332 * 333 * => caller must call with object unlocked. 334 * => caller must be using the same accessprot as was used at attach time 335 */ 336 337 338 static void 339 uvn_reference(uobj) 340 struct uvm_object *uobj; 341 { 342 #ifdef DEBUG 343 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 344 #endif 345 UVMHIST_FUNC("uvn_reference"); UVMHIST_CALLED(maphist); 346 347 simple_lock(&uobj->vmobjlock); 348 #ifdef DEBUG 349 if ((uvn->u_flags & UVM_VNODE_VALID) == 0) { 350 printf("uvn_reference: ref=%d, flags=0x%x\n", uvn->u_flags, 351 uobj->uo_refs); 352 panic("uvn_reference: invalid state"); 353 } 354 #endif 355 uobj->uo_refs++; 356 UVMHIST_LOG(maphist, "<- done (uobj=0x%x, ref = %d)", 357 uobj, uobj->uo_refs,0,0); 358 simple_unlock(&uobj->vmobjlock); 359 } 360 361 /* 362 * uvn_detach 363 * 364 * remove a reference to a VM object. 365 * 366 * => caller must call with object unlocked and map locked. 367 * => this starts the detach process, but doesn't have to finish it 368 * (async i/o could still be pending). 369 */ 370 static void 371 uvn_detach(uobj) 372 struct uvm_object *uobj; 373 { 374 struct uvm_vnode *uvn; 375 struct vnode *vp; 376 int oldflags; 377 UVMHIST_FUNC("uvn_detach"); UVMHIST_CALLED(maphist); 378 379 simple_lock(&uobj->vmobjlock); 380 381 UVMHIST_LOG(maphist," (uobj=0x%x) ref=%d", uobj,uobj->uo_refs,0,0); 382 uobj->uo_refs--; /* drop ref! */ 383 if (uobj->uo_refs) { /* still more refs */ 384 simple_unlock(&uobj->vmobjlock); 385 UVMHIST_LOG(maphist, "<- done (rc>0)", 0,0,0,0); 386 return; 387 } 388 389 /* 390 * get other pointers ... 391 */ 392 393 uvn = (struct uvm_vnode *) uobj; 394 vp = (struct vnode *) uobj; 395 396 /* 397 * clear VTEXT flag now that there are no mappings left (VTEXT is used 398 * to keep an active text file from being overwritten). 399 */ 400 vp->v_flag &= ~VTEXT; 401 402 /* 403 * we just dropped the last reference to the uvn. see if we can 404 * let it "stick around". 405 */ 406 407 if (uvn->u_flags & UVM_VNODE_CANPERSIST) { 408 /* won't block */ 409 uvn_flush(uobj, 0, 0, PGO_DEACTIVATE|PGO_ALLPAGES); 410 simple_unlock(&uobj->vmobjlock); 411 vrele(vp); /* drop vnode reference */ 412 UVMHIST_LOG(maphist,"<- done/vrele! (persist)", 0,0,0,0); 413 return; 414 } 415 416 /* 417 * its a goner! 418 */ 419 420 UVMHIST_LOG(maphist," its a goner (flushing)!", 0,0,0,0); 421 422 uvn->u_flags |= UVM_VNODE_DYING; 423 424 /* 425 * even though we may unlock in flush, no one can gain a reference 426 * to us until we clear the "dying" flag [because it blocks 427 * attaches]. we will not do that until after we've disposed of all 428 * the pages with uvn_flush(). note that before the flush the only 429 * pages that could be marked PG_BUSY are ones that are in async 430 * pageout by the daemon. (there can't be any pending "get"'s 431 * because there are no references to the object). 432 */ 433 434 (void) uvn_flush(uobj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES); 435 436 UVMHIST_LOG(maphist," its a goner (done flush)!", 0,0,0,0); 437 438 /* 439 * given the structure of this pager, the above flush request will 440 * create the following state: all the pages that were in the object 441 * have either been free'd or they are marked PG_BUSY|PG_RELEASED. 442 * the PG_BUSY bit was set either by us or the daemon for async I/O. 443 * in either case, if we have pages left we can't kill the object 444 * yet because i/o is pending. in this case we set the "relkill" 445 * flag which will cause pgo_releasepg to kill the object once all 446 * the I/O's are done [pgo_releasepg will be called from the aiodone 447 * routine or from the page daemon]. 448 */ 449 450 if (uobj->uo_npages) { /* I/O pending. iodone will free */ 451 #ifdef DEBUG 452 /* 453 * XXXCDC: very unlikely to happen until we have async i/o 454 * so print a little info message in case it does. 455 */ 456 printf("uvn_detach: vn %p has pages left after flush - " 457 "relkill mode\n", uobj); 458 #endif 459 uvn->u_flags |= UVM_VNODE_RELKILL; 460 simple_unlock(&uobj->vmobjlock); 461 UVMHIST_LOG(maphist,"<- done! (releasepg will kill obj)", 0, 0, 462 0, 0); 463 return; 464 } 465 466 /* 467 * kill object now. note that we can't be on the sync q because 468 * all references are gone. 469 */ 470 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 471 simple_lock(&uvn_wl_lock); /* protect uvn_wlist */ 472 LIST_REMOVE(uvn, u_wlist); 473 simple_unlock(&uvn_wl_lock); 474 } 475 #ifdef DIAGNOSTIC 476 if (uobj->memq.tqh_first != NULL) 477 panic("uvn_deref: vnode VM object still has pages afer " 478 "syncio/free flush"); 479 #endif 480 oldflags = uvn->u_flags; 481 uvn->u_flags = 0; 482 simple_unlock(&uobj->vmobjlock); 483 484 /* wake up any sleepers */ 485 if (oldflags & UVM_VNODE_WANTED) 486 wakeup(uvn); 487 488 /* 489 * drop our reference to the vnode. 490 */ 491 vrele(vp); 492 UVMHIST_LOG(maphist,"<- done (vrele) final", 0,0,0,0); 493 494 return; 495 } 496 497 /* 498 * uvm_vnp_terminate: external hook to clear out a vnode's VM 499 * 500 * called in two cases: 501 * [1] when a persisting vnode vm object (i.e. one with a zero reference 502 * count) needs to be freed so that a vnode can be reused. this 503 * happens under "getnewvnode" in vfs_subr.c. if the vnode from 504 * the free list is still attached (i.e. not VBAD) then vgone is 505 * called. as part of the vgone trace this should get called to 506 * free the vm object. this is the common case. 507 * [2] when a filesystem is being unmounted by force (MNT_FORCE, 508 * "umount -f") the vgone() function is called on active vnodes 509 * on the mounted file systems to kill their data (the vnodes become 510 * "dead" ones [see src/sys/miscfs/deadfs/...]). that results in a 511 * call here (even if the uvn is still in use -- i.e. has a non-zero 512 * reference count). this case happens at "umount -f" and during a 513 * "reboot/halt" operation. 514 * 515 * => the caller must XLOCK and VOP_LOCK the vnode before calling us 516 * [protects us from getting a vnode that is already in the DYING 517 * state...] 518 * => unlike uvn_detach, this function must not return until all the 519 * uvn's pages are disposed of. 520 * => in case [2] the uvn is still alive after this call, but all I/O 521 * ops will fail (due to the backing vnode now being "dead"). this 522 * will prob. kill any process using the uvn due to pgo_get failing. 523 */ 524 525 void 526 uvm_vnp_terminate(vp) 527 struct vnode *vp; 528 { 529 struct uvm_vnode *uvn = &vp->v_uvm; 530 int oldflags; 531 UVMHIST_FUNC("uvm_vnp_terminate"); UVMHIST_CALLED(maphist); 532 533 /* 534 * lock object and check if it is valid 535 */ 536 simple_lock(&uvn->u_obj.vmobjlock); 537 UVMHIST_LOG(maphist, " vp=0x%x, ref=%d, flag=0x%x", vp, 538 uvn->u_obj.uo_refs, uvn->u_flags, 0); 539 if ((uvn->u_flags & UVM_VNODE_VALID) == 0) { 540 simple_unlock(&uvn->u_obj.vmobjlock); 541 UVMHIST_LOG(maphist, "<- done (not active)", 0, 0, 0, 0); 542 return; 543 } 544 545 /* 546 * must be a valid uvn that is not already dying (because XLOCK 547 * protects us from that). the uvn can't in the ALOCK state 548 * because it is valid, and uvn's that are in the ALOCK state haven't 549 * been marked valid yet. 550 */ 551 552 #ifdef DEBUG 553 /* 554 * debug check: are we yanking the vnode out from under our uvn? 555 */ 556 if (uvn->u_obj.uo_refs) { 557 printf("uvm_vnp_terminate(%p): terminating active vnode " 558 "(refs=%d)\n", uvn, uvn->u_obj.uo_refs); 559 } 560 #endif 561 562 /* 563 * it is possible that the uvn was detached and is in the relkill 564 * state [i.e. waiting for async i/o to finish so that releasepg can 565 * kill object]. we take over the vnode now and cancel the relkill. 566 * we want to know when the i/o is done so we can recycle right 567 * away. note that a uvn can only be in the RELKILL state if it 568 * has a zero reference count. 569 */ 570 571 if (uvn->u_flags & UVM_VNODE_RELKILL) 572 uvn->u_flags &= ~UVM_VNODE_RELKILL; /* cancel RELKILL */ 573 574 /* 575 * block the uvn by setting the dying flag, and then flush the 576 * pages. (note that flush may unlock object while doing I/O, but 577 * it will re-lock it before it returns control here). 578 * 579 * also, note that we tell I/O that we are already VOP_LOCK'd so 580 * that uvn_io doesn't attempt to VOP_LOCK again. 581 * 582 * XXXCDC: setting VNISLOCKED on an active uvn which is being terminated 583 * due to a forceful unmount might not be a good idea. maybe we 584 * need a way to pass in this info to uvn_flush through a 585 * pager-defined PGO_ constant [currently there are none]. 586 */ 587 uvn->u_flags |= UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED; 588 589 (void) uvn_flush(&uvn->u_obj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES); 590 591 /* 592 * as we just did a flush we expect all the pages to be gone or in 593 * the process of going. sleep to wait for the rest to go [via iosync]. 594 */ 595 596 while (uvn->u_obj.uo_npages) { 597 #ifdef DEBUG 598 struct vm_page *pp; 599 for (pp = uvn->u_obj.memq.tqh_first ; pp != NULL ; 600 pp = pp->listq.tqe_next) { 601 if ((pp->flags & PG_BUSY) == 0) 602 panic("uvm_vnp_terminate: detected unbusy pg"); 603 } 604 if (uvn->u_nio == 0) 605 panic("uvm_vnp_terminate: no I/O to wait for?"); 606 printf("uvm_vnp_terminate: waiting for I/O to fin.\n"); 607 /* 608 * XXXCDC: this is unlikely to happen without async i/o so we 609 * put a printf in just to keep an eye on it. 610 */ 611 #endif 612 uvn->u_flags |= UVM_VNODE_IOSYNC; 613 UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock, FALSE, 614 "uvn_term",0); 615 simple_lock(&uvn->u_obj.vmobjlock); 616 } 617 618 /* 619 * done. now we free the uvn if its reference count is zero 620 * (true if we are zapping a persisting uvn). however, if we are 621 * terminating a uvn with active mappings we let it live ... future 622 * calls down to the vnode layer will fail. 623 */ 624 625 oldflags = uvn->u_flags; 626 if (uvn->u_obj.uo_refs) { 627 628 /* 629 * uvn must live on it is dead-vnode state until all references 630 * are gone. restore flags. clear CANPERSIST state. 631 */ 632 633 uvn->u_flags &= ~(UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED| 634 UVM_VNODE_WANTED|UVM_VNODE_CANPERSIST); 635 636 } else { 637 638 /* 639 * free the uvn now. note that the VREF reference is already 640 * gone [it is dropped when we enter the persist state]. 641 */ 642 if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED) 643 panic("uvm_vnp_terminate: io sync wanted bit set"); 644 645 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 646 simple_lock(&uvn_wl_lock); 647 LIST_REMOVE(uvn, u_wlist); 648 simple_unlock(&uvn_wl_lock); 649 } 650 uvn->u_flags = 0; /* uvn is history, clear all bits */ 651 } 652 653 if (oldflags & UVM_VNODE_WANTED) 654 wakeup(uvn); /* object lock still held */ 655 656 simple_unlock(&uvn->u_obj.vmobjlock); 657 UVMHIST_LOG(maphist, "<- done", 0, 0, 0, 0); 658 659 } 660 661 /* 662 * uvn_releasepg: handled a released page in a uvn 663 * 664 * => "pg" is a PG_BUSY [caller owns it], PG_RELEASED page that we need 665 * to dispose of. 666 * => caller must handled PG_WANTED case 667 * => called with page's object locked, pageq's unlocked 668 * => returns TRUE if page's object is still alive, FALSE if we 669 * killed the page's object. if we return TRUE, then we 670 * return with the object locked. 671 * => if (nextpgp != NULL) => we return pageq.tqe_next here, and return 672 * with the page queues locked [for pagedaemon] 673 * => if (nextpgp == NULL) => we return with page queues unlocked [normal case] 674 * => we kill the uvn if it is not referenced and we are suppose to 675 * kill it ("relkill"). 676 */ 677 678 boolean_t 679 uvn_releasepg(pg, nextpgp) 680 struct vm_page *pg; 681 struct vm_page **nextpgp; /* OUT */ 682 { 683 struct uvm_vnode *uvn = (struct uvm_vnode *) pg->uobject; 684 #ifdef DIAGNOSTIC 685 if ((pg->flags & PG_RELEASED) == 0) 686 panic("uvn_releasepg: page not released!"); 687 #endif 688 689 /* 690 * dispose of the page [caller handles PG_WANTED] 691 */ 692 pmap_page_protect(pg, VM_PROT_NONE); 693 uvm_lock_pageq(); 694 if (nextpgp) 695 *nextpgp = pg->pageq.tqe_next; /* next page for daemon */ 696 uvm_pagefree(pg); 697 if (!nextpgp) 698 uvm_unlock_pageq(); 699 700 /* 701 * now see if we need to kill the object 702 */ 703 if (uvn->u_flags & UVM_VNODE_RELKILL) { 704 if (uvn->u_obj.uo_refs) 705 panic("uvn_releasepg: kill flag set on referenced " 706 "object!"); 707 if (uvn->u_obj.uo_npages == 0) { 708 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 709 simple_lock(&uvn_wl_lock); 710 LIST_REMOVE(uvn, u_wlist); 711 simple_unlock(&uvn_wl_lock); 712 } 713 #ifdef DIAGNOSTIC 714 if (uvn->u_obj.memq.tqh_first) 715 panic("uvn_releasepg: pages in object with npages == 0"); 716 #endif 717 if (uvn->u_flags & UVM_VNODE_WANTED) 718 /* still holding object lock */ 719 wakeup(uvn); 720 721 uvn->u_flags = 0; /* DEAD! */ 722 simple_unlock(&uvn->u_obj.vmobjlock); 723 return (FALSE); 724 } 725 } 726 return (TRUE); 727 } 728 729 /* 730 * NOTE: currently we have to use VOP_READ/VOP_WRITE because they go 731 * through the buffer cache and allow I/O in any size. These VOPs use 732 * synchronous i/o. [vs. VOP_STRATEGY which can be async, but doesn't 733 * go through the buffer cache or allow I/O sizes larger than a 734 * block]. we will eventually want to change this. 735 * 736 * issues to consider: 737 * uvm provides the uvm_aiodesc structure for async i/o management. 738 * there are two tailq's in the uvm. structure... one for pending async 739 * i/o and one for "done" async i/o. to do an async i/o one puts 740 * an aiodesc on the "pending" list (protected by splbio()), starts the 741 * i/o and returns VM_PAGER_PEND. when the i/o is done, we expect 742 * some sort of "i/o done" function to be called (at splbio(), interrupt 743 * time). this function should remove the aiodesc from the pending list 744 * and place it on the "done" list and wakeup the daemon. the daemon 745 * will run at normal spl() and will remove all items from the "done" 746 * list and call the "aiodone" hook for each done request (see uvm_pager.c). 747 * [in the old vm code, this was done by calling the "put" routine with 748 * null arguments which made the code harder to read and understand because 749 * you had one function ("put") doing two things.] 750 * 751 * so the current pager needs: 752 * int uvn_aiodone(struct uvm_aiodesc *) 753 * 754 * => return KERN_SUCCESS (aio finished, free it). otherwise requeue for 755 * later collection. 756 * => called with pageq's locked by the daemon. 757 * 758 * general outline: 759 * - "try" to lock object. if fail, just return (will try again later) 760 * - drop "u_nio" (this req is done!) 761 * - if (object->iosync && u_naio == 0) { wakeup &uvn->u_naio } 762 * - get "page" structures (atop?). 763 * - handle "wanted" pages 764 * - handle "released" pages [using pgo_releasepg] 765 * >>> pgo_releasepg may kill the object 766 * dont forget to look at "object" wanted flag in all cases. 767 */ 768 769 770 /* 771 * uvn_flush: flush pages out of a uvm object. 772 * 773 * => object should be locked by caller. we may _unlock_ the object 774 * if (and only if) we need to clean a page (PGO_CLEANIT). 775 * we return with the object locked. 776 * => if PGO_CLEANIT is set, we may block (due to I/O). thus, a caller 777 * might want to unlock higher level resources (e.g. vm_map) 778 * before calling flush. 779 * => if PGO_CLEANIT is not set, then we will neither unlock the object 780 * or block. 781 * => if PGO_ALLPAGE is set, then all pages in the object are valid targets 782 * for flushing. 783 * => NOTE: we rely on the fact that the object's memq is a TAILQ and 784 * that new pages are inserted on the tail end of the list. thus, 785 * we can make a complete pass through the object in one go by starting 786 * at the head and working towards the tail (new pages are put in 787 * front of us). 788 * => NOTE: we are allowed to lock the page queues, so the caller 789 * must not be holding the lock on them [e.g. pagedaemon had 790 * better not call us with the queues locked] 791 * => we return TRUE unless we encountered some sort of I/O error 792 * 793 * comment on "cleaning" object and PG_BUSY pages: 794 * this routine is holding the lock on the object. the only time 795 * that it can run into a PG_BUSY page that it does not own is if 796 * some other process has started I/O on the page (e.g. either 797 * a pagein, or a pageout). if the PG_BUSY page is being paged 798 * in, then it can not be dirty (!PG_CLEAN) because no one has 799 * had a chance to modify it yet. if the PG_BUSY page is being 800 * paged out then it means that someone else has already started 801 * cleaning the page for us (how nice!). in this case, if we 802 * have syncio specified, then after we make our pass through the 803 * object we need to wait for the other PG_BUSY pages to clear 804 * off (i.e. we need to do an iosync). also note that once a 805 * page is PG_BUSY it must stay in its object until it is un-busyed. 806 * 807 * note on page traversal: 808 * we can traverse the pages in an object either by going down the 809 * linked list in "uobj->memq", or we can go over the address range 810 * by page doing hash table lookups for each address. depending 811 * on how many pages are in the object it may be cheaper to do one 812 * or the other. we set "by_list" to true if we are using memq. 813 * if the cost of a hash lookup was equal to the cost of the list 814 * traversal we could compare the number of pages in the start->stop 815 * range to the total number of pages in the object. however, it 816 * seems that a hash table lookup is more expensive than the linked 817 * list traversal, so we multiply the number of pages in the 818 * start->stop range by a penalty which we define below. 819 */ 820 821 #define UVN_HASH_PENALTY 4 /* XXX: a guess */ 822 823 static boolean_t 824 uvn_flush(uobj, start, stop, flags) 825 struct uvm_object *uobj; 826 voff_t start, stop; 827 int flags; 828 { 829 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 830 struct vm_page *pp, *ppnext, *ptmp; 831 struct vm_page *pps[MAXBSIZE >> PAGE_SHIFT], **ppsp; 832 int npages, result, lcv; 833 boolean_t retval, need_iosync, by_list, needs_clean, all; 834 voff_t curoff; 835 u_short pp_version; 836 UVMHIST_FUNC("uvn_flush"); UVMHIST_CALLED(maphist); 837 838 curoff = 0; /* XXX: shut up gcc */ 839 /* 840 * get init vals and determine how we are going to traverse object 841 */ 842 843 need_iosync = FALSE; 844 retval = TRUE; /* return value */ 845 if (flags & PGO_ALLPAGES) { 846 all = TRUE; 847 by_list = TRUE; /* always go by the list */ 848 } else { 849 start = trunc_page(start); 850 stop = round_page(stop); 851 #ifdef DEBUG 852 if (stop > round_page(uvn->u_size)) 853 printf("uvn_flush: strange, got an out of range " 854 "flush (fixed)\n"); 855 #endif 856 all = FALSE; 857 by_list = (uobj->uo_npages <= 858 ((stop - start) >> PAGE_SHIFT) * UVN_HASH_PENALTY); 859 } 860 861 UVMHIST_LOG(maphist, 862 " flush start=0x%x, stop=0x%x, by_list=%d, flags=0x%x", 863 start, stop, by_list, flags); 864 865 /* 866 * PG_CLEANCHK: this bit is used by the pgo_mk_pcluster function as 867 * a _hint_ as to how up to date the PG_CLEAN bit is. if the hint 868 * is wrong it will only prevent us from clustering... it won't break 869 * anything. we clear all PG_CLEANCHK bits here, and pgo_mk_pcluster 870 * will set them as it syncs PG_CLEAN. This is only an issue if we 871 * are looking at non-inactive pages (because inactive page's PG_CLEAN 872 * bit is always up to date since there are no mappings). 873 * [borrowed PG_CLEANCHK idea from FreeBSD VM] 874 */ 875 876 if ((flags & PGO_CLEANIT) != 0 && 877 uobj->pgops->pgo_mk_pcluster != NULL) { 878 if (by_list) { 879 for (pp = uobj->memq.tqh_first ; pp != NULL ; 880 pp = pp->listq.tqe_next) { 881 if (!all && 882 (pp->offset < start || pp->offset >= stop)) 883 continue; 884 pp->flags &= ~PG_CLEANCHK; 885 } 886 887 } else { /* by hash */ 888 for (curoff = start ; curoff < stop; 889 curoff += PAGE_SIZE) { 890 pp = uvm_pagelookup(uobj, curoff); 891 if (pp) 892 pp->flags &= ~PG_CLEANCHK; 893 } 894 } 895 } 896 897 /* 898 * now do it. note: we must update ppnext in body of loop or we 899 * will get stuck. we need to use ppnext because we may free "pp" 900 * before doing the next loop. 901 */ 902 903 if (by_list) { 904 pp = uobj->memq.tqh_first; 905 } else { 906 curoff = start; 907 pp = uvm_pagelookup(uobj, curoff); 908 } 909 910 ppnext = NULL; /* XXX: shut up gcc */ 911 ppsp = NULL; /* XXX: shut up gcc */ 912 uvm_lock_pageq(); /* page queues locked */ 913 914 /* locked: both page queues and uobj */ 915 for ( ; (by_list && pp != NULL) || 916 (!by_list && curoff < stop) ; pp = ppnext) { 917 918 if (by_list) { 919 920 /* 921 * range check 922 */ 923 924 if (!all && 925 (pp->offset < start || pp->offset >= stop)) { 926 ppnext = pp->listq.tqe_next; 927 continue; 928 } 929 930 } else { 931 932 /* 933 * null check 934 */ 935 936 curoff += PAGE_SIZE; 937 if (pp == NULL) { 938 if (curoff < stop) 939 ppnext = uvm_pagelookup(uobj, curoff); 940 continue; 941 } 942 943 } 944 945 /* 946 * handle case where we do not need to clean page (either 947 * because we are not clean or because page is not dirty or 948 * is busy): 949 * 950 * NOTE: we are allowed to deactivate a non-wired active 951 * PG_BUSY page, but once a PG_BUSY page is on the inactive 952 * queue it must stay put until it is !PG_BUSY (so as not to 953 * confuse pagedaemon). 954 */ 955 956 if ((flags & PGO_CLEANIT) == 0 || (pp->flags & PG_BUSY) != 0) { 957 needs_clean = FALSE; 958 if ((pp->flags & PG_BUSY) != 0 && 959 (flags & (PGO_CLEANIT|PGO_SYNCIO)) == 960 (PGO_CLEANIT|PGO_SYNCIO)) 961 need_iosync = TRUE; 962 } else { 963 /* 964 * freeing: nuke all mappings so we can sync 965 * PG_CLEAN bit with no race 966 */ 967 if ((pp->flags & PG_CLEAN) != 0 && 968 (flags & PGO_FREE) != 0 && 969 (pp->pqflags & PQ_ACTIVE) != 0) 970 pmap_page_protect(pp, VM_PROT_NONE); 971 if ((pp->flags & PG_CLEAN) != 0 && 972 pmap_is_modified(pp)) 973 pp->flags &= ~(PG_CLEAN); 974 pp->flags |= PG_CLEANCHK; /* update "hint" */ 975 976 needs_clean = ((pp->flags & PG_CLEAN) == 0); 977 } 978 979 /* 980 * if we don't need a clean... load ppnext and dispose of pp 981 */ 982 if (!needs_clean) { 983 /* load ppnext */ 984 if (by_list) 985 ppnext = pp->listq.tqe_next; 986 else { 987 if (curoff < stop) 988 ppnext = uvm_pagelookup(uobj, curoff); 989 } 990 991 /* now dispose of pp */ 992 if (flags & PGO_DEACTIVATE) { 993 if ((pp->pqflags & PQ_INACTIVE) == 0 && 994 pp->wire_count == 0) { 995 pmap_page_protect(pp, VM_PROT_NONE); 996 uvm_pagedeactivate(pp); 997 } 998 999 } else if (flags & PGO_FREE) { 1000 if (pp->flags & PG_BUSY) { 1001 /* release busy pages */ 1002 pp->flags |= PG_RELEASED; 1003 } else { 1004 pmap_page_protect(pp, VM_PROT_NONE); 1005 /* removed page from object */ 1006 uvm_pagefree(pp); 1007 } 1008 } 1009 /* ppnext is valid so we can continue... */ 1010 continue; 1011 } 1012 1013 /* 1014 * pp points to a page in the locked object that we are 1015 * working on. if it is !PG_CLEAN,!PG_BUSY and we asked 1016 * for cleaning (PGO_CLEANIT). we clean it now. 1017 * 1018 * let uvm_pager_put attempted a clustered page out. 1019 * note: locked: uobj and page queues. 1020 */ 1021 1022 pp->flags |= PG_BUSY; /* we 'own' page now */ 1023 UVM_PAGE_OWN(pp, "uvn_flush"); 1024 pmap_page_protect(pp, VM_PROT_READ); 1025 pp_version = pp->version; 1026 ReTry: 1027 ppsp = pps; 1028 npages = sizeof(pps) / sizeof(struct vm_page *); 1029 1030 /* locked: page queues, uobj */ 1031 result = uvm_pager_put(uobj, pp, &ppsp, &npages, 1032 flags | PGO_DOACTCLUST, start, stop); 1033 /* unlocked: page queues, uobj */ 1034 1035 /* 1036 * at this point nothing is locked. if we did an async I/O 1037 * it is remotely possible for the async i/o to complete and 1038 * the page "pp" be freed or what not before we get a chance 1039 * to relock the object. in order to detect this, we have 1040 * saved the version number of the page in "pp_version". 1041 */ 1042 1043 /* relock! */ 1044 simple_lock(&uobj->vmobjlock); 1045 uvm_lock_pageq(); 1046 1047 /* 1048 * VM_PAGER_AGAIN: given the structure of this pager, this 1049 * can only happen when we are doing async I/O and can't 1050 * map the pages into kernel memory (pager_map) due to lack 1051 * of vm space. if this happens we drop back to sync I/O. 1052 */ 1053 1054 if (result == VM_PAGER_AGAIN) { 1055 /* 1056 * it is unlikely, but page could have been released 1057 * while we had the object lock dropped. we ignore 1058 * this now and retry the I/O. we will detect and 1059 * handle the released page after the syncio I/O 1060 * completes. 1061 */ 1062 #ifdef DIAGNOSTIC 1063 if (flags & PGO_SYNCIO) 1064 panic("uvn_flush: PGO_SYNCIO return 'try again' error (impossible)"); 1065 #endif 1066 flags |= PGO_SYNCIO; 1067 goto ReTry; 1068 } 1069 1070 /* 1071 * the cleaning operation is now done. finish up. note that 1072 * on error (!OK, !PEND) uvm_pager_put drops the cluster for us. 1073 * if success (OK, PEND) then uvm_pager_put returns the cluster 1074 * to us in ppsp/npages. 1075 */ 1076 1077 /* 1078 * for pending async i/o if we are not deactivating/freeing 1079 * we can move on to the next page. 1080 */ 1081 1082 if (result == VM_PAGER_PEND) { 1083 1084 if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) { 1085 /* 1086 * no per-page ops: refresh ppnext and continue 1087 */ 1088 if (by_list) { 1089 if (pp->version == pp_version) 1090 ppnext = pp->listq.tqe_next; 1091 else 1092 /* reset */ 1093 ppnext = uobj->memq.tqh_first; 1094 } else { 1095 if (curoff < stop) 1096 ppnext = uvm_pagelookup(uobj, 1097 curoff); 1098 } 1099 continue; 1100 } 1101 1102 /* need to do anything here? */ 1103 } 1104 1105 /* 1106 * need to look at each page of the I/O operation. we defer 1107 * processing "pp" until the last trip through this "for" loop 1108 * so that we can load "ppnext" for the main loop after we 1109 * play with the cluster pages [thus the "npages + 1" in the 1110 * loop below]. 1111 */ 1112 1113 for (lcv = 0 ; lcv < npages + 1 ; lcv++) { 1114 1115 /* 1116 * handle ppnext for outside loop, and saving pp 1117 * until the end. 1118 */ 1119 if (lcv < npages) { 1120 if (ppsp[lcv] == pp) 1121 continue; /* skip pp until the end */ 1122 ptmp = ppsp[lcv]; 1123 } else { 1124 ptmp = pp; 1125 1126 /* set up next page for outer loop */ 1127 if (by_list) { 1128 if (pp->version == pp_version) 1129 ppnext = pp->listq.tqe_next; 1130 else 1131 /* reset */ 1132 ppnext = uobj->memq.tqh_first; 1133 } else { 1134 if (curoff < stop) 1135 ppnext = uvm_pagelookup(uobj, curoff); 1136 } 1137 } 1138 1139 /* 1140 * verify the page didn't get moved while obj was 1141 * unlocked 1142 */ 1143 if (result == VM_PAGER_PEND && ptmp->uobject != uobj) 1144 continue; 1145 1146 /* 1147 * unbusy the page if I/O is done. note that for 1148 * pending I/O it is possible that the I/O op 1149 * finished before we relocked the object (in 1150 * which case the page is no longer busy). 1151 */ 1152 1153 if (result != VM_PAGER_PEND) { 1154 if (ptmp->flags & PG_WANTED) 1155 /* still holding object lock */ 1156 wakeup(ptmp); 1157 1158 ptmp->flags &= ~(PG_WANTED|PG_BUSY); 1159 UVM_PAGE_OWN(ptmp, NULL); 1160 if (ptmp->flags & PG_RELEASED) { 1161 1162 /* pgo_releasepg wants this */ 1163 uvm_unlock_pageq(); 1164 if (!uvn_releasepg(ptmp, NULL)) 1165 return (TRUE); 1166 1167 uvm_lock_pageq(); /* relock */ 1168 continue; /* next page */ 1169 1170 } else { 1171 ptmp->flags |= (PG_CLEAN|PG_CLEANCHK); 1172 if ((flags & PGO_FREE) == 0) 1173 pmap_clear_modify(ptmp); 1174 } 1175 } 1176 1177 /* 1178 * dispose of page 1179 */ 1180 1181 if (flags & PGO_DEACTIVATE) { 1182 if ((pp->pqflags & PQ_INACTIVE) == 0 && 1183 pp->wire_count == 0) { 1184 pmap_page_protect(ptmp, VM_PROT_NONE); 1185 uvm_pagedeactivate(ptmp); 1186 } 1187 1188 } else if (flags & PGO_FREE) { 1189 if (result == VM_PAGER_PEND) { 1190 if ((ptmp->flags & PG_BUSY) != 0) 1191 /* signal for i/o done */ 1192 ptmp->flags |= PG_RELEASED; 1193 } else { 1194 if (result != VM_PAGER_OK) { 1195 printf("uvn_flush: obj=%p, " 1196 "offset=0x%llx. error " 1197 "during pageout.\n", 1198 pp->uobject, 1199 (long long)pp->offset); 1200 printf("uvn_flush: WARNING: " 1201 "changes to page may be " 1202 "lost!\n"); 1203 retval = FALSE; 1204 } 1205 pmap_page_protect(ptmp, VM_PROT_NONE); 1206 uvm_pagefree(ptmp); 1207 } 1208 } 1209 1210 } /* end of "lcv" for loop */ 1211 1212 } /* end of "pp" for loop */ 1213 1214 /* 1215 * done with pagequeues: unlock 1216 */ 1217 uvm_unlock_pageq(); 1218 1219 /* 1220 * now wait for all I/O if required. 1221 */ 1222 if (need_iosync) { 1223 1224 UVMHIST_LOG(maphist," <<DOING IOSYNC>>",0,0,0,0); 1225 while (uvn->u_nio != 0) { 1226 uvn->u_flags |= UVM_VNODE_IOSYNC; 1227 UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock, 1228 FALSE, "uvn_flush",0); 1229 simple_lock(&uvn->u_obj.vmobjlock); 1230 } 1231 if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED) 1232 wakeup(&uvn->u_flags); 1233 uvn->u_flags &= ~(UVM_VNODE_IOSYNC|UVM_VNODE_IOSYNCWANTED); 1234 } 1235 1236 /* return, with object locked! */ 1237 UVMHIST_LOG(maphist,"<- done (retval=0x%x)",retval,0,0,0); 1238 return(retval); 1239 } 1240 1241 /* 1242 * uvn_cluster 1243 * 1244 * we are about to do I/O in an object at offset. this function is called 1245 * to establish a range of offsets around "offset" in which we can cluster 1246 * I/O. 1247 * 1248 * - currently doesn't matter if obj locked or not. 1249 */ 1250 1251 static void 1252 uvn_cluster(uobj, offset, loffset, hoffset) 1253 struct uvm_object *uobj; 1254 voff_t offset; 1255 voff_t *loffset, *hoffset; /* OUT */ 1256 { 1257 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 1258 *loffset = offset; 1259 1260 if (*loffset >= uvn->u_size) 1261 panic("uvn_cluster: offset out of range"); 1262 1263 /* 1264 * XXX: old pager claims we could use VOP_BMAP to get maxcontig value. 1265 */ 1266 *hoffset = *loffset + MAXBSIZE; 1267 if (*hoffset > round_page(uvn->u_size)) /* past end? */ 1268 *hoffset = round_page(uvn->u_size); 1269 1270 return; 1271 } 1272 1273 /* 1274 * uvn_put: flush page data to backing store. 1275 * 1276 * => prefer map unlocked (not required) 1277 * => object must be locked! we will _unlock_ it before starting I/O. 1278 * => flags: PGO_SYNCIO -- use sync. I/O 1279 * => note: caller must set PG_CLEAN and pmap_clear_modify (if needed) 1280 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync. 1281 * [thus we never do async i/o! see iodone comment] 1282 */ 1283 1284 static int 1285 uvn_put(uobj, pps, npages, flags) 1286 struct uvm_object *uobj; 1287 struct vm_page **pps; 1288 int npages, flags; 1289 { 1290 int retval; 1291 1292 /* note: object locked */ 1293 retval = uvn_io((struct uvm_vnode*)uobj, pps, npages, flags, UIO_WRITE); 1294 /* note: object unlocked */ 1295 1296 return(retval); 1297 } 1298 1299 1300 /* 1301 * uvn_get: get pages (synchronously) from backing store 1302 * 1303 * => prefer map unlocked (not required) 1304 * => object must be locked! we will _unlock_ it before starting any I/O. 1305 * => flags: PGO_ALLPAGES: get all of the pages 1306 * PGO_LOCKED: fault data structures are locked 1307 * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx] 1308 * => NOTE: caller must check for released pages!! 1309 */ 1310 1311 static int 1312 uvn_get(uobj, offset, pps, npagesp, centeridx, access_type, advice, flags) 1313 struct uvm_object *uobj; 1314 voff_t offset; 1315 struct vm_page **pps; /* IN/OUT */ 1316 int *npagesp; /* IN (OUT if PGO_LOCKED) */ 1317 int centeridx, advice, flags; 1318 vm_prot_t access_type; 1319 { 1320 voff_t current_offset; 1321 struct vm_page *ptmp; 1322 int lcv, result, gotpages; 1323 boolean_t done; 1324 UVMHIST_FUNC("uvn_get"); UVMHIST_CALLED(maphist); 1325 UVMHIST_LOG(maphist, "flags=%d", flags,0,0,0); 1326 1327 /* 1328 * step 1: handled the case where fault data structures are locked. 1329 */ 1330 1331 if (flags & PGO_LOCKED) { 1332 1333 /* 1334 * gotpages is the current number of pages we've gotten (which 1335 * we pass back up to caller via *npagesp. 1336 */ 1337 1338 gotpages = 0; 1339 1340 /* 1341 * step 1a: get pages that are already resident. only do this 1342 * if the data structures are locked (i.e. the first time 1343 * through). 1344 */ 1345 1346 done = TRUE; /* be optimistic */ 1347 1348 for (lcv = 0, current_offset = offset ; lcv < *npagesp ; 1349 lcv++, current_offset += PAGE_SIZE) { 1350 1351 /* do we care about this page? if not, skip it */ 1352 if (pps[lcv] == PGO_DONTCARE) 1353 continue; 1354 1355 /* lookup page */ 1356 ptmp = uvm_pagelookup(uobj, current_offset); 1357 1358 /* to be useful must get a non-busy, non-released pg */ 1359 if (ptmp == NULL || 1360 (ptmp->flags & (PG_BUSY|PG_RELEASED)) != 0) { 1361 if (lcv == centeridx || (flags & PGO_ALLPAGES) 1362 != 0) 1363 done = FALSE; /* need to do a wait or I/O! */ 1364 continue; 1365 } 1366 1367 /* 1368 * useful page: busy/lock it and plug it in our 1369 * result array 1370 */ 1371 ptmp->flags |= PG_BUSY; /* loan up to caller */ 1372 UVM_PAGE_OWN(ptmp, "uvn_get1"); 1373 pps[lcv] = ptmp; 1374 gotpages++; 1375 1376 } /* "for" lcv loop */ 1377 1378 /* 1379 * XXX: given the "advice", should we consider async read-ahead? 1380 * XXX: fault current does deactive of pages behind us. is 1381 * this good (other callers might now). 1382 */ 1383 /* 1384 * XXX: read-ahead currently handled by buffer cache (bread) 1385 * level. 1386 * XXX: no async i/o available. 1387 * XXX: so we don't do anything now. 1388 */ 1389 1390 /* 1391 * step 1c: now we've either done everything needed or we to 1392 * unlock and do some waiting or I/O. 1393 */ 1394 1395 *npagesp = gotpages; /* let caller know */ 1396 if (done) 1397 return(VM_PAGER_OK); /* bingo! */ 1398 else 1399 /* EEK! Need to unlock and I/O */ 1400 return(VM_PAGER_UNLOCK); 1401 } 1402 1403 /* 1404 * step 2: get non-resident or busy pages. 1405 * object is locked. data structures are unlocked. 1406 * 1407 * XXX: because we can't do async I/O at this level we get things 1408 * page at a time (otherwise we'd chunk). the VOP_READ() will do 1409 * async-read-ahead for us at a lower level. 1410 */ 1411 1412 for (lcv = 0, current_offset = offset ; 1413 lcv < *npagesp ; lcv++, current_offset += PAGE_SIZE) { 1414 1415 /* skip over pages we've already gotten or don't want */ 1416 /* skip over pages we don't _have_ to get */ 1417 if (pps[lcv] != NULL || (lcv != centeridx && 1418 (flags & PGO_ALLPAGES) == 0)) 1419 continue; 1420 1421 /* 1422 * we have yet to locate the current page (pps[lcv]). we first 1423 * look for a page that is already at the current offset. if 1424 * we fine a page, we check to see if it is busy or released. 1425 * if that is the case, then we sleep on the page until it is 1426 * no longer busy or released and repeat the lookup. if the 1427 * page we found is neither busy nor released, then we busy it 1428 * (so we own it) and plug it into pps[lcv]. this breaks the 1429 * following while loop and indicates we are ready to move on 1430 * to the next page in the "lcv" loop above. 1431 * 1432 * if we exit the while loop with pps[lcv] still set to NULL, 1433 * then it means that we allocated a new busy/fake/clean page 1434 * ptmp in the object and we need to do I/O to fill in the data. 1435 */ 1436 1437 while (pps[lcv] == NULL) { /* top of "pps" while loop */ 1438 1439 /* look for a current page */ 1440 ptmp = uvm_pagelookup(uobj, current_offset); 1441 1442 /* nope? allocate one now (if we can) */ 1443 if (ptmp == NULL) { 1444 1445 ptmp = uvm_pagealloc(uobj, current_offset, 1446 NULL, 0); 1447 1448 /* out of RAM? */ 1449 if (ptmp == NULL) { 1450 simple_unlock(&uobj->vmobjlock); 1451 uvm_wait("uvn_getpage"); 1452 simple_lock(&uobj->vmobjlock); 1453 1454 /* goto top of pps while loop */ 1455 continue; 1456 } 1457 1458 /* 1459 * got new page ready for I/O. break pps 1460 * while loop. pps[lcv] is still NULL. 1461 */ 1462 break; 1463 } 1464 1465 /* page is there, see if we need to wait on it */ 1466 if ((ptmp->flags & (PG_BUSY|PG_RELEASED)) != 0) { 1467 ptmp->flags |= PG_WANTED; 1468 UVM_UNLOCK_AND_WAIT(ptmp, 1469 &uobj->vmobjlock, FALSE, "uvn_get",0); 1470 simple_lock(&uobj->vmobjlock); 1471 continue; /* goto top of pps while loop */ 1472 } 1473 1474 /* 1475 * if we get here then the page has become resident 1476 * and unbusy between steps 1 and 2. we busy it 1477 * now (so we own it) and set pps[lcv] (so that we 1478 * exit the while loop). 1479 */ 1480 ptmp->flags |= PG_BUSY; 1481 UVM_PAGE_OWN(ptmp, "uvn_get2"); 1482 pps[lcv] = ptmp; 1483 } 1484 1485 /* 1486 * if we own the a valid page at the correct offset, pps[lcv] 1487 * will point to it. nothing more to do except go to the 1488 * next page. 1489 */ 1490 1491 if (pps[lcv]) 1492 continue; /* next lcv */ 1493 1494 /* 1495 * we have a "fake/busy/clean" page that we just allocated. do 1496 * I/O to fill it with valid data. note that object must be 1497 * locked going into uvn_io, but will be unlocked afterwards. 1498 */ 1499 1500 result = uvn_io((struct uvm_vnode *) uobj, &ptmp, 1, 1501 PGO_SYNCIO, UIO_READ); 1502 1503 /* 1504 * I/O done. object is unlocked (by uvn_io). because we used 1505 * syncio the result can not be PEND or AGAIN. we must relock 1506 * and check for errors. 1507 */ 1508 1509 /* lock object. check for errors. */ 1510 simple_lock(&uobj->vmobjlock); 1511 if (result != VM_PAGER_OK) { 1512 if (ptmp->flags & PG_WANTED) 1513 /* object lock still held */ 1514 wakeup(ptmp); 1515 1516 ptmp->flags &= ~(PG_WANTED|PG_BUSY); 1517 UVM_PAGE_OWN(ptmp, NULL); 1518 uvm_lock_pageq(); 1519 uvm_pagefree(ptmp); 1520 uvm_unlock_pageq(); 1521 simple_unlock(&uobj->vmobjlock); 1522 return(result); 1523 } 1524 1525 /* 1526 * we got the page! clear the fake flag (indicates valid 1527 * data now in page) and plug into our result array. note 1528 * that page is still busy. 1529 * 1530 * it is the callers job to: 1531 * => check if the page is released 1532 * => unbusy the page 1533 * => activate the page 1534 */ 1535 1536 ptmp->flags &= ~PG_FAKE; /* data is valid ... */ 1537 pmap_clear_modify(ptmp); /* ... and clean */ 1538 pps[lcv] = ptmp; 1539 1540 } /* lcv loop */ 1541 1542 /* 1543 * finally, unlock object and return. 1544 */ 1545 1546 simple_unlock(&uobj->vmobjlock); 1547 return (VM_PAGER_OK); 1548 } 1549 1550 /* 1551 * uvn_asyncget: start async I/O to bring pages into ram 1552 * 1553 * => caller must lock object(???XXX: see if this is best) 1554 * => could be called from uvn_get or a madvise() fault-ahead. 1555 * => if it fails, it doesn't matter. 1556 */ 1557 1558 static int 1559 uvn_asyncget(uobj, offset, npages) 1560 struct uvm_object *uobj; 1561 voff_t offset; 1562 int npages; 1563 { 1564 1565 /* 1566 * XXXCDC: we can't do async I/O yet 1567 */ 1568 printf("uvn_asyncget called\n"); 1569 return (KERN_SUCCESS); 1570 } 1571 1572 /* 1573 * uvn_io: do I/O to a vnode 1574 * 1575 * => prefer map unlocked (not required) 1576 * => object must be locked! we will _unlock_ it before starting I/O. 1577 * => flags: PGO_SYNCIO -- use sync. I/O 1578 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync. 1579 * [thus we never do async i/o! see iodone comment] 1580 */ 1581 1582 static int 1583 uvn_io(uvn, pps, npages, flags, rw) 1584 struct uvm_vnode *uvn; 1585 vm_page_t *pps; 1586 int npages, flags, rw; 1587 { 1588 struct vnode *vn; 1589 struct uio uio; 1590 struct iovec iov; 1591 vaddr_t kva; 1592 off_t file_offset; 1593 int waitf, result, mapinflags; 1594 size_t got, wanted; 1595 UVMHIST_FUNC("uvn_io"); UVMHIST_CALLED(maphist); 1596 1597 UVMHIST_LOG(maphist, "rw=%d", rw,0,0,0); 1598 1599 /* 1600 * init values 1601 */ 1602 1603 waitf = (flags & PGO_SYNCIO) ? M_WAITOK : M_NOWAIT; 1604 vn = (struct vnode *) uvn; 1605 file_offset = pps[0]->offset; 1606 1607 /* 1608 * check for sync'ing I/O. 1609 */ 1610 1611 while (uvn->u_flags & UVM_VNODE_IOSYNC) { 1612 if (waitf == M_NOWAIT) { 1613 simple_unlock(&uvn->u_obj.vmobjlock); 1614 UVMHIST_LOG(maphist,"<- try again (iosync)",0,0,0,0); 1615 return(VM_PAGER_AGAIN); 1616 } 1617 uvn->u_flags |= UVM_VNODE_IOSYNCWANTED; 1618 UVM_UNLOCK_AND_WAIT(&uvn->u_flags, &uvn->u_obj.vmobjlock, 1619 FALSE, "uvn_iosync",0); 1620 simple_lock(&uvn->u_obj.vmobjlock); 1621 } 1622 1623 /* 1624 * check size 1625 */ 1626 1627 if (file_offset >= uvn->u_size) { 1628 simple_unlock(&uvn->u_obj.vmobjlock); 1629 UVMHIST_LOG(maphist,"<- BAD (size check)",0,0,0,0); 1630 return(VM_PAGER_BAD); 1631 } 1632 1633 /* 1634 * first try and map the pages in (without waiting) 1635 */ 1636 1637 mapinflags = (rw == UIO_READ) ? 1638 UVMPAGER_MAPIN_READ : UVMPAGER_MAPIN_WRITE; 1639 1640 kva = uvm_pagermapin(pps, npages, NULL, mapinflags); 1641 if (kva == 0 && waitf == M_NOWAIT) { 1642 simple_unlock(&uvn->u_obj.vmobjlock); 1643 UVMHIST_LOG(maphist,"<- mapin failed (try again)",0,0,0,0); 1644 return(VM_PAGER_AGAIN); 1645 } 1646 1647 /* 1648 * ok, now bump u_nio up. at this point we are done with uvn 1649 * and can unlock it. if we still don't have a kva, try again 1650 * (this time with sleep ok). 1651 */ 1652 1653 uvn->u_nio++; /* we have an I/O in progress! */ 1654 simple_unlock(&uvn->u_obj.vmobjlock); 1655 /* NOTE: object now unlocked */ 1656 if (kva == 0) 1657 kva = uvm_pagermapin(pps, npages, NULL, 1658 mapinflags | UVMPAGER_MAPIN_WAITOK); 1659 1660 /* 1661 * ok, mapped in. our pages are PG_BUSY so they are not going to 1662 * get touched (so we can look at "offset" without having to lock 1663 * the object). set up for I/O. 1664 */ 1665 1666 /* 1667 * fill out uio/iov 1668 */ 1669 1670 iov.iov_base = (caddr_t) kva; 1671 wanted = npages << PAGE_SHIFT; 1672 if (file_offset + wanted > uvn->u_size) 1673 wanted = uvn->u_size - file_offset; /* XXX: needed? */ 1674 iov.iov_len = wanted; 1675 uio.uio_iov = &iov; 1676 uio.uio_iovcnt = 1; 1677 uio.uio_offset = file_offset; 1678 uio.uio_segflg = UIO_SYSSPACE; 1679 uio.uio_rw = rw; 1680 uio.uio_resid = wanted; 1681 uio.uio_procp = NULL; 1682 1683 /* 1684 * do the I/O! (XXX: curproc?) 1685 */ 1686 1687 UVMHIST_LOG(maphist, "calling VOP",0,0,0,0); 1688 1689 /* 1690 * This process may already have this vnode locked, if we faulted in 1691 * copyin() or copyout() on a region backed by this vnode 1692 * while doing I/O to the vnode. If this is the case, don't 1693 * panic.. instead, return the error to the user. 1694 * 1695 * XXX this is a stopgap to prevent a panic. 1696 * Ideally, this kind of operation *should* work. 1697 */ 1698 result = 0; 1699 if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0) 1700 result = vn_lock(vn, LK_EXCLUSIVE | LK_RETRY | LK_RECURSEFAIL); 1701 1702 if (result == 0) { 1703 /* NOTE: vnode now locked! */ 1704 1705 if (rw == UIO_READ) 1706 result = VOP_READ(vn, &uio, 0, curproc->p_ucred); 1707 else 1708 result = VOP_WRITE(vn, &uio, 0, curproc->p_ucred); 1709 1710 if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0) 1711 VOP_UNLOCK(vn, 0); 1712 } 1713 1714 /* NOTE: vnode now unlocked (unless vnislocked) */ 1715 1716 UVMHIST_LOG(maphist, "done calling VOP",0,0,0,0); 1717 1718 /* 1719 * result == unix style errno (0 == OK!) 1720 * 1721 * zero out rest of buffer (if needed) 1722 */ 1723 1724 if (result == 0) { 1725 got = wanted - uio.uio_resid; 1726 1727 if (wanted && got == 0) { 1728 result = EIO; /* XXX: error? */ 1729 } else if (got < PAGE_SIZE * npages && rw == UIO_READ) { 1730 memset((void *) (kva + got), 0, 1731 (npages << PAGE_SHIFT) - got); 1732 } 1733 } 1734 1735 /* 1736 * now remove pager mapping 1737 */ 1738 uvm_pagermapout(kva, npages); 1739 1740 /* 1741 * now clean up the object (i.e. drop I/O count) 1742 */ 1743 1744 simple_lock(&uvn->u_obj.vmobjlock); 1745 /* NOTE: object now locked! */ 1746 1747 uvn->u_nio--; /* I/O DONE! */ 1748 if ((uvn->u_flags & UVM_VNODE_IOSYNC) != 0 && uvn->u_nio == 0) { 1749 wakeup(&uvn->u_nio); 1750 } 1751 simple_unlock(&uvn->u_obj.vmobjlock); 1752 /* NOTE: object now unlocked! */ 1753 1754 /* 1755 * done! 1756 */ 1757 1758 UVMHIST_LOG(maphist, "<- done (result %d)", result,0,0,0); 1759 if (result == 0) 1760 return(VM_PAGER_OK); 1761 else 1762 return(VM_PAGER_ERROR); 1763 } 1764 1765 /* 1766 * uvm_vnp_uncache: disable "persisting" in a vnode... when last reference 1767 * is gone we will kill the object (flushing dirty pages back to the vnode 1768 * if needed). 1769 * 1770 * => returns TRUE if there was no uvm_object attached or if there was 1771 * one and we killed it [i.e. if there is no active uvn] 1772 * => called with the vnode VOP_LOCK'd [we will unlock it for I/O, if 1773 * needed] 1774 * 1775 * => XXX: given that we now kill uvn's when a vnode is recycled (without 1776 * having to hold a reference on the vnode) and given a working 1777 * uvm_vnp_sync(), how does that effect the need for this function? 1778 * [XXXCDC: seems like it can die?] 1779 * 1780 * => XXX: this function should DIE once we merge the VM and buffer 1781 * cache. 1782 * 1783 * research shows that this is called in the following places: 1784 * ext2fs_truncate, ffs_truncate, detrunc[msdosfs]: called when vnode 1785 * changes sizes 1786 * ext2fs_write, WRITE [ufs_readwrite], msdosfs_write: called when we 1787 * are written to 1788 * ex2fs_chmod, ufs_chmod: called if VTEXT vnode and the sticky bit 1789 * is off 1790 * ffs_realloccg: when we can't extend the current block and have 1791 * to allocate a new one we call this [XXX: why?] 1792 * nfsrv_rename, rename_files: called when the target filename is there 1793 * and we want to remove it 1794 * nfsrv_remove, sys_unlink: called on file we are removing 1795 * nfsrv_access: if VTEXT and we want WRITE access and we don't uncache 1796 * then return "text busy" 1797 * nfs_open: seems to uncache any file opened with nfs 1798 * vn_writechk: if VTEXT vnode and can't uncache return "text busy" 1799 */ 1800 1801 boolean_t 1802 uvm_vnp_uncache(vp) 1803 struct vnode *vp; 1804 { 1805 struct uvm_vnode *uvn = &vp->v_uvm; 1806 1807 /* 1808 * lock uvn part of the vnode and check to see if we need to do anything 1809 */ 1810 1811 simple_lock(&uvn->u_obj.vmobjlock); 1812 if ((uvn->u_flags & UVM_VNODE_VALID) == 0 || 1813 (uvn->u_flags & UVM_VNODE_BLOCKED) != 0) { 1814 simple_unlock(&uvn->u_obj.vmobjlock); 1815 return(TRUE); 1816 } 1817 1818 /* 1819 * we have a valid, non-blocked uvn. clear persist flag. 1820 * if uvn is currently active we can return now. 1821 */ 1822 1823 uvn->u_flags &= ~UVM_VNODE_CANPERSIST; 1824 if (uvn->u_obj.uo_refs) { 1825 simple_unlock(&uvn->u_obj.vmobjlock); 1826 return(FALSE); 1827 } 1828 1829 /* 1830 * uvn is currently persisting! we have to gain a reference to 1831 * it so that we can call uvn_detach to kill the uvn. 1832 */ 1833 1834 VREF(vp); /* seems ok, even with VOP_LOCK */ 1835 uvn->u_obj.uo_refs++; /* value is now 1 */ 1836 simple_unlock(&uvn->u_obj.vmobjlock); 1837 1838 1839 #ifdef DEBUG 1840 /* 1841 * carry over sanity check from old vnode pager: the vnode should 1842 * be VOP_LOCK'd, and we confirm it here. 1843 */ 1844 if (!VOP_ISLOCKED(vp)) { 1845 boolean_t is_ok_anyway = FALSE; 1846 #ifdef NFS 1847 extern int (**nfsv2_vnodeop_p) __P((void *)); 1848 extern int (**spec_nfsv2nodeop_p) __P((void *)); 1849 extern int (**fifo_nfsv2nodeop_p) __P((void *)); 1850 1851 /* vnode is NOT VOP_LOCKed: some vnode types _never_ lock */ 1852 if (vp->v_op == nfsv2_vnodeop_p || 1853 vp->v_op == spec_nfsv2nodeop_p) { 1854 is_ok_anyway = TRUE; 1855 } 1856 if (vp->v_op == fifo_nfsv2nodeop_p) { 1857 is_ok_anyway = TRUE; 1858 } 1859 #endif /* NFS */ 1860 if (!is_ok_anyway) 1861 panic("uvm_vnp_uncache: vnode not locked!"); 1862 } 1863 #endif /* DEBUG */ 1864 1865 /* 1866 * now drop our reference to the vnode. if we have the sole 1867 * reference to the vnode then this will cause it to die [as we 1868 * just cleared the persist flag]. we have to unlock the vnode 1869 * while we are doing this as it may trigger I/O. 1870 * 1871 * XXX: it might be possible for uvn to get reclaimed while we are 1872 * unlocked causing us to return TRUE when we should not. we ignore 1873 * this as a false-positive return value doesn't hurt us. 1874 */ 1875 VOP_UNLOCK(vp, 0); 1876 uvn_detach(&uvn->u_obj); 1877 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 1878 1879 /* 1880 * and return... 1881 */ 1882 1883 return(TRUE); 1884 } 1885 1886 /* 1887 * uvm_vnp_setsize: grow or shrink a vnode uvn 1888 * 1889 * grow => just update size value 1890 * shrink => toss un-needed pages 1891 * 1892 * => we assume that the caller has a reference of some sort to the 1893 * vnode in question so that it will not be yanked out from under 1894 * us. 1895 * 1896 * called from: 1897 * => truncate fns (ext2fs_truncate, ffs_truncate, detrunc[msdos]) 1898 * => "write" fns (ext2fs_write, WRITE [ufs/ufs], msdosfs_write, nfs_write) 1899 * => ffs_balloc [XXX: why? doesn't WRITE handle?] 1900 * => NFS: nfs_loadattrcache, nfs_getattrcache, nfs_setattr 1901 * => union fs: union_newsize 1902 */ 1903 1904 void 1905 uvm_vnp_setsize(vp, newsize) 1906 struct vnode *vp; 1907 voff_t newsize; 1908 { 1909 struct uvm_vnode *uvn = &vp->v_uvm; 1910 1911 /* 1912 * lock uvn and check for valid object, and if valid: do it! 1913 */ 1914 simple_lock(&uvn->u_obj.vmobjlock); 1915 if (uvn->u_flags & UVM_VNODE_VALID) { 1916 1917 /* 1918 * now check if the size has changed: if we shrink we had better 1919 * toss some pages... 1920 */ 1921 1922 if (uvn->u_size > newsize) { 1923 (void)uvn_flush(&uvn->u_obj, newsize, 1924 uvn->u_size, PGO_FREE); 1925 } 1926 uvn->u_size = newsize; 1927 } 1928 simple_unlock(&uvn->u_obj.vmobjlock); 1929 1930 /* 1931 * done 1932 */ 1933 return; 1934 } 1935 1936 /* 1937 * uvm_vnp_sync: flush all dirty VM pages back to their backing vnodes. 1938 * 1939 * => called from sys_sync with no VM structures locked 1940 * => only one process can do a sync at a time (because the uvn 1941 * structure only has one queue for sync'ing). we ensure this 1942 * by holding the uvn_sync_lock while the sync is in progress. 1943 * other processes attempting a sync will sleep on this lock 1944 * until we are done. 1945 */ 1946 1947 void 1948 uvm_vnp_sync(mp) 1949 struct mount *mp; 1950 { 1951 struct uvm_vnode *uvn; 1952 struct vnode *vp; 1953 boolean_t got_lock; 1954 1955 /* 1956 * step 1: ensure we are only ones using the uvn_sync_q by locking 1957 * our lock... 1958 */ 1959 lockmgr(&uvn_sync_lock, LK_EXCLUSIVE, (void *)0); 1960 1961 /* 1962 * step 2: build up a simpleq of uvns of interest based on the 1963 * write list. we gain a reference to uvns of interest. must 1964 * be careful about locking uvn's since we will be holding uvn_wl_lock 1965 * in the body of the loop. 1966 */ 1967 SIMPLEQ_INIT(&uvn_sync_q); 1968 simple_lock(&uvn_wl_lock); 1969 for (uvn = uvn_wlist.lh_first ; uvn != NULL ; 1970 uvn = uvn->u_wlist.le_next) { 1971 1972 vp = (struct vnode *) uvn; 1973 if (mp && vp->v_mount != mp) 1974 continue; 1975 1976 /* attempt to gain reference */ 1977 while ((got_lock = simple_lock_try(&uvn->u_obj.vmobjlock)) == 1978 FALSE && 1979 (uvn->u_flags & UVM_VNODE_BLOCKED) == 0) 1980 /* spin */ ; 1981 1982 /* 1983 * we will exit the loop if either if the following are true: 1984 * - we got the lock [always true if NCPU == 1] 1985 * - we failed to get the lock but noticed the vnode was 1986 * "blocked" -- in this case the vnode must be a dying 1987 * vnode, and since dying vnodes are in the process of 1988 * being flushed out, we can safely skip this one 1989 * 1990 * we want to skip over the vnode if we did not get the lock, 1991 * or if the vnode is already dying (due to the above logic). 1992 * 1993 * note that uvn must already be valid because we found it on 1994 * the wlist (this also means it can't be ALOCK'd). 1995 */ 1996 if (!got_lock || (uvn->u_flags & UVM_VNODE_BLOCKED) != 0) { 1997 if (got_lock) 1998 simple_unlock(&uvn->u_obj.vmobjlock); 1999 continue; /* skip it */ 2000 } 2001 2002 /* 2003 * gain reference. watch out for persisting uvns (need to 2004 * regain vnode REF). 2005 */ 2006 if (uvn->u_obj.uo_refs == 0) 2007 VREF(vp); 2008 uvn->u_obj.uo_refs++; 2009 simple_unlock(&uvn->u_obj.vmobjlock); 2010 2011 /* 2012 * got it! 2013 */ 2014 SIMPLEQ_INSERT_HEAD(&uvn_sync_q, uvn, u_syncq); 2015 } 2016 simple_unlock(&uvn_wl_lock); 2017 2018 /* 2019 * step 3: we now have a list of uvn's that may need cleaning. 2020 * we are holding the uvn_sync_lock, but have dropped the uvn_wl_lock 2021 * (so we can now safely lock uvn's again). 2022 */ 2023 2024 for (uvn = uvn_sync_q.sqh_first ; uvn ; uvn = uvn->u_syncq.sqe_next) { 2025 simple_lock(&uvn->u_obj.vmobjlock); 2026 #ifdef DEBUG 2027 if (uvn->u_flags & UVM_VNODE_DYING) { 2028 printf("uvm_vnp_sync: dying vnode on sync list\n"); 2029 } 2030 #endif 2031 uvn_flush(&uvn->u_obj, 0, 0, 2032 PGO_CLEANIT|PGO_ALLPAGES|PGO_DOACTCLUST); 2033 2034 /* 2035 * if we have the only reference and we just cleaned the uvn, 2036 * then we can pull it out of the UVM_VNODE_WRITEABLE state 2037 * thus allowing us to avoid thinking about flushing it again 2038 * on later sync ops. 2039 */ 2040 if (uvn->u_obj.uo_refs == 1 && 2041 (uvn->u_flags & UVM_VNODE_WRITEABLE)) { 2042 LIST_REMOVE(uvn, u_wlist); 2043 uvn->u_flags &= ~UVM_VNODE_WRITEABLE; 2044 } 2045 2046 simple_unlock(&uvn->u_obj.vmobjlock); 2047 2048 /* now drop our reference to the uvn */ 2049 uvn_detach(&uvn->u_obj); 2050 } 2051 2052 /* 2053 * done! release sync lock 2054 */ 2055 lockmgr(&uvn_sync_lock, LK_RELEASE, (void *)0); 2056 } 2057