1 /* $OpenBSD: uvm_vnode.c,v 1.76 2011/07/04 20:35:35 deraadt Exp $ */ 2 /* $NetBSD: uvm_vnode.c,v 1.36 2000/11/24 20:34:01 chs Exp $ */ 3 4 /* 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * Copyright (c) 1991, 1993 7 * The Regents of the University of California. 8 * Copyright (c) 1990 University of Utah. 9 * 10 * All rights reserved. 11 * 12 * This code is derived from software contributed to Berkeley by 13 * the Systems Programming Group of the University of Utah Computer 14 * Science Department. 15 * 16 * Redistribution and use in source and binary forms, with or without 17 * modification, are permitted provided that the following conditions 18 * are met: 19 * 1. Redistributions of source code must retain the above copyright 20 * notice, this list of conditions and the following disclaimer. 21 * 2. Redistributions in binary form must reproduce the above copyright 22 * notice, this list of conditions and the following disclaimer in the 23 * documentation and/or other materials provided with the distribution. 24 * 3. All advertising materials mentioning features or use of this software 25 * must display the following acknowledgement: 26 * This product includes software developed by Charles D. Cranor, 27 * Washington University, the University of California, Berkeley and 28 * its contributors. 29 * 4. Neither the name of the University nor the names of its contributors 30 * may be used to endorse or promote products derived from this software 31 * without specific prior written permission. 32 * 33 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 34 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 35 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 36 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 37 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 38 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 39 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 40 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 41 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 42 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 43 * SUCH DAMAGE. 44 * 45 * @(#)vnode_pager.c 8.8 (Berkeley) 2/13/94 46 * from: Id: uvm_vnode.c,v 1.1.2.26 1998/02/02 20:38:07 chuck Exp 47 */ 48 49 /* 50 * uvm_vnode.c: the vnode pager. 51 */ 52 53 #include <sys/param.h> 54 #include <sys/systm.h> 55 #include <sys/proc.h> 56 #include <sys/malloc.h> 57 #include <sys/vnode.h> 58 #include <sys/disklabel.h> 59 #include <sys/ioctl.h> 60 #include <sys/fcntl.h> 61 #include <sys/conf.h> 62 #include <sys/rwlock.h> 63 #include <sys/dkio.h> 64 #include <sys/specdev.h> 65 66 #include <uvm/uvm.h> 67 #include <uvm/uvm_vnode.h> 68 69 /* 70 * private global data structure 71 * 72 * we keep a list of writeable active vnode-backed VM objects for sync op. 73 * we keep a simpleq of vnodes that are currently being sync'd. 74 */ 75 76 LIST_HEAD(uvn_list_struct, uvm_vnode); 77 struct uvn_list_struct uvn_wlist; /* writeable uvns */ 78 79 SIMPLEQ_HEAD(uvn_sq_struct, uvm_vnode); 80 struct uvn_sq_struct uvn_sync_q; /* sync'ing uvns */ 81 struct rwlock uvn_sync_lock; /* locks sync operation */ 82 83 /* 84 * functions 85 */ 86 87 void uvn_cluster(struct uvm_object *, voff_t, voff_t *, voff_t *); 88 void uvn_detach(struct uvm_object *); 89 boolean_t uvn_flush(struct uvm_object *, voff_t, voff_t, int); 90 int uvn_get(struct uvm_object *, voff_t, vm_page_t *, int *, int, 91 vm_prot_t, int, int); 92 void uvn_init(void); 93 int uvn_io(struct uvm_vnode *, vm_page_t *, int, int, int); 94 int uvn_put(struct uvm_object *, vm_page_t *, int, boolean_t); 95 void uvn_reference(struct uvm_object *); 96 97 /* 98 * master pager structure 99 */ 100 101 struct uvm_pagerops uvm_vnodeops = { 102 uvn_init, 103 uvn_reference, 104 uvn_detach, 105 NULL, /* no specialized fault routine required */ 106 uvn_flush, 107 uvn_get, 108 uvn_put, 109 uvn_cluster, 110 uvm_mk_pcluster, /* use generic version of this: see uvm_pager.c */ 111 }; 112 113 /* 114 * the ops! 115 */ 116 117 /* 118 * uvn_init 119 * 120 * init pager private data structures. 121 */ 122 123 void 124 uvn_init(void) 125 { 126 127 LIST_INIT(&uvn_wlist); 128 /* note: uvn_sync_q init'd in uvm_vnp_sync() */ 129 rw_init(&uvn_sync_lock, "uvnsync"); 130 } 131 132 /* 133 * uvn_attach 134 * 135 * attach a vnode structure to a VM object. if the vnode is already 136 * attached, then just bump the reference count by one and return the 137 * VM object. if not already attached, attach and return the new VM obj. 138 * the "accessprot" tells the max access the attaching thread wants to 139 * our pages. 140 * 141 * => caller must _not_ already be holding the lock on the uvm_object. 142 * => in fact, nothing should be locked so that we can sleep here. 143 * => note that uvm_object is first thing in vnode structure, so their 144 * pointers are equiv. 145 */ 146 147 struct uvm_object * 148 uvn_attach(void *arg, vm_prot_t accessprot) 149 { 150 struct vnode *vp = arg; 151 struct uvm_vnode *uvn = &vp->v_uvm; 152 struct vattr vattr; 153 int oldflags, result; 154 struct partinfo pi; 155 u_quad_t used_vnode_size; 156 157 used_vnode_size = (u_quad_t)0; /* XXX gcc -Wuninitialized */ 158 159 /* 160 * first get a lock on the uvn. 161 */ 162 simple_lock(&uvn->u_obj.vmobjlock); 163 while (uvn->u_flags & UVM_VNODE_BLOCKED) { 164 uvn->u_flags |= UVM_VNODE_WANTED; 165 UVM_UNLOCK_AND_WAIT(uvn, &uvn->u_obj.vmobjlock, FALSE, 166 "uvn_attach", 0); 167 simple_lock(&uvn->u_obj.vmobjlock); 168 } 169 170 /* 171 * if we're mapping a BLK device, make sure it is a disk. 172 */ 173 if (vp->v_type == VBLK && bdevsw[major(vp->v_rdev)].d_type != D_DISK) { 174 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock */ 175 return(NULL); 176 } 177 178 /* 179 * now we have lock and uvn must not be in a blocked state. 180 * first check to see if it is already active, in which case 181 * we can bump the reference count, check to see if we need to 182 * add it to the writeable list, and then return. 183 */ 184 if (uvn->u_flags & UVM_VNODE_VALID) { /* already active? */ 185 186 /* regain vref if we were persisting */ 187 if (uvn->u_obj.uo_refs == 0) { 188 vref(vp); 189 } 190 uvn->u_obj.uo_refs++; /* bump uvn ref! */ 191 192 /* check for new writeable uvn */ 193 if ((accessprot & VM_PROT_WRITE) != 0 && 194 (uvn->u_flags & UVM_VNODE_WRITEABLE) == 0) { 195 LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist); 196 /* we are now on wlist! */ 197 uvn->u_flags |= UVM_VNODE_WRITEABLE; 198 } 199 200 /* unlock and return */ 201 simple_unlock(&uvn->u_obj.vmobjlock); 202 return (&uvn->u_obj); 203 } 204 205 /* 206 * need to call VOP_GETATTR() to get the attributes, but that could 207 * block (due to I/O), so we want to unlock the object before calling. 208 * however, we want to keep anyone else from playing with the object 209 * while it is unlocked. to do this we set UVM_VNODE_ALOCK which 210 * prevents anyone from attaching to the vnode until we are done with 211 * it. 212 */ 213 uvn->u_flags = UVM_VNODE_ALOCK; 214 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock in case we sleep */ 215 /* XXX: curproc? */ 216 217 if (vp->v_type == VBLK) { 218 /* 219 * We could implement this as a specfs getattr call, but: 220 * 221 * (1) VOP_GETATTR() would get the file system 222 * vnode operation, not the specfs operation. 223 * 224 * (2) All we want is the size, anyhow. 225 */ 226 result = (*bdevsw[major(vp->v_rdev)].d_ioctl)(vp->v_rdev, 227 DIOCGPART, (caddr_t)&pi, FREAD, curproc); 228 if (result == 0) { 229 /* XXX should remember blocksize */ 230 used_vnode_size = (u_quad_t)pi.disklab->d_secsize * 231 (u_quad_t)DL_GETPSIZE(pi.part); 232 } 233 } else { 234 result = VOP_GETATTR(vp, &vattr, curproc->p_ucred, curproc); 235 if (result == 0) 236 used_vnode_size = vattr.va_size; 237 } 238 239 /* relock object */ 240 simple_lock(&uvn->u_obj.vmobjlock); 241 242 if (result != 0) { 243 if (uvn->u_flags & UVM_VNODE_WANTED) 244 wakeup(uvn); 245 uvn->u_flags = 0; 246 simple_unlock(&uvn->u_obj.vmobjlock); /* drop lock */ 247 return(NULL); 248 } 249 250 /* 251 * make sure that the newsize fits within a vaddr_t 252 * XXX: need to revise addressing data types 253 */ 254 #ifdef DEBUG 255 if (vp->v_type == VBLK) 256 printf("used_vnode_size = %llu\n", (long long)used_vnode_size); 257 #endif 258 259 /* 260 * now set up the uvn. 261 */ 262 uvm_objinit(&uvn->u_obj, &uvm_vnodeops, 1); 263 oldflags = uvn->u_flags; 264 uvn->u_flags = UVM_VNODE_VALID|UVM_VNODE_CANPERSIST; 265 uvn->u_nio = 0; 266 uvn->u_size = used_vnode_size; 267 268 /* if write access, we need to add it to the wlist */ 269 if (accessprot & VM_PROT_WRITE) { 270 LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist); 271 uvn->u_flags |= UVM_VNODE_WRITEABLE; /* we are on wlist! */ 272 } 273 274 /* 275 * add a reference to the vnode. this reference will stay as long 276 * as there is a valid mapping of the vnode. dropped when the 277 * reference count goes to zero [and we either free or persist]. 278 */ 279 vref(vp); 280 simple_unlock(&uvn->u_obj.vmobjlock); 281 if (oldflags & UVM_VNODE_WANTED) 282 wakeup(uvn); 283 284 return(&uvn->u_obj); 285 } 286 287 288 /* 289 * uvn_reference 290 * 291 * duplicate a reference to a VM object. Note that the reference 292 * count must already be at least one (the passed in reference) so 293 * there is no chance of the uvn being killed or locked out here. 294 * 295 * => caller must call with object unlocked. 296 * => caller must be using the same accessprot as was used at attach time 297 */ 298 299 300 void 301 uvn_reference(struct uvm_object *uobj) 302 { 303 #ifdef DEBUG 304 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 305 #endif 306 307 simple_lock(&uobj->vmobjlock); 308 #ifdef DEBUG 309 if ((uvn->u_flags & UVM_VNODE_VALID) == 0) { 310 printf("uvn_reference: ref=%d, flags=0x%x\n", uvn->u_flags, 311 uobj->uo_refs); 312 panic("uvn_reference: invalid state"); 313 } 314 #endif 315 uobj->uo_refs++; 316 simple_unlock(&uobj->vmobjlock); 317 } 318 319 /* 320 * uvn_detach 321 * 322 * remove a reference to a VM object. 323 * 324 * => caller must call with object unlocked and map locked. 325 * => this starts the detach process, but doesn't have to finish it 326 * (async i/o could still be pending). 327 */ 328 void 329 uvn_detach(struct uvm_object *uobj) 330 { 331 struct uvm_vnode *uvn; 332 struct vnode *vp; 333 int oldflags; 334 335 simple_lock(&uobj->vmobjlock); 336 337 uobj->uo_refs--; /* drop ref! */ 338 if (uobj->uo_refs) { /* still more refs */ 339 simple_unlock(&uobj->vmobjlock); 340 return; 341 } 342 343 /* 344 * get other pointers ... 345 */ 346 347 uvn = (struct uvm_vnode *) uobj; 348 vp = (struct vnode *) uobj; 349 350 /* 351 * clear VTEXT flag now that there are no mappings left (VTEXT is used 352 * to keep an active text file from being overwritten). 353 */ 354 vp->v_flag &= ~VTEXT; 355 356 /* 357 * we just dropped the last reference to the uvn. see if we can 358 * let it "stick around". 359 */ 360 361 if (uvn->u_flags & UVM_VNODE_CANPERSIST) { 362 /* won't block */ 363 uvn_flush(uobj, 0, 0, PGO_DEACTIVATE|PGO_ALLPAGES); 364 simple_unlock(&uobj->vmobjlock); 365 vrele(vp); /* drop vnode reference */ 366 return; 367 } 368 369 /* 370 * its a goner! 371 */ 372 373 uvn->u_flags |= UVM_VNODE_DYING; 374 375 /* 376 * even though we may unlock in flush, no one can gain a reference 377 * to us until we clear the "dying" flag [because it blocks 378 * attaches]. we will not do that until after we've disposed of all 379 * the pages with uvn_flush(). note that before the flush the only 380 * pages that could be marked PG_BUSY are ones that are in async 381 * pageout by the daemon. (there can't be any pending "get"'s 382 * because there are no references to the object). 383 */ 384 385 (void) uvn_flush(uobj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES); 386 387 /* 388 * given the structure of this pager, the above flush request will 389 * create the following state: all the pages that were in the object 390 * have either been free'd or they are marked PG_BUSY and in the 391 * middle of an async io. If we still have pages we set the "relkill" 392 * state, so that in the case the vnode gets terminated we know 393 * to leave it alone. Otherwise we'll kill the vnode when it's empty. 394 */ 395 396 uvn->u_flags |= UVM_VNODE_RELKILL; 397 /* wait on any outstanding io */ 398 while (uobj->uo_npages && uvn->u_flags & UVM_VNODE_RELKILL) { 399 uvn->u_flags |= UVM_VNODE_IOSYNC; 400 UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock, FALSE, 401 "uvn_term",0); 402 simple_lock(&uvn->u_obj.vmobjlock); 403 } 404 405 if ((uvn->u_flags & UVM_VNODE_RELKILL) == 0) 406 return; 407 408 /* 409 * kill object now. note that we can't be on the sync q because 410 * all references are gone. 411 */ 412 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 413 LIST_REMOVE(uvn, u_wlist); 414 } 415 KASSERT(RB_EMPTY(&uobj->memt)); 416 oldflags = uvn->u_flags; 417 uvn->u_flags = 0; 418 simple_unlock(&uobj->vmobjlock); 419 420 /* wake up any sleepers */ 421 if (oldflags & UVM_VNODE_WANTED) 422 wakeup(uvn); 423 424 /* 425 * drop our reference to the vnode. 426 */ 427 vrele(vp); 428 429 return; 430 } 431 432 /* 433 * uvm_vnp_terminate: external hook to clear out a vnode's VM 434 * 435 * called in two cases: 436 * [1] when a persisting vnode vm object (i.e. one with a zero reference 437 * count) needs to be freed so that a vnode can be reused. this 438 * happens under "getnewvnode" in vfs_subr.c. if the vnode from 439 * the free list is still attached (i.e. not VBAD) then vgone is 440 * called. as part of the vgone trace this should get called to 441 * free the vm object. this is the common case. 442 * [2] when a filesystem is being unmounted by force (MNT_FORCE, 443 * "umount -f") the vgone() function is called on active vnodes 444 * on the mounted file systems to kill their data (the vnodes become 445 * "dead" ones [see src/sys/miscfs/deadfs/...]). that results in a 446 * call here (even if the uvn is still in use -- i.e. has a non-zero 447 * reference count). this case happens at "umount -f" and during a 448 * "reboot/halt" operation. 449 * 450 * => the caller must XLOCK and VOP_LOCK the vnode before calling us 451 * [protects us from getting a vnode that is already in the DYING 452 * state...] 453 * => in case [2] the uvn is still alive after this call, but all I/O 454 * ops will fail (due to the backing vnode now being "dead"). this 455 * will prob. kill any process using the uvn due to pgo_get failing. 456 */ 457 458 void 459 uvm_vnp_terminate(struct vnode *vp) 460 { 461 struct uvm_vnode *uvn = &vp->v_uvm; 462 int oldflags; 463 464 /* 465 * lock object and check if it is valid 466 */ 467 simple_lock(&uvn->u_obj.vmobjlock); 468 if ((uvn->u_flags & UVM_VNODE_VALID) == 0) { 469 simple_unlock(&uvn->u_obj.vmobjlock); 470 return; 471 } 472 473 /* 474 * must be a valid uvn that is not already dying (because XLOCK 475 * protects us from that). the uvn can't in the ALOCK state 476 * because it is valid, and uvn's that are in the ALOCK state haven't 477 * been marked valid yet. 478 */ 479 480 #ifdef DEBUG 481 /* 482 * debug check: are we yanking the vnode out from under our uvn? 483 */ 484 if (uvn->u_obj.uo_refs) { 485 printf("uvm_vnp_terminate(%p): terminating active vnode " 486 "(refs=%d)\n", uvn, uvn->u_obj.uo_refs); 487 } 488 #endif 489 490 /* 491 * it is possible that the uvn was detached and is in the relkill 492 * state [i.e. waiting for async i/o to finish]. 493 * we take over the vnode now and cancel the relkill. 494 * we want to know when the i/o is done so we can recycle right 495 * away. note that a uvn can only be in the RELKILL state if it 496 * has a zero reference count. 497 */ 498 499 if (uvn->u_flags & UVM_VNODE_RELKILL) 500 uvn->u_flags &= ~UVM_VNODE_RELKILL; /* cancel RELKILL */ 501 502 /* 503 * block the uvn by setting the dying flag, and then flush the 504 * pages. (note that flush may unlock object while doing I/O, but 505 * it will re-lock it before it returns control here). 506 * 507 * also, note that we tell I/O that we are already VOP_LOCK'd so 508 * that uvn_io doesn't attempt to VOP_LOCK again. 509 * 510 * XXXCDC: setting VNISLOCKED on an active uvn which is being terminated 511 * due to a forceful unmount might not be a good idea. maybe we 512 * need a way to pass in this info to uvn_flush through a 513 * pager-defined PGO_ constant [currently there are none]. 514 */ 515 uvn->u_flags |= UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED; 516 517 (void) uvn_flush(&uvn->u_obj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES); 518 519 /* 520 * as we just did a flush we expect all the pages to be gone or in 521 * the process of going. sleep to wait for the rest to go [via iosync]. 522 */ 523 524 while (uvn->u_obj.uo_npages) { 525 #ifdef DEBUG 526 struct vm_page *pp; 527 RB_FOREACH(pp, uvm_objtree, &uvn->u_obj.memt) { 528 if ((pp->pg_flags & PG_BUSY) == 0) 529 panic("uvm_vnp_terminate: detected unbusy pg"); 530 } 531 if (uvn->u_nio == 0) 532 panic("uvm_vnp_terminate: no I/O to wait for?"); 533 printf("uvm_vnp_terminate: waiting for I/O to fin.\n"); 534 /* 535 * XXXCDC: this is unlikely to happen without async i/o so we 536 * put a printf in just to keep an eye on it. 537 */ 538 #endif 539 uvn->u_flags |= UVM_VNODE_IOSYNC; 540 UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock, FALSE, 541 "uvn_term",0); 542 simple_lock(&uvn->u_obj.vmobjlock); 543 } 544 545 /* 546 * done. now we free the uvn if its reference count is zero 547 * (true if we are zapping a persisting uvn). however, if we are 548 * terminating a uvn with active mappings we let it live ... future 549 * calls down to the vnode layer will fail. 550 */ 551 552 oldflags = uvn->u_flags; 553 if (uvn->u_obj.uo_refs) { 554 555 /* 556 * uvn must live on it is dead-vnode state until all references 557 * are gone. restore flags. clear CANPERSIST state. 558 */ 559 560 uvn->u_flags &= ~(UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED| 561 UVM_VNODE_WANTED|UVM_VNODE_CANPERSIST); 562 563 } else { 564 565 /* 566 * free the uvn now. note that the vref reference is already 567 * gone [it is dropped when we enter the persist state]. 568 */ 569 if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED) 570 panic("uvm_vnp_terminate: io sync wanted bit set"); 571 572 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 573 LIST_REMOVE(uvn, u_wlist); 574 } 575 uvn->u_flags = 0; /* uvn is history, clear all bits */ 576 } 577 578 if (oldflags & UVM_VNODE_WANTED) 579 wakeup(uvn); /* object lock still held */ 580 581 simple_unlock(&uvn->u_obj.vmobjlock); 582 583 } 584 585 /* 586 * NOTE: currently we have to use VOP_READ/VOP_WRITE because they go 587 * through the buffer cache and allow I/O in any size. These VOPs use 588 * synchronous i/o. [vs. VOP_STRATEGY which can be async, but doesn't 589 * go through the buffer cache or allow I/O sizes larger than a 590 * block]. we will eventually want to change this. 591 * 592 * issues to consider: 593 * uvm provides the uvm_aiodesc structure for async i/o management. 594 * there are two tailq's in the uvm. structure... one for pending async 595 * i/o and one for "done" async i/o. to do an async i/o one puts 596 * an aiodesc on the "pending" list (protected by splbio()), starts the 597 * i/o and returns VM_PAGER_PEND. when the i/o is done, we expect 598 * some sort of "i/o done" function to be called (at splbio(), interrupt 599 * time). this function should remove the aiodesc from the pending list 600 * and place it on the "done" list and wakeup the daemon. the daemon 601 * will run at normal spl() and will remove all items from the "done" 602 * list and call the "aiodone" hook for each done request (see uvm_pager.c). 603 * [in the old vm code, this was done by calling the "put" routine with 604 * null arguments which made the code harder to read and understand because 605 * you had one function ("put") doing two things.] 606 * 607 * so the current pager needs: 608 * int uvn_aiodone(struct uvm_aiodesc *) 609 * 610 * => return 0 (aio finished, free it). otherwise requeue for later collection. 611 * => called with pageq's locked by the daemon. 612 * 613 * general outline: 614 * - "try" to lock object. if fail, just return (will try again later) 615 * - drop "u_nio" (this req is done!) 616 * - if (object->iosync && u_naio == 0) { wakeup &uvn->u_naio } 617 * - get "page" structures (atop?). 618 * - handle "wanted" pages 619 * dont forget to look at "object" wanted flag in all cases. 620 */ 621 622 623 /* 624 * uvn_flush: flush pages out of a uvm object. 625 * 626 * => object should be locked by caller. we may _unlock_ the object 627 * if (and only if) we need to clean a page (PGO_CLEANIT). 628 * we return with the object locked. 629 * => if PGO_CLEANIT is set, we may block (due to I/O). thus, a caller 630 * might want to unlock higher level resources (e.g. vm_map) 631 * before calling flush. 632 * => if PGO_CLEANIT is not set, then we will neither unlock the object 633 * or block. 634 * => if PGO_ALLPAGE is set, then all pages in the object are valid targets 635 * for flushing. 636 * => NOTE: we are allowed to lock the page queues, so the caller 637 * must not be holding the lock on them [e.g. pagedaemon had 638 * better not call us with the queues locked] 639 * => we return TRUE unless we encountered some sort of I/O error 640 * 641 * comment on "cleaning" object and PG_BUSY pages: 642 * this routine is holding the lock on the object. the only time 643 * that it can run into a PG_BUSY page that it does not own is if 644 * some other process has started I/O on the page (e.g. either 645 * a pagein, or a pageout). if the PG_BUSY page is being paged 646 * in, then it can not be dirty (!PG_CLEAN) because no one has 647 * had a chance to modify it yet. if the PG_BUSY page is being 648 * paged out then it means that someone else has already started 649 * cleaning the page for us (how nice!). in this case, if we 650 * have syncio specified, then after we make our pass through the 651 * object we need to wait for the other PG_BUSY pages to clear 652 * off (i.e. we need to do an iosync). also note that once a 653 * page is PG_BUSY it must stay in its object until it is un-busyed. 654 */ 655 656 #define UVN_HASH_PENALTY 4 /* XXX: a guess */ 657 658 boolean_t 659 uvn_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags) 660 { 661 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 662 struct vm_page *pp, *ptmp; 663 struct vm_page *pps[MAXBSIZE >> PAGE_SHIFT], **ppsp; 664 int npages, result, lcv; 665 boolean_t retval, need_iosync, needs_clean; 666 voff_t curoff; 667 668 /* 669 * get init vals and determine how we are going to traverse object 670 */ 671 672 need_iosync = FALSE; 673 retval = TRUE; /* return value */ 674 if (flags & PGO_ALLPAGES) { 675 start = 0; 676 stop = round_page(uvn->u_size); 677 } else { 678 start = trunc_page(start); 679 stop = MIN(round_page(stop), round_page(uvn->u_size)); 680 } 681 682 /* 683 * PG_CLEANCHK: this bit is used by the pgo_mk_pcluster function as 684 * a _hint_ as to how up to date the PG_CLEAN bit is. if the hint 685 * is wrong it will only prevent us from clustering... it won't break 686 * anything. we clear all PG_CLEANCHK bits here, and pgo_mk_pcluster 687 * will set them as it syncs PG_CLEAN. This is only an issue if we 688 * are looking at non-inactive pages (because inactive page's PG_CLEAN 689 * bit is always up to date since there are no mappings). 690 * [borrowed PG_CLEANCHK idea from FreeBSD VM] 691 */ 692 693 if ((flags & PGO_CLEANIT) != 0) { 694 KASSERT(uobj->pgops->pgo_mk_pcluster != 0); 695 for (curoff = start ; curoff < stop; curoff += PAGE_SIZE) { 696 if ((pp = uvm_pagelookup(uobj, curoff)) != NULL) 697 atomic_clearbits_int(&pp->pg_flags, 698 PG_CLEANCHK); 699 } 700 } 701 702 ppsp = NULL; /* XXX: shut up gcc */ 703 uvm_lock_pageq(); /* page queues locked */ 704 /* locked: both page queues and uobj */ 705 for (curoff = start; curoff < stop; curoff += PAGE_SIZE) { 706 if ((pp = uvm_pagelookup(uobj, curoff)) == NULL) 707 continue; 708 709 /* 710 * handle case where we do not need to clean page (either 711 * because we are not clean or because page is not dirty or 712 * is busy): 713 * 714 * NOTE: we are allowed to deactivate a non-wired active 715 * PG_BUSY page, but once a PG_BUSY page is on the inactive 716 * queue it must stay put until it is !PG_BUSY (so as not to 717 * confuse pagedaemon). 718 */ 719 720 if ((flags & PGO_CLEANIT) == 0 || (pp->pg_flags & PG_BUSY) != 0) { 721 needs_clean = FALSE; 722 if ((pp->pg_flags & PG_BUSY) != 0 && 723 (flags & (PGO_CLEANIT|PGO_SYNCIO)) == 724 (PGO_CLEANIT|PGO_SYNCIO)) 725 need_iosync = TRUE; 726 } else { 727 /* 728 * freeing: nuke all mappings so we can sync 729 * PG_CLEAN bit with no race 730 */ 731 if ((pp->pg_flags & PG_CLEAN) != 0 && 732 (flags & PGO_FREE) != 0 && 733 (pp->pg_flags & PQ_ACTIVE) != 0) 734 pmap_page_protect(pp, VM_PROT_NONE); 735 if ((pp->pg_flags & PG_CLEAN) != 0 && 736 pmap_is_modified(pp)) 737 atomic_clearbits_int(&pp->pg_flags, PG_CLEAN); 738 atomic_setbits_int(&pp->pg_flags, PG_CLEANCHK); 739 740 needs_clean = ((pp->pg_flags & PG_CLEAN) == 0); 741 } 742 743 /* 744 * if we don't need a clean... deactivate/free pages then cont. 745 */ 746 if (!needs_clean) { 747 if (flags & PGO_DEACTIVATE) { 748 if (pp->wire_count == 0) { 749 pmap_page_protect(pp, VM_PROT_NONE); 750 uvm_pagedeactivate(pp); 751 } 752 } else if (flags & PGO_FREE) { 753 if (pp->pg_flags & PG_BUSY) { 754 atomic_setbits_int(&pp->pg_flags, 755 PG_WANTED); 756 uvm_unlock_pageq(); 757 UVM_UNLOCK_AND_WAIT(pp, 758 &uobj->vmobjlock, 0, "uvn_flsh", 0); 759 simple_lock(&uobj->vmobjlock); 760 uvm_lock_pageq(); 761 curoff -= PAGE_SIZE; 762 continue; 763 } else { 764 pmap_page_protect(pp, VM_PROT_NONE); 765 /* removed page from object */ 766 uvm_pagefree(pp); 767 } 768 } 769 continue; 770 } 771 772 /* 773 * pp points to a page in the locked object that we are 774 * working on. if it is !PG_CLEAN,!PG_BUSY and we asked 775 * for cleaning (PGO_CLEANIT). we clean it now. 776 * 777 * let uvm_pager_put attempted a clustered page out. 778 * note: locked: uobj and page queues. 779 */ 780 781 atomic_setbits_int(&pp->pg_flags, PG_BUSY); 782 UVM_PAGE_OWN(pp, "uvn_flush"); 783 pmap_page_protect(pp, VM_PROT_READ); 784 /* if we're async, free the page in aiodoned */ 785 if ((flags & (PGO_FREE|PGO_SYNCIO)) == PGO_FREE) 786 atomic_setbits_int(&pp->pg_flags, PG_RELEASED); 787 ReTry: 788 ppsp = pps; 789 npages = sizeof(pps) / sizeof(struct vm_page *); 790 791 /* locked: page queues, uobj */ 792 result = uvm_pager_put(uobj, pp, &ppsp, &npages, 793 flags | PGO_DOACTCLUST, start, stop); 794 /* unlocked: page queues, uobj */ 795 796 /* 797 * if we did an async I/O it is remotely possible for the 798 * async i/o to complete and the page "pp" be freed or what 799 * not before we get a chance to relock the object. Therefore, 800 * we only touch it when it won't be freed, RELEASED took care 801 * of the rest. 802 */ 803 804 /* relock! */ 805 simple_lock(&uobj->vmobjlock); 806 uvm_lock_pageq(); 807 808 /* 809 * VM_PAGER_AGAIN: given the structure of this pager, this 810 * can only happen when we are doing async I/O and can't 811 * map the pages into kernel memory (pager_map) due to lack 812 * of vm space. if this happens we drop back to sync I/O. 813 */ 814 815 if (result == VM_PAGER_AGAIN) { 816 /* 817 * it is unlikely, but page could have been released 818 * while we had the object lock dropped. we ignore 819 * this now and retry the I/O. we will detect and 820 * handle the released page after the syncio I/O 821 * completes. 822 */ 823 #ifdef DIAGNOSTIC 824 if (flags & PGO_SYNCIO) 825 panic("uvn_flush: PGO_SYNCIO return 'try again' error (impossible)"); 826 #endif 827 flags |= PGO_SYNCIO; 828 if (flags & PGO_FREE) 829 atomic_clearbits_int(&pp->pg_flags, 830 PG_RELEASED); 831 832 goto ReTry; 833 } 834 835 /* 836 * the cleaning operation is now done. finish up. note that 837 * on error (!OK, !PEND) uvm_pager_put drops the cluster for us. 838 * if success (OK, PEND) then uvm_pager_put returns the cluster 839 * to us in ppsp/npages. 840 */ 841 842 /* 843 * for pending async i/o if we are not deactivating 844 * we can move on to the next page. aiodoned deals with 845 * the freeing case for us. 846 */ 847 if (result == VM_PAGER_PEND && (flags & PGO_DEACTIVATE) == 0) 848 continue; 849 850 /* 851 * need to look at each page of the I/O operation, and do what 852 * we gotta do. 853 */ 854 855 for (lcv = 0 ; lcv < npages; lcv++) { 856 ptmp = ppsp[lcv]; 857 858 /* 859 * verify the page didn't get moved while obj was 860 * unlocked 861 */ 862 if (result == VM_PAGER_PEND && ptmp->uobject != uobj) 863 continue; 864 865 /* 866 * unbusy the page if I/O is done. note that for 867 * pending I/O it is possible that the I/O op 868 * finished before we relocked the object (in 869 * which case the page is no longer busy). 870 */ 871 872 if (result != VM_PAGER_PEND) { 873 if (ptmp->pg_flags & PG_WANTED) 874 /* still holding object lock */ 875 wakeup(ptmp); 876 877 atomic_clearbits_int(&ptmp->pg_flags, 878 PG_WANTED|PG_BUSY); 879 UVM_PAGE_OWN(ptmp, NULL); 880 atomic_setbits_int(&ptmp->pg_flags, 881 PG_CLEAN|PG_CLEANCHK); 882 if ((flags & PGO_FREE) == 0) 883 pmap_clear_modify(ptmp); 884 } 885 886 /* 887 * dispose of page 888 */ 889 890 if (flags & PGO_DEACTIVATE) { 891 if (ptmp->wire_count == 0) { 892 pmap_page_protect(ptmp, VM_PROT_NONE); 893 uvm_pagedeactivate(ptmp); 894 } 895 } else if (flags & PGO_FREE && 896 result != VM_PAGER_PEND) { 897 if (result != VM_PAGER_OK) { 898 printf("uvn_flush: obj=%p, " 899 "offset=0x%llx. error " 900 "during pageout.\n", 901 pp->uobject, 902 (long long)pp->offset); 903 printf("uvn_flush: WARNING: " 904 "changes to page may be " 905 "lost!\n"); 906 retval = FALSE; 907 } 908 pmap_page_protect(ptmp, VM_PROT_NONE); 909 uvm_pagefree(ptmp); 910 } 911 912 } /* end of "lcv" for loop */ 913 914 } /* end of "pp" for loop */ 915 916 /* 917 * done with pagequeues: unlock 918 */ 919 uvm_unlock_pageq(); 920 921 /* 922 * now wait for all I/O if required. 923 */ 924 if (need_iosync) { 925 while (uvn->u_nio != 0) { 926 uvn->u_flags |= UVM_VNODE_IOSYNC; 927 UVM_UNLOCK_AND_WAIT(&uvn->u_nio, &uvn->u_obj.vmobjlock, 928 FALSE, "uvn_flush",0); 929 simple_lock(&uvn->u_obj.vmobjlock); 930 } 931 if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED) 932 wakeup(&uvn->u_flags); 933 uvn->u_flags &= ~(UVM_VNODE_IOSYNC|UVM_VNODE_IOSYNCWANTED); 934 } 935 936 /* return, with object locked! */ 937 return(retval); 938 } 939 940 /* 941 * uvn_cluster 942 * 943 * we are about to do I/O in an object at offset. this function is called 944 * to establish a range of offsets around "offset" in which we can cluster 945 * I/O. 946 * 947 * - currently doesn't matter if obj locked or not. 948 */ 949 950 void 951 uvn_cluster(struct uvm_object *uobj, voff_t offset, voff_t *loffset, 952 voff_t *hoffset) 953 { 954 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 955 *loffset = offset; 956 957 if (*loffset >= uvn->u_size) 958 panic("uvn_cluster: offset out of range"); 959 960 /* 961 * XXX: old pager claims we could use VOP_BMAP to get maxcontig value. 962 */ 963 *hoffset = *loffset + MAXBSIZE; 964 if (*hoffset > round_page(uvn->u_size)) /* past end? */ 965 *hoffset = round_page(uvn->u_size); 966 967 return; 968 } 969 970 /* 971 * uvn_put: flush page data to backing store. 972 * 973 * => prefer map unlocked (not required) 974 * => object must be locked! we will _unlock_ it before starting I/O. 975 * => flags: PGO_SYNCIO -- use sync. I/O 976 * => note: caller must set PG_CLEAN and pmap_clear_modify (if needed) 977 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync. 978 * [thus we never do async i/o! see iodone comment] 979 */ 980 981 int 982 uvn_put(struct uvm_object *uobj, struct vm_page **pps, int npages, int flags) 983 { 984 int retval; 985 986 /* note: object locked */ 987 retval = uvn_io((struct uvm_vnode*)uobj, pps, npages, flags, UIO_WRITE); 988 /* note: object unlocked */ 989 990 return(retval); 991 } 992 993 994 /* 995 * uvn_get: get pages (synchronously) from backing store 996 * 997 * => prefer map unlocked (not required) 998 * => object must be locked! we will _unlock_ it before starting any I/O. 999 * => flags: PGO_ALLPAGES: get all of the pages 1000 * PGO_LOCKED: fault data structures are locked 1001 * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx] 1002 * => NOTE: caller must check for released pages!! 1003 */ 1004 1005 int 1006 uvn_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps, 1007 int *npagesp, int centeridx, vm_prot_t access_type, int advice, int flags) 1008 { 1009 voff_t current_offset; 1010 struct vm_page *ptmp; 1011 int lcv, result, gotpages; 1012 boolean_t done; 1013 1014 /* 1015 * step 1: handled the case where fault data structures are locked. 1016 */ 1017 1018 if (flags & PGO_LOCKED) { 1019 1020 /* 1021 * gotpages is the current number of pages we've gotten (which 1022 * we pass back up to caller via *npagesp. 1023 */ 1024 1025 gotpages = 0; 1026 1027 /* 1028 * step 1a: get pages that are already resident. only do this 1029 * if the data structures are locked (i.e. the first time 1030 * through). 1031 */ 1032 1033 done = TRUE; /* be optimistic */ 1034 1035 for (lcv = 0, current_offset = offset ; lcv < *npagesp ; 1036 lcv++, current_offset += PAGE_SIZE) { 1037 1038 /* do we care about this page? if not, skip it */ 1039 if (pps[lcv] == PGO_DONTCARE) 1040 continue; 1041 1042 /* lookup page */ 1043 ptmp = uvm_pagelookup(uobj, current_offset); 1044 1045 /* to be useful must get a non-busy, non-released pg */ 1046 if (ptmp == NULL || 1047 (ptmp->pg_flags & PG_BUSY) != 0) { 1048 if (lcv == centeridx || (flags & PGO_ALLPAGES) 1049 != 0) 1050 done = FALSE; /* need to do a wait or I/O! */ 1051 continue; 1052 } 1053 1054 /* 1055 * useful page: busy/lock it and plug it in our 1056 * result array 1057 */ 1058 atomic_setbits_int(&ptmp->pg_flags, PG_BUSY); 1059 UVM_PAGE_OWN(ptmp, "uvn_get1"); 1060 pps[lcv] = ptmp; 1061 gotpages++; 1062 1063 } /* "for" lcv loop */ 1064 1065 /* 1066 * XXX: given the "advice", should we consider async read-ahead? 1067 * XXX: fault current does deactive of pages behind us. is 1068 * this good (other callers might now). 1069 */ 1070 /* 1071 * XXX: read-ahead currently handled by buffer cache (bread) 1072 * level. 1073 * XXX: no async i/o available. 1074 * XXX: so we don't do anything now. 1075 */ 1076 1077 /* 1078 * step 1c: now we've either done everything needed or we to 1079 * unlock and do some waiting or I/O. 1080 */ 1081 1082 *npagesp = gotpages; /* let caller know */ 1083 if (done) 1084 return(VM_PAGER_OK); /* bingo! */ 1085 else 1086 /* EEK! Need to unlock and I/O */ 1087 return(VM_PAGER_UNLOCK); 1088 } 1089 1090 /* 1091 * step 2: get non-resident or busy pages. 1092 * object is locked. data structures are unlocked. 1093 * 1094 * XXX: because we can't do async I/O at this level we get things 1095 * page at a time (otherwise we'd chunk). the VOP_READ() will do 1096 * async-read-ahead for us at a lower level. 1097 */ 1098 1099 for (lcv = 0, current_offset = offset; 1100 lcv < *npagesp ; lcv++, current_offset += PAGE_SIZE) { 1101 1102 /* skip over pages we've already gotten or don't want */ 1103 /* skip over pages we don't _have_ to get */ 1104 if (pps[lcv] != NULL || (lcv != centeridx && 1105 (flags & PGO_ALLPAGES) == 0)) 1106 continue; 1107 1108 /* 1109 * we have yet to locate the current page (pps[lcv]). we first 1110 * look for a page that is already at the current offset. if 1111 * we fine a page, we check to see if it is busy or released. 1112 * if that is the case, then we sleep on the page until it is 1113 * no longer busy or released and repeat the lookup. if the 1114 * page we found is neither busy nor released, then we busy it 1115 * (so we own it) and plug it into pps[lcv]. this breaks the 1116 * following while loop and indicates we are ready to move on 1117 * to the next page in the "lcv" loop above. 1118 * 1119 * if we exit the while loop with pps[lcv] still set to NULL, 1120 * then it means that we allocated a new busy/fake/clean page 1121 * ptmp in the object and we need to do I/O to fill in the data. 1122 */ 1123 1124 while (pps[lcv] == NULL) { /* top of "pps" while loop */ 1125 1126 /* look for a current page */ 1127 ptmp = uvm_pagelookup(uobj, current_offset); 1128 1129 /* nope? allocate one now (if we can) */ 1130 if (ptmp == NULL) { 1131 1132 ptmp = uvm_pagealloc(uobj, current_offset, 1133 NULL, 0); 1134 1135 /* out of RAM? */ 1136 if (ptmp == NULL) { 1137 simple_unlock(&uobj->vmobjlock); 1138 uvm_wait("uvn_getpage"); 1139 simple_lock(&uobj->vmobjlock); 1140 1141 /* goto top of pps while loop */ 1142 continue; 1143 } 1144 1145 /* 1146 * got new page ready for I/O. break pps 1147 * while loop. pps[lcv] is still NULL. 1148 */ 1149 break; 1150 } 1151 1152 /* page is there, see if we need to wait on it */ 1153 if ((ptmp->pg_flags & PG_BUSY) != 0) { 1154 atomic_setbits_int(&ptmp->pg_flags, PG_WANTED); 1155 UVM_UNLOCK_AND_WAIT(ptmp, 1156 &uobj->vmobjlock, FALSE, "uvn_get",0); 1157 simple_lock(&uobj->vmobjlock); 1158 continue; /* goto top of pps while loop */ 1159 } 1160 1161 /* 1162 * if we get here then the page has become resident 1163 * and unbusy between steps 1 and 2. we busy it 1164 * now (so we own it) and set pps[lcv] (so that we 1165 * exit the while loop). 1166 */ 1167 atomic_setbits_int(&ptmp->pg_flags, PG_BUSY); 1168 UVM_PAGE_OWN(ptmp, "uvn_get2"); 1169 pps[lcv] = ptmp; 1170 } 1171 1172 /* 1173 * if we own the a valid page at the correct offset, pps[lcv] 1174 * will point to it. nothing more to do except go to the 1175 * next page. 1176 */ 1177 1178 if (pps[lcv]) 1179 continue; /* next lcv */ 1180 1181 /* 1182 * we have a "fake/busy/clean" page that we just allocated. do 1183 * I/O to fill it with valid data. note that object must be 1184 * locked going into uvn_io, but will be unlocked afterwards. 1185 */ 1186 1187 result = uvn_io((struct uvm_vnode *) uobj, &ptmp, 1, 1188 PGO_SYNCIO, UIO_READ); 1189 1190 /* 1191 * I/O done. object is unlocked (by uvn_io). because we used 1192 * syncio the result can not be PEND or AGAIN. we must relock 1193 * and check for errors. 1194 */ 1195 1196 /* lock object. check for errors. */ 1197 simple_lock(&uobj->vmobjlock); 1198 if (result != VM_PAGER_OK) { 1199 if (ptmp->pg_flags & PG_WANTED) 1200 /* object lock still held */ 1201 wakeup(ptmp); 1202 1203 atomic_clearbits_int(&ptmp->pg_flags, 1204 PG_WANTED|PG_BUSY); 1205 UVM_PAGE_OWN(ptmp, NULL); 1206 uvm_lock_pageq(); 1207 uvm_pagefree(ptmp); 1208 uvm_unlock_pageq(); 1209 simple_unlock(&uobj->vmobjlock); 1210 return(result); 1211 } 1212 1213 /* 1214 * we got the page! clear the fake flag (indicates valid 1215 * data now in page) and plug into our result array. note 1216 * that page is still busy. 1217 * 1218 * it is the callers job to: 1219 * => check if the page is released 1220 * => unbusy the page 1221 * => activate the page 1222 */ 1223 1224 /* data is valid ... */ 1225 atomic_clearbits_int(&ptmp->pg_flags, PG_FAKE); 1226 pmap_clear_modify(ptmp); /* ... and clean */ 1227 pps[lcv] = ptmp; 1228 1229 } /* lcv loop */ 1230 1231 /* 1232 * finally, unlock object and return. 1233 */ 1234 1235 simple_unlock(&uobj->vmobjlock); 1236 return (VM_PAGER_OK); 1237 } 1238 1239 /* 1240 * uvn_io: do I/O to a vnode 1241 * 1242 * => prefer map unlocked (not required) 1243 * => object must be locked! we will _unlock_ it before starting I/O. 1244 * => flags: PGO_SYNCIO -- use sync. I/O 1245 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync. 1246 * [thus we never do async i/o! see iodone comment] 1247 */ 1248 1249 int 1250 uvn_io(struct uvm_vnode *uvn, vm_page_t *pps, int npages, int flags, int rw) 1251 { 1252 struct vnode *vn; 1253 struct uio uio; 1254 struct iovec iov; 1255 vaddr_t kva; 1256 off_t file_offset; 1257 int waitf, result, mapinflags; 1258 size_t got, wanted; 1259 1260 /* 1261 * init values 1262 */ 1263 1264 waitf = (flags & PGO_SYNCIO) ? M_WAITOK : M_NOWAIT; 1265 vn = (struct vnode *) uvn; 1266 file_offset = pps[0]->offset; 1267 1268 /* 1269 * check for sync'ing I/O. 1270 */ 1271 1272 while (uvn->u_flags & UVM_VNODE_IOSYNC) { 1273 if (waitf == M_NOWAIT) { 1274 simple_unlock(&uvn->u_obj.vmobjlock); 1275 return(VM_PAGER_AGAIN); 1276 } 1277 uvn->u_flags |= UVM_VNODE_IOSYNCWANTED; 1278 UVM_UNLOCK_AND_WAIT(&uvn->u_flags, &uvn->u_obj.vmobjlock, 1279 FALSE, "uvn_iosync",0); 1280 simple_lock(&uvn->u_obj.vmobjlock); 1281 } 1282 1283 /* 1284 * check size 1285 */ 1286 1287 if (file_offset >= uvn->u_size) { 1288 simple_unlock(&uvn->u_obj.vmobjlock); 1289 return(VM_PAGER_BAD); 1290 } 1291 1292 /* 1293 * first try and map the pages in (without waiting) 1294 */ 1295 1296 mapinflags = (rw == UIO_READ) ? 1297 UVMPAGER_MAPIN_READ : UVMPAGER_MAPIN_WRITE; 1298 1299 kva = uvm_pagermapin(pps, npages, mapinflags); 1300 if (kva == 0 && waitf == M_NOWAIT) { 1301 simple_unlock(&uvn->u_obj.vmobjlock); 1302 return(VM_PAGER_AGAIN); 1303 } 1304 1305 /* 1306 * ok, now bump u_nio up. at this point we are done with uvn 1307 * and can unlock it. if we still don't have a kva, try again 1308 * (this time with sleep ok). 1309 */ 1310 1311 uvn->u_nio++; /* we have an I/O in progress! */ 1312 simple_unlock(&uvn->u_obj.vmobjlock); 1313 /* NOTE: object now unlocked */ 1314 if (kva == 0) 1315 kva = uvm_pagermapin(pps, npages, 1316 mapinflags | UVMPAGER_MAPIN_WAITOK); 1317 1318 /* 1319 * ok, mapped in. our pages are PG_BUSY so they are not going to 1320 * get touched (so we can look at "offset" without having to lock 1321 * the object). set up for I/O. 1322 */ 1323 1324 /* 1325 * fill out uio/iov 1326 */ 1327 1328 iov.iov_base = (caddr_t) kva; 1329 wanted = npages << PAGE_SHIFT; 1330 if (file_offset + wanted > uvn->u_size) 1331 wanted = uvn->u_size - file_offset; /* XXX: needed? */ 1332 iov.iov_len = wanted; 1333 uio.uio_iov = &iov; 1334 uio.uio_iovcnt = 1; 1335 uio.uio_offset = file_offset; 1336 uio.uio_segflg = UIO_SYSSPACE; 1337 uio.uio_rw = rw; 1338 uio.uio_resid = wanted; 1339 uio.uio_procp = curproc; 1340 1341 /* 1342 * do the I/O! (XXX: curproc?) 1343 */ 1344 1345 /* 1346 * This process may already have this vnode locked, if we faulted in 1347 * copyin() or copyout() on a region backed by this vnode 1348 * while doing I/O to the vnode. If this is the case, don't 1349 * panic.. instead, return the error to the user. 1350 * 1351 * XXX this is a stopgap to prevent a panic. 1352 * Ideally, this kind of operation *should* work. 1353 */ 1354 result = 0; 1355 if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0) 1356 result = vn_lock(vn, LK_EXCLUSIVE | LK_RECURSEFAIL, curproc); 1357 1358 if (result == 0) { 1359 /* NOTE: vnode now locked! */ 1360 1361 if (rw == UIO_READ) 1362 result = VOP_READ(vn, &uio, 0, curproc->p_ucred); 1363 else 1364 result = VOP_WRITE(vn, &uio, 0, curproc->p_ucred); 1365 1366 if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0) 1367 VOP_UNLOCK(vn, 0, curproc); 1368 } 1369 1370 /* NOTE: vnode now unlocked (unless vnislocked) */ 1371 1372 /* 1373 * result == unix style errno (0 == OK!) 1374 * 1375 * zero out rest of buffer (if needed) 1376 */ 1377 1378 if (result == 0) { 1379 got = wanted - uio.uio_resid; 1380 1381 if (wanted && got == 0) { 1382 result = EIO; /* XXX: error? */ 1383 } else if (got < PAGE_SIZE * npages && rw == UIO_READ) { 1384 memset((void *) (kva + got), 0, 1385 (npages << PAGE_SHIFT) - got); 1386 } 1387 } 1388 1389 /* 1390 * now remove pager mapping 1391 */ 1392 uvm_pagermapout(kva, npages); 1393 1394 /* 1395 * now clean up the object (i.e. drop I/O count) 1396 */ 1397 1398 simple_lock(&uvn->u_obj.vmobjlock); 1399 /* NOTE: object now locked! */ 1400 1401 uvn->u_nio--; /* I/O DONE! */ 1402 if ((uvn->u_flags & UVM_VNODE_IOSYNC) != 0 && uvn->u_nio == 0) { 1403 wakeup(&uvn->u_nio); 1404 } 1405 simple_unlock(&uvn->u_obj.vmobjlock); 1406 /* NOTE: object now unlocked! */ 1407 1408 /* 1409 * done! 1410 */ 1411 1412 if (result == 0) 1413 return(VM_PAGER_OK); 1414 else 1415 return(VM_PAGER_ERROR); 1416 } 1417 1418 /* 1419 * uvm_vnp_uncache: disable "persisting" in a vnode... when last reference 1420 * is gone we will kill the object (flushing dirty pages back to the vnode 1421 * if needed). 1422 * 1423 * => returns TRUE if there was no uvm_object attached or if there was 1424 * one and we killed it [i.e. if there is no active uvn] 1425 * => called with the vnode VOP_LOCK'd [we will unlock it for I/O, if 1426 * needed] 1427 * 1428 * => XXX: given that we now kill uvn's when a vnode is recycled (without 1429 * having to hold a reference on the vnode) and given a working 1430 * uvm_vnp_sync(), how does that effect the need for this function? 1431 * [XXXCDC: seems like it can die?] 1432 * 1433 * => XXX: this function should DIE once we merge the VM and buffer 1434 * cache. 1435 * 1436 * research shows that this is called in the following places: 1437 * ext2fs_truncate, ffs_truncate, detrunc[msdosfs]: called when vnode 1438 * changes sizes 1439 * ext2fs_write, WRITE [ufs_readwrite], msdosfs_write: called when we 1440 * are written to 1441 * ex2fs_chmod, ufs_chmod: called if VTEXT vnode and the sticky bit 1442 * is off 1443 * ffs_realloccg: when we can't extend the current block and have 1444 * to allocate a new one we call this [XXX: why?] 1445 * nfsrv_rename, rename_files: called when the target filename is there 1446 * and we want to remove it 1447 * nfsrv_remove, sys_unlink: called on file we are removing 1448 * nfsrv_access: if VTEXT and we want WRITE access and we don't uncache 1449 * then return "text busy" 1450 * nfs_open: seems to uncache any file opened with nfs 1451 * vn_writechk: if VTEXT vnode and can't uncache return "text busy" 1452 */ 1453 1454 boolean_t 1455 uvm_vnp_uncache(struct vnode *vp) 1456 { 1457 struct uvm_vnode *uvn = &vp->v_uvm; 1458 1459 /* 1460 * lock uvn part of the vnode and check to see if we need to do anything 1461 */ 1462 1463 simple_lock(&uvn->u_obj.vmobjlock); 1464 if ((uvn->u_flags & UVM_VNODE_VALID) == 0 || 1465 (uvn->u_flags & UVM_VNODE_BLOCKED) != 0) { 1466 simple_unlock(&uvn->u_obj.vmobjlock); 1467 return(TRUE); 1468 } 1469 1470 /* 1471 * we have a valid, non-blocked uvn. clear persist flag. 1472 * if uvn is currently active we can return now. 1473 */ 1474 1475 uvn->u_flags &= ~UVM_VNODE_CANPERSIST; 1476 if (uvn->u_obj.uo_refs) { 1477 simple_unlock(&uvn->u_obj.vmobjlock); 1478 return(FALSE); 1479 } 1480 1481 /* 1482 * uvn is currently persisting! we have to gain a reference to 1483 * it so that we can call uvn_detach to kill the uvn. 1484 */ 1485 1486 vref(vp); /* seems ok, even with VOP_LOCK */ 1487 uvn->u_obj.uo_refs++; /* value is now 1 */ 1488 simple_unlock(&uvn->u_obj.vmobjlock); 1489 1490 #ifdef VFSLCKDEBUG 1491 /* 1492 * carry over sanity check from old vnode pager: the vnode should 1493 * be VOP_LOCK'd, and we confirm it here. 1494 */ 1495 if ((vp->v_flag & VLOCKSWORK) && !VOP_ISLOCKED(vp)) 1496 panic("uvm_vnp_uncache: vnode not locked!"); 1497 #endif 1498 1499 /* 1500 * now drop our reference to the vnode. if we have the sole 1501 * reference to the vnode then this will cause it to die [as we 1502 * just cleared the persist flag]. we have to unlock the vnode 1503 * while we are doing this as it may trigger I/O. 1504 * 1505 * XXX: it might be possible for uvn to get reclaimed while we are 1506 * unlocked causing us to return TRUE when we should not. we ignore 1507 * this as a false-positive return value doesn't hurt us. 1508 */ 1509 VOP_UNLOCK(vp, 0, curproc); 1510 uvn_detach(&uvn->u_obj); 1511 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, curproc); 1512 1513 /* 1514 * and return... 1515 */ 1516 1517 return(TRUE); 1518 } 1519 1520 /* 1521 * uvm_vnp_setsize: grow or shrink a vnode uvn 1522 * 1523 * grow => just update size value 1524 * shrink => toss un-needed pages 1525 * 1526 * => we assume that the caller has a reference of some sort to the 1527 * vnode in question so that it will not be yanked out from under 1528 * us. 1529 * 1530 * called from: 1531 * => truncate fns (ext2fs_truncate, ffs_truncate, detrunc[msdos]) 1532 * => "write" fns (ext2fs_write, WRITE [ufs/ufs], msdosfs_write, nfs_write) 1533 * => ffs_balloc [XXX: why? doesn't WRITE handle?] 1534 * => NFS: nfs_loadattrcache, nfs_getattrcache, nfs_setattr 1535 * => union fs: union_newsize 1536 */ 1537 1538 void 1539 uvm_vnp_setsize(struct vnode *vp, voff_t newsize) 1540 { 1541 struct uvm_vnode *uvn = &vp->v_uvm; 1542 1543 /* 1544 * lock uvn and check for valid object, and if valid: do it! 1545 */ 1546 simple_lock(&uvn->u_obj.vmobjlock); 1547 if (uvn->u_flags & UVM_VNODE_VALID) { 1548 1549 /* 1550 * now check if the size has changed: if we shrink we had better 1551 * toss some pages... 1552 */ 1553 1554 if (uvn->u_size > newsize) { 1555 (void)uvn_flush(&uvn->u_obj, newsize, 1556 uvn->u_size, PGO_FREE); 1557 } 1558 uvn->u_size = newsize; 1559 } 1560 simple_unlock(&uvn->u_obj.vmobjlock); 1561 1562 /* 1563 * done 1564 */ 1565 return; 1566 } 1567 1568 /* 1569 * uvm_vnp_sync: flush all dirty VM pages back to their backing vnodes. 1570 * 1571 * => called from sys_sync with no VM structures locked 1572 * => only one process can do a sync at a time (because the uvn 1573 * structure only has one queue for sync'ing). we ensure this 1574 * by holding the uvn_sync_lock while the sync is in progress. 1575 * other processes attempting a sync will sleep on this lock 1576 * until we are done. 1577 */ 1578 void 1579 uvm_vnp_sync(struct mount *mp) 1580 { 1581 struct uvm_vnode *uvn; 1582 struct vnode *vp; 1583 1584 /* 1585 * step 1: ensure we are only ones using the uvn_sync_q by locking 1586 * our lock... 1587 */ 1588 rw_enter_write(&uvn_sync_lock); 1589 1590 /* 1591 * step 2: build up a simpleq of uvns of interest based on the 1592 * write list. we gain a reference to uvns of interest. 1593 */ 1594 SIMPLEQ_INIT(&uvn_sync_q); 1595 LIST_FOREACH(uvn, &uvn_wlist, u_wlist) { 1596 1597 vp = (struct vnode *)uvn; 1598 if (mp && vp->v_mount != mp) 1599 continue; 1600 1601 /* 1602 * If the vnode is "blocked" it means it must be dying, which 1603 * in turn means its in the process of being flushed out so 1604 * we can safely skip it. 1605 * 1606 * note that uvn must already be valid because we found it on 1607 * the wlist (this also means it can't be ALOCK'd). 1608 */ 1609 if ((uvn->u_flags & UVM_VNODE_BLOCKED) != 0) 1610 continue; 1611 1612 1613 /* 1614 * gain reference. watch out for persisting uvns (need to 1615 * regain vnode REF). 1616 */ 1617 if (uvn->u_obj.uo_refs == 0) 1618 vref(vp); 1619 uvn->u_obj.uo_refs++; 1620 1621 SIMPLEQ_INSERT_HEAD(&uvn_sync_q, uvn, u_syncq); 1622 } 1623 1624 /* step 3: we now have a list of uvn's that may need cleaning. */ 1625 SIMPLEQ_FOREACH(uvn, &uvn_sync_q, u_syncq) { 1626 #ifdef DEBUG 1627 if (uvn->u_flags & UVM_VNODE_DYING) { 1628 printf("uvm_vnp_sync: dying vnode on sync list\n"); 1629 } 1630 #endif 1631 uvn_flush(&uvn->u_obj, 0, 0, PGO_CLEANIT|PGO_ALLPAGES|PGO_DOACTCLUST); 1632 1633 /* 1634 * if we have the only reference and we just cleaned the uvn, 1635 * then we can pull it out of the UVM_VNODE_WRITEABLE state 1636 * thus allowing us to avoid thinking about flushing it again 1637 * on later sync ops. 1638 */ 1639 if (uvn->u_obj.uo_refs == 1 && 1640 (uvn->u_flags & UVM_VNODE_WRITEABLE)) { 1641 LIST_REMOVE(uvn, u_wlist); 1642 uvn->u_flags &= ~UVM_VNODE_WRITEABLE; 1643 } 1644 1645 /* now drop our reference to the uvn */ 1646 uvn_detach(&uvn->u_obj); 1647 } 1648 1649 rw_exit_write(&uvn_sync_lock); 1650 } 1651