1 /* $OpenBSD: uvm_vnode.c,v 1.129 2022/09/21 07:32:59 mpi Exp $ */ 2 /* $NetBSD: uvm_vnode.c,v 1.36 2000/11/24 20:34:01 chs Exp $ */ 3 4 /* 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * Copyright (c) 1991, 1993 7 * The Regents of the University of California. 8 * Copyright (c) 1990 University of Utah. 9 * 10 * All rights reserved. 11 * 12 * This code is derived from software contributed to Berkeley by 13 * the Systems Programming Group of the University of Utah Computer 14 * Science Department. 15 * 16 * Redistribution and use in source and binary forms, with or without 17 * modification, are permitted provided that the following conditions 18 * are met: 19 * 1. Redistributions of source code must retain the above copyright 20 * notice, this list of conditions and the following disclaimer. 21 * 2. Redistributions in binary form must reproduce the above copyright 22 * notice, this list of conditions and the following disclaimer in the 23 * documentation and/or other materials provided with the distribution. 24 * 3. Neither the name of the University nor the names of its contributors 25 * may be used to endorse or promote products derived from this software 26 * without specific prior written permission. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 29 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 30 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 31 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 32 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 33 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 34 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 35 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 36 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 37 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 38 * SUCH DAMAGE. 39 * 40 * @(#)vnode_pager.c 8.8 (Berkeley) 2/13/94 41 * from: Id: uvm_vnode.c,v 1.1.2.26 1998/02/02 20:38:07 chuck Exp 42 */ 43 44 /* 45 * uvm_vnode.c: the vnode pager. 46 */ 47 48 #include <sys/param.h> 49 #include <sys/systm.h> 50 #include <sys/proc.h> 51 #include <sys/malloc.h> 52 #include <sys/vnode.h> 53 #include <sys/lock.h> 54 #include <sys/disklabel.h> 55 #include <sys/fcntl.h> 56 #include <sys/conf.h> 57 #include <sys/rwlock.h> 58 #include <sys/dkio.h> 59 #include <sys/specdev.h> 60 61 #include <uvm/uvm.h> 62 #include <uvm/uvm_vnode.h> 63 64 /* 65 * private global data structure 66 * 67 * we keep a list of writeable active vnode-backed VM objects for sync op. 68 * we keep a simpleq of vnodes that are currently being sync'd. 69 */ 70 71 LIST_HEAD(uvn_list_struct, uvm_vnode); 72 struct uvn_list_struct uvn_wlist; /* writeable uvns */ 73 74 SIMPLEQ_HEAD(uvn_sq_struct, uvm_vnode); 75 struct uvn_sq_struct uvn_sync_q; /* sync'ing uvns */ 76 struct rwlock uvn_sync_lock; /* locks sync operation */ 77 78 extern int rebooting; 79 80 /* 81 * functions 82 */ 83 void uvn_cluster(struct uvm_object *, voff_t, voff_t *, voff_t *); 84 void uvn_detach(struct uvm_object *); 85 boolean_t uvn_flush(struct uvm_object *, voff_t, voff_t, int); 86 int uvn_get(struct uvm_object *, voff_t, vm_page_t *, int *, int, 87 vm_prot_t, int, int); 88 void uvn_init(void); 89 int uvn_io(struct uvm_vnode *, vm_page_t *, int, int, int); 90 int uvn_put(struct uvm_object *, vm_page_t *, int, boolean_t); 91 void uvn_reference(struct uvm_object *); 92 93 /* 94 * master pager structure 95 */ 96 const struct uvm_pagerops uvm_vnodeops = { 97 .pgo_init = uvn_init, 98 .pgo_reference = uvn_reference, 99 .pgo_detach = uvn_detach, 100 .pgo_flush = uvn_flush, 101 .pgo_get = uvn_get, 102 .pgo_put = uvn_put, 103 .pgo_cluster = uvn_cluster, 104 /* use generic version of this: see uvm_pager.c */ 105 .pgo_mk_pcluster = uvm_mk_pcluster, 106 }; 107 108 /* 109 * the ops! 110 */ 111 /* 112 * uvn_init 113 * 114 * init pager private data structures. 115 */ 116 void 117 uvn_init(void) 118 { 119 120 LIST_INIT(&uvn_wlist); 121 /* note: uvn_sync_q init'd in uvm_vnp_sync() */ 122 rw_init_flags(&uvn_sync_lock, "uvnsync", RWL_IS_VNODE); 123 } 124 125 /* 126 * uvn_attach 127 * 128 * attach a vnode structure to a VM object. if the vnode is already 129 * attached, then just bump the reference count by one and return the 130 * VM object. if not already attached, attach and return the new VM obj. 131 * the "accessprot" tells the max access the attaching thread wants to 132 * our pages. 133 * 134 * => in fact, nothing should be locked so that we can sleep here. 135 * => note that uvm_object is first thing in vnode structure, so their 136 * pointers are equiv. 137 */ 138 struct uvm_object * 139 uvn_attach(struct vnode *vp, vm_prot_t accessprot) 140 { 141 struct uvm_vnode *uvn = vp->v_uvm; 142 struct vattr vattr; 143 int oldflags, result; 144 struct partinfo pi; 145 u_quad_t used_vnode_size = 0; 146 147 /* first get a lock on the uvn. */ 148 while (uvn->u_flags & UVM_VNODE_BLOCKED) { 149 uvn->u_flags |= UVM_VNODE_WANTED; 150 tsleep_nsec(uvn, PVM, "uvn_attach", INFSLP); 151 } 152 153 /* if we're mapping a BLK device, make sure it is a disk. */ 154 if (vp->v_type == VBLK && bdevsw[major(vp->v_rdev)].d_type != D_DISK) { 155 return NULL; 156 } 157 158 /* 159 * now uvn must not be in a blocked state. 160 * first check to see if it is already active, in which case 161 * we can bump the reference count, check to see if we need to 162 * add it to the writeable list, and then return. 163 */ 164 rw_enter(uvn->u_obj.vmobjlock, RW_WRITE); 165 if (uvn->u_flags & UVM_VNODE_VALID) { /* already active? */ 166 167 /* regain vref if we were persisting */ 168 if (uvn->u_obj.uo_refs == 0) { 169 vref(vp); 170 } 171 uvn->u_obj.uo_refs++; /* bump uvn ref! */ 172 rw_exit(uvn->u_obj.vmobjlock); 173 174 /* check for new writeable uvn */ 175 if ((accessprot & PROT_WRITE) != 0 && 176 (uvn->u_flags & UVM_VNODE_WRITEABLE) == 0) { 177 LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist); 178 /* we are now on wlist! */ 179 uvn->u_flags |= UVM_VNODE_WRITEABLE; 180 } 181 182 return (&uvn->u_obj); 183 } 184 rw_exit(uvn->u_obj.vmobjlock); 185 186 /* 187 * need to call VOP_GETATTR() to get the attributes, but that could 188 * block (due to I/O), so we want to unlock the object before calling. 189 * however, we want to keep anyone else from playing with the object 190 * while it is unlocked. to do this we set UVM_VNODE_ALOCK which 191 * prevents anyone from attaching to the vnode until we are done with 192 * it. 193 */ 194 uvn->u_flags = UVM_VNODE_ALOCK; 195 196 if (vp->v_type == VBLK) { 197 /* 198 * We could implement this as a specfs getattr call, but: 199 * 200 * (1) VOP_GETATTR() would get the file system 201 * vnode operation, not the specfs operation. 202 * 203 * (2) All we want is the size, anyhow. 204 */ 205 result = (*bdevsw[major(vp->v_rdev)].d_ioctl)(vp->v_rdev, 206 DIOCGPART, (caddr_t)&pi, FREAD, curproc); 207 if (result == 0) { 208 /* XXX should remember blocksize */ 209 used_vnode_size = (u_quad_t)pi.disklab->d_secsize * 210 (u_quad_t)DL_GETPSIZE(pi.part); 211 } 212 } else { 213 result = VOP_GETATTR(vp, &vattr, curproc->p_ucred, curproc); 214 if (result == 0) 215 used_vnode_size = vattr.va_size; 216 } 217 218 if (result != 0) { 219 if (uvn->u_flags & UVM_VNODE_WANTED) 220 wakeup(uvn); 221 uvn->u_flags = 0; 222 return NULL; 223 } 224 225 /* 226 * make sure that the newsize fits within a vaddr_t 227 * XXX: need to revise addressing data types 228 */ 229 #ifdef DEBUG 230 if (vp->v_type == VBLK) 231 printf("used_vnode_size = %llu\n", (long long)used_vnode_size); 232 #endif 233 234 /* now set up the uvn. */ 235 KASSERT(uvn->u_obj.uo_refs == 0); 236 uvn->u_obj.uo_refs++; 237 oldflags = uvn->u_flags; 238 uvn->u_flags = UVM_VNODE_VALID|UVM_VNODE_CANPERSIST; 239 uvn->u_nio = 0; 240 uvn->u_size = used_vnode_size; 241 242 /* if write access, we need to add it to the wlist */ 243 if (accessprot & PROT_WRITE) { 244 LIST_INSERT_HEAD(&uvn_wlist, uvn, u_wlist); 245 uvn->u_flags |= UVM_VNODE_WRITEABLE; /* we are on wlist! */ 246 } 247 248 /* 249 * add a reference to the vnode. this reference will stay as long 250 * as there is a valid mapping of the vnode. dropped when the 251 * reference count goes to zero [and we either free or persist]. 252 */ 253 vref(vp); 254 if (oldflags & UVM_VNODE_WANTED) 255 wakeup(uvn); 256 257 return &uvn->u_obj; 258 } 259 260 261 /* 262 * uvn_reference 263 * 264 * duplicate a reference to a VM object. Note that the reference 265 * count must already be at least one (the passed in reference) so 266 * there is no chance of the uvn being killed out here. 267 * 268 * => caller must be using the same accessprot as was used at attach time 269 */ 270 271 272 void 273 uvn_reference(struct uvm_object *uobj) 274 { 275 #ifdef DEBUG 276 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 277 #endif 278 279 #ifdef DEBUG 280 if ((uvn->u_flags & UVM_VNODE_VALID) == 0) { 281 printf("uvn_reference: ref=%d, flags=0x%x\n", 282 uobj->uo_refs, uvn->u_flags); 283 panic("uvn_reference: invalid state"); 284 } 285 #endif 286 rw_enter(uobj->vmobjlock, RW_WRITE); 287 uobj->uo_refs++; 288 rw_exit(uobj->vmobjlock); 289 } 290 291 /* 292 * uvn_detach 293 * 294 * remove a reference to a VM object. 295 * 296 * => caller must call with map locked. 297 * => this starts the detach process, but doesn't have to finish it 298 * (async i/o could still be pending). 299 */ 300 void 301 uvn_detach(struct uvm_object *uobj) 302 { 303 struct uvm_vnode *uvn; 304 struct vnode *vp; 305 int oldflags; 306 307 rw_enter(uobj->vmobjlock, RW_WRITE); 308 uobj->uo_refs--; /* drop ref! */ 309 if (uobj->uo_refs) { /* still more refs */ 310 rw_exit(uobj->vmobjlock); 311 return; 312 } 313 314 /* get other pointers ... */ 315 uvn = (struct uvm_vnode *) uobj; 316 vp = uvn->u_vnode; 317 318 /* 319 * clear VTEXT flag now that there are no mappings left (VTEXT is used 320 * to keep an active text file from being overwritten). 321 */ 322 vp->v_flag &= ~VTEXT; 323 324 /* 325 * we just dropped the last reference to the uvn. see if we can 326 * let it "stick around". 327 */ 328 if (uvn->u_flags & UVM_VNODE_CANPERSIST) { 329 /* won't block */ 330 uvn_flush(uobj, 0, 0, PGO_DEACTIVATE|PGO_ALLPAGES); 331 goto out; 332 } 333 334 /* its a goner! */ 335 uvn->u_flags |= UVM_VNODE_DYING; 336 337 /* 338 * even though we may unlock in flush, no one can gain a reference 339 * to us until we clear the "dying" flag [because it blocks 340 * attaches]. we will not do that until after we've disposed of all 341 * the pages with uvn_flush(). note that before the flush the only 342 * pages that could be marked PG_BUSY are ones that are in async 343 * pageout by the daemon. (there can't be any pending "get"'s 344 * because there are no references to the object). 345 */ 346 (void) uvn_flush(uobj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES); 347 348 /* 349 * given the structure of this pager, the above flush request will 350 * create the following state: all the pages that were in the object 351 * have either been free'd or they are marked PG_BUSY and in the 352 * middle of an async io. If we still have pages we set the "relkill" 353 * state, so that in the case the vnode gets terminated we know 354 * to leave it alone. Otherwise we'll kill the vnode when it's empty. 355 */ 356 uvn->u_flags |= UVM_VNODE_RELKILL; 357 /* wait on any outstanding io */ 358 while (uobj->uo_npages && uvn->u_flags & UVM_VNODE_RELKILL) { 359 uvn->u_flags |= UVM_VNODE_IOSYNC; 360 rwsleep_nsec(&uvn->u_nio, uobj->vmobjlock, PVM, "uvn_term", 361 INFSLP); 362 } 363 364 if ((uvn->u_flags & UVM_VNODE_RELKILL) == 0) { 365 rw_exit(uobj->vmobjlock); 366 return; 367 } 368 369 /* 370 * kill object now. note that we can't be on the sync q because 371 * all references are gone. 372 */ 373 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 374 LIST_REMOVE(uvn, u_wlist); 375 } 376 KASSERT(RBT_EMPTY(uvm_objtree, &uobj->memt)); 377 oldflags = uvn->u_flags; 378 uvn->u_flags = 0; 379 380 /* wake up any sleepers */ 381 if (oldflags & UVM_VNODE_WANTED) 382 wakeup(uvn); 383 out: 384 rw_exit(uobj->vmobjlock); 385 386 /* drop our reference to the vnode. */ 387 vrele(vp); 388 389 return; 390 } 391 392 /* 393 * uvm_vnp_terminate: external hook to clear out a vnode's VM 394 * 395 * called in two cases: 396 * [1] when a persisting vnode vm object (i.e. one with a zero reference 397 * count) needs to be freed so that a vnode can be reused. this 398 * happens under "getnewvnode" in vfs_subr.c. if the vnode from 399 * the free list is still attached (i.e. not VBAD) then vgone is 400 * called. as part of the vgone trace this should get called to 401 * free the vm object. this is the common case. 402 * [2] when a filesystem is being unmounted by force (MNT_FORCE, 403 * "umount -f") the vgone() function is called on active vnodes 404 * on the mounted file systems to kill their data (the vnodes become 405 * "dead" ones [see src/sys/miscfs/deadfs/...]). that results in a 406 * call here (even if the uvn is still in use -- i.e. has a non-zero 407 * reference count). this case happens at "umount -f" and during a 408 * "reboot/halt" operation. 409 * 410 * => the caller must XLOCK and VOP_LOCK the vnode before calling us 411 * [protects us from getting a vnode that is already in the DYING 412 * state...] 413 * => in case [2] the uvn is still alive after this call, but all I/O 414 * ops will fail (due to the backing vnode now being "dead"). this 415 * will prob. kill any process using the uvn due to pgo_get failing. 416 */ 417 void 418 uvm_vnp_terminate(struct vnode *vp) 419 { 420 struct uvm_vnode *uvn = vp->v_uvm; 421 struct uvm_object *uobj = &uvn->u_obj; 422 int oldflags; 423 424 /* check if it is valid */ 425 rw_enter(uobj->vmobjlock, RW_WRITE); 426 if ((uvn->u_flags & UVM_VNODE_VALID) == 0) { 427 rw_exit(uobj->vmobjlock); 428 return; 429 } 430 431 /* 432 * must be a valid uvn that is not already dying (because XLOCK 433 * protects us from that). the uvn can't in the ALOCK state 434 * because it is valid, and uvn's that are in the ALOCK state haven't 435 * been marked valid yet. 436 */ 437 #ifdef DEBUG 438 /* 439 * debug check: are we yanking the vnode out from under our uvn? 440 */ 441 if (uvn->u_obj.uo_refs) { 442 printf("uvm_vnp_terminate(%p): terminating active vnode " 443 "(refs=%d)\n", uvn, uvn->u_obj.uo_refs); 444 } 445 #endif 446 447 /* 448 * it is possible that the uvn was detached and is in the relkill 449 * state [i.e. waiting for async i/o to finish]. 450 * we take over the vnode now and cancel the relkill. 451 * we want to know when the i/o is done so we can recycle right 452 * away. note that a uvn can only be in the RELKILL state if it 453 * has a zero reference count. 454 */ 455 if (uvn->u_flags & UVM_VNODE_RELKILL) 456 uvn->u_flags &= ~UVM_VNODE_RELKILL; /* cancel RELKILL */ 457 458 /* 459 * block the uvn by setting the dying flag, and then flush the 460 * pages. 461 * 462 * also, note that we tell I/O that we are already VOP_LOCK'd so 463 * that uvn_io doesn't attempt to VOP_LOCK again. 464 * 465 * XXXCDC: setting VNISLOCKED on an active uvn which is being terminated 466 * due to a forceful unmount might not be a good idea. maybe we 467 * need a way to pass in this info to uvn_flush through a 468 * pager-defined PGO_ constant [currently there are none]. 469 */ 470 uvn->u_flags |= UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED; 471 472 (void) uvn_flush(&uvn->u_obj, 0, 0, PGO_CLEANIT|PGO_FREE|PGO_ALLPAGES); 473 474 /* 475 * as we just did a flush we expect all the pages to be gone or in 476 * the process of going. sleep to wait for the rest to go [via iosync]. 477 */ 478 while (uvn->u_obj.uo_npages) { 479 #ifdef DEBUG 480 struct vm_page *pp; 481 RBT_FOREACH(pp, uvm_objtree, &uvn->u_obj.memt) { 482 if ((pp->pg_flags & PG_BUSY) == 0) 483 panic("uvm_vnp_terminate: detected unbusy pg"); 484 } 485 if (uvn->u_nio == 0) 486 panic("uvm_vnp_terminate: no I/O to wait for?"); 487 printf("uvm_vnp_terminate: waiting for I/O to fin.\n"); 488 /* 489 * XXXCDC: this is unlikely to happen without async i/o so we 490 * put a printf in just to keep an eye on it. 491 */ 492 #endif 493 uvn->u_flags |= UVM_VNODE_IOSYNC; 494 rwsleep_nsec(&uvn->u_nio, uobj->vmobjlock, PVM, "uvn_term", 495 INFSLP); 496 } 497 498 /* 499 * done. now we free the uvn if its reference count is zero 500 * (true if we are zapping a persisting uvn). however, if we are 501 * terminating a uvn with active mappings we let it live ... future 502 * calls down to the vnode layer will fail. 503 */ 504 oldflags = uvn->u_flags; 505 if (uvn->u_obj.uo_refs) { 506 /* 507 * uvn must live on it is dead-vnode state until all references 508 * are gone. restore flags. clear CANPERSIST state. 509 */ 510 uvn->u_flags &= ~(UVM_VNODE_DYING|UVM_VNODE_VNISLOCKED| 511 UVM_VNODE_WANTED|UVM_VNODE_CANPERSIST); 512 } else { 513 /* 514 * free the uvn now. note that the vref reference is already 515 * gone [it is dropped when we enter the persist state]. 516 */ 517 if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED) 518 panic("uvm_vnp_terminate: io sync wanted bit set"); 519 520 if (uvn->u_flags & UVM_VNODE_WRITEABLE) { 521 LIST_REMOVE(uvn, u_wlist); 522 } 523 uvn->u_flags = 0; /* uvn is history, clear all bits */ 524 } 525 526 if (oldflags & UVM_VNODE_WANTED) 527 wakeup(uvn); 528 529 rw_exit(uobj->vmobjlock); 530 } 531 532 /* 533 * NOTE: currently we have to use VOP_READ/VOP_WRITE because they go 534 * through the buffer cache and allow I/O in any size. These VOPs use 535 * synchronous i/o. [vs. VOP_STRATEGY which can be async, but doesn't 536 * go through the buffer cache or allow I/O sizes larger than a 537 * block]. we will eventually want to change this. 538 * 539 * issues to consider: 540 * uvm provides the uvm_aiodesc structure for async i/o management. 541 * there are two tailq's in the uvm. structure... one for pending async 542 * i/o and one for "done" async i/o. to do an async i/o one puts 543 * an aiodesc on the "pending" list (protected by splbio()), starts the 544 * i/o and returns VM_PAGER_PEND. when the i/o is done, we expect 545 * some sort of "i/o done" function to be called (at splbio(), interrupt 546 * time). this function should remove the aiodesc from the pending list 547 * and place it on the "done" list and wakeup the daemon. the daemon 548 * will run at normal spl() and will remove all items from the "done" 549 * list and call the "aiodone" hook for each done request (see uvm_pager.c). 550 * [in the old vm code, this was done by calling the "put" routine with 551 * null arguments which made the code harder to read and understand because 552 * you had one function ("put") doing two things.] 553 * 554 * so the current pager needs: 555 * int uvn_aiodone(struct uvm_aiodesc *) 556 * 557 * => return 0 (aio finished, free it). otherwise requeue for later collection. 558 * => called with pageq's locked by the daemon. 559 * 560 * general outline: 561 * - drop "u_nio" (this req is done!) 562 * - if (object->iosync && u_naio == 0) { wakeup &uvn->u_naio } 563 * - get "page" structures (atop?). 564 * - handle "wanted" pages 565 * dont forget to look at "object" wanted flag in all cases. 566 */ 567 568 /* 569 * uvn_flush: flush pages out of a uvm object. 570 * 571 * => if PGO_CLEANIT is set, we may block (due to I/O). thus, a caller 572 * might want to unlock higher level resources (e.g. vm_map) 573 * before calling flush. 574 * => if PGO_CLEANIT is not set, then we will not block 575 * => if PGO_ALLPAGE is set, then all pages in the object are valid targets 576 * for flushing. 577 * => NOTE: we are allowed to lock the page queues, so the caller 578 * must not be holding the lock on them [e.g. pagedaemon had 579 * better not call us with the queues locked] 580 * => we return TRUE unless we encountered some sort of I/O error 581 * 582 * comment on "cleaning" object and PG_BUSY pages: 583 * this routine is holding the lock on the object. the only time 584 * that it can run into a PG_BUSY page that it does not own is if 585 * some other process has started I/O on the page (e.g. either 586 * a pagein, or a pageout). if the PG_BUSY page is being paged 587 * in, then it can not be dirty (!PG_CLEAN) because no one has 588 * had a chance to modify it yet. if the PG_BUSY page is being 589 * paged out then it means that someone else has already started 590 * cleaning the page for us (how nice!). in this case, if we 591 * have syncio specified, then after we make our pass through the 592 * object we need to wait for the other PG_BUSY pages to clear 593 * off (i.e. we need to do an iosync). also note that once a 594 * page is PG_BUSY it must stay in its object until it is un-busyed. 595 */ 596 boolean_t 597 uvn_flush(struct uvm_object *uobj, voff_t start, voff_t stop, int flags) 598 { 599 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 600 struct vm_page *pp, *ptmp; 601 struct vm_page *pps[MAXBSIZE >> PAGE_SHIFT], **ppsp; 602 struct pglist dead; 603 int npages, result, lcv; 604 boolean_t retval, need_iosync, needs_clean; 605 voff_t curoff; 606 607 KASSERT(rw_write_held(uobj->vmobjlock)); 608 TAILQ_INIT(&dead); 609 610 /* get init vals and determine how we are going to traverse object */ 611 need_iosync = FALSE; 612 retval = TRUE; /* return value */ 613 if (flags & PGO_ALLPAGES) { 614 start = 0; 615 stop = round_page(uvn->u_size); 616 } else { 617 start = trunc_page(start); 618 stop = MIN(round_page(stop), round_page(uvn->u_size)); 619 } 620 621 /* 622 * PG_CLEANCHK: this bit is used by the pgo_mk_pcluster function as 623 * a _hint_ as to how up to date the PG_CLEAN bit is. if the hint 624 * is wrong it will only prevent us from clustering... it won't break 625 * anything. we clear all PG_CLEANCHK bits here, and pgo_mk_pcluster 626 * will set them as it syncs PG_CLEAN. This is only an issue if we 627 * are looking at non-inactive pages (because inactive page's PG_CLEAN 628 * bit is always up to date since there are no mappings). 629 * [borrowed PG_CLEANCHK idea from FreeBSD VM] 630 */ 631 if ((flags & PGO_CLEANIT) != 0) { 632 KASSERT(uobj->pgops->pgo_mk_pcluster != 0); 633 for (curoff = start ; curoff < stop; curoff += PAGE_SIZE) { 634 if ((pp = uvm_pagelookup(uobj, curoff)) != NULL) 635 atomic_clearbits_int(&pp->pg_flags, 636 PG_CLEANCHK); 637 } 638 } 639 640 ppsp = NULL; /* XXX: shut up gcc */ 641 uvm_lock_pageq(); 642 /* locked: both page queues */ 643 for (curoff = start; curoff < stop; curoff += PAGE_SIZE) { 644 if ((pp = uvm_pagelookup(uobj, curoff)) == NULL) 645 continue; 646 /* 647 * handle case where we do not need to clean page (either 648 * because we are not clean or because page is not dirty or 649 * is busy): 650 * 651 * NOTE: we are allowed to deactivate a non-wired active 652 * PG_BUSY page, but once a PG_BUSY page is on the inactive 653 * queue it must stay put until it is !PG_BUSY (so as not to 654 * confuse pagedaemon). 655 */ 656 if ((flags & PGO_CLEANIT) == 0 || (pp->pg_flags & PG_BUSY) != 0) { 657 needs_clean = FALSE; 658 if ((pp->pg_flags & PG_BUSY) != 0 && 659 (flags & (PGO_CLEANIT|PGO_SYNCIO)) == 660 (PGO_CLEANIT|PGO_SYNCIO)) 661 need_iosync = TRUE; 662 } else { 663 /* 664 * freeing: nuke all mappings so we can sync 665 * PG_CLEAN bit with no race 666 */ 667 if ((pp->pg_flags & PG_CLEAN) != 0 && 668 (flags & PGO_FREE) != 0 && 669 (pp->pg_flags & PQ_ACTIVE) != 0) 670 pmap_page_protect(pp, PROT_NONE); 671 if ((pp->pg_flags & PG_CLEAN) != 0 && 672 pmap_is_modified(pp)) 673 atomic_clearbits_int(&pp->pg_flags, PG_CLEAN); 674 atomic_setbits_int(&pp->pg_flags, PG_CLEANCHK); 675 676 needs_clean = ((pp->pg_flags & PG_CLEAN) == 0); 677 } 678 679 /* if we don't need a clean, deactivate/free pages then cont. */ 680 if (!needs_clean) { 681 if (flags & PGO_DEACTIVATE) { 682 if (pp->wire_count == 0) { 683 pmap_page_protect(pp, PROT_NONE); 684 uvm_pagedeactivate(pp); 685 } 686 } else if (flags & PGO_FREE) { 687 if (pp->pg_flags & PG_BUSY) { 688 uvm_unlock_pageq(); 689 uvm_pagewait(pp, uobj->vmobjlock, 690 "uvn_flsh"); 691 rw_enter(uobj->vmobjlock, RW_WRITE); 692 uvm_lock_pageq(); 693 curoff -= PAGE_SIZE; 694 continue; 695 } else { 696 pmap_page_protect(pp, PROT_NONE); 697 /* removed page from object */ 698 uvm_pageclean(pp); 699 TAILQ_INSERT_HEAD(&dead, pp, pageq); 700 } 701 } 702 continue; 703 } 704 705 /* 706 * pp points to a page in the object that we are 707 * working on. if it is !PG_CLEAN,!PG_BUSY and we asked 708 * for cleaning (PGO_CLEANIT). we clean it now. 709 * 710 * let uvm_pager_put attempted a clustered page out. 711 * note: locked: page queues. 712 */ 713 atomic_setbits_int(&pp->pg_flags, PG_BUSY); 714 UVM_PAGE_OWN(pp, "uvn_flush"); 715 pmap_page_protect(pp, PROT_READ); 716 /* if we're async, free the page in aiodoned */ 717 if ((flags & (PGO_FREE|PGO_SYNCIO)) == PGO_FREE) 718 atomic_setbits_int(&pp->pg_flags, PG_RELEASED); 719 ReTry: 720 ppsp = pps; 721 npages = sizeof(pps) / sizeof(struct vm_page *); 722 723 result = uvm_pager_put(uobj, pp, &ppsp, &npages, 724 flags | PGO_DOACTCLUST, start, stop); 725 726 /* 727 * if we did an async I/O it is remotely possible for the 728 * async i/o to complete and the page "pp" be freed or what 729 * not before we get a chance to relock the object. Therefore, 730 * we only touch it when it won't be freed, RELEASED took care 731 * of the rest. 732 */ 733 uvm_lock_pageq(); 734 735 /* 736 * VM_PAGER_AGAIN: given the structure of this pager, this 737 * can only happen when we are doing async I/O and can't 738 * map the pages into kernel memory (pager_map) due to lack 739 * of vm space. if this happens we drop back to sync I/O. 740 */ 741 if (result == VM_PAGER_AGAIN) { 742 /* 743 * it is unlikely, but page could have been released 744 * we ignore this now and retry the I/O. 745 * we will detect and 746 * handle the released page after the syncio I/O 747 * completes. 748 */ 749 #ifdef DIAGNOSTIC 750 if (flags & PGO_SYNCIO) 751 panic("%s: PGO_SYNCIO return 'try again' error (impossible)", __func__); 752 #endif 753 flags |= PGO_SYNCIO; 754 if (flags & PGO_FREE) 755 atomic_clearbits_int(&pp->pg_flags, 756 PG_RELEASED); 757 758 goto ReTry; 759 } 760 761 /* 762 * the cleaning operation is now done. finish up. note that 763 * on error (!OK, !PEND) uvm_pager_put drops the cluster for us. 764 * if success (OK, PEND) then uvm_pager_put returns the cluster 765 * to us in ppsp/npages. 766 */ 767 /* 768 * for pending async i/o if we are not deactivating 769 * we can move on to the next page. aiodoned deals with 770 * the freeing case for us. 771 */ 772 if (result == VM_PAGER_PEND && (flags & PGO_DEACTIVATE) == 0) 773 continue; 774 775 /* 776 * need to look at each page of the I/O operation, and do what 777 * we gotta do. 778 */ 779 for (lcv = 0 ; lcv < npages; lcv++) { 780 ptmp = ppsp[lcv]; 781 /* 782 * verify the page didn't get moved 783 */ 784 if (result == VM_PAGER_PEND && ptmp->uobject != uobj) 785 continue; 786 787 /* 788 * unbusy the page if I/O is done. note that for 789 * pending I/O it is possible that the I/O op 790 * finished 791 * (in which case the page is no longer busy). 792 */ 793 if (result != VM_PAGER_PEND) { 794 if (ptmp->pg_flags & PG_WANTED) 795 wakeup(ptmp); 796 797 atomic_clearbits_int(&ptmp->pg_flags, 798 PG_WANTED|PG_BUSY); 799 UVM_PAGE_OWN(ptmp, NULL); 800 atomic_setbits_int(&ptmp->pg_flags, 801 PG_CLEAN|PG_CLEANCHK); 802 if ((flags & PGO_FREE) == 0) 803 pmap_clear_modify(ptmp); 804 } 805 806 /* dispose of page */ 807 if (flags & PGO_DEACTIVATE) { 808 if (ptmp->wire_count == 0) { 809 pmap_page_protect(ptmp, PROT_NONE); 810 uvm_pagedeactivate(ptmp); 811 } 812 } else if (flags & PGO_FREE && 813 result != VM_PAGER_PEND) { 814 if (result != VM_PAGER_OK) { 815 static struct timeval lasttime; 816 static const struct timeval interval = 817 { 5, 0 }; 818 819 if (ratecheck(&lasttime, &interval)) { 820 printf("%s: obj=%p, " 821 "offset=0x%llx. error " 822 "during pageout.\n", 823 __func__, pp->uobject, 824 (long long)pp->offset); 825 printf("%s: WARNING: " 826 "changes to page may be " 827 "lost!\n", __func__); 828 } 829 retval = FALSE; 830 } 831 pmap_page_protect(ptmp, PROT_NONE); 832 uvm_pageclean(ptmp); 833 TAILQ_INSERT_TAIL(&dead, ptmp, pageq); 834 } 835 836 } /* end of "lcv" for loop */ 837 838 } /* end of "pp" for loop */ 839 840 /* done with pagequeues: unlock */ 841 uvm_unlock_pageq(); 842 843 /* now wait for all I/O if required. */ 844 if (need_iosync) { 845 while (uvn->u_nio != 0) { 846 uvn->u_flags |= UVM_VNODE_IOSYNC; 847 rwsleep_nsec(&uvn->u_nio, uobj->vmobjlock, PVM, 848 "uvn_flush", INFSLP); 849 } 850 if (uvn->u_flags & UVM_VNODE_IOSYNCWANTED) 851 wakeup(&uvn->u_flags); 852 uvn->u_flags &= ~(UVM_VNODE_IOSYNC|UVM_VNODE_IOSYNCWANTED); 853 } 854 855 uvm_pglistfree(&dead); 856 857 return retval; 858 } 859 860 /* 861 * uvn_cluster 862 * 863 * we are about to do I/O in an object at offset. this function is called 864 * to establish a range of offsets around "offset" in which we can cluster 865 * I/O. 866 */ 867 868 void 869 uvn_cluster(struct uvm_object *uobj, voff_t offset, voff_t *loffset, 870 voff_t *hoffset) 871 { 872 struct uvm_vnode *uvn = (struct uvm_vnode *) uobj; 873 *loffset = offset; 874 875 if (*loffset >= uvn->u_size) 876 panic("uvn_cluster: offset out of range"); 877 878 /* 879 * XXX: old pager claims we could use VOP_BMAP to get maxcontig value. 880 */ 881 *hoffset = *loffset + MAXBSIZE; 882 if (*hoffset > round_page(uvn->u_size)) /* past end? */ 883 *hoffset = round_page(uvn->u_size); 884 885 return; 886 } 887 888 /* 889 * uvn_put: flush page data to backing store. 890 * 891 * => prefer map unlocked (not required) 892 * => flags: PGO_SYNCIO -- use sync. I/O 893 * => note: caller must set PG_CLEAN and pmap_clear_modify (if needed) 894 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync. 895 * [thus we never do async i/o! see iodone comment] 896 */ 897 int 898 uvn_put(struct uvm_object *uobj, struct vm_page **pps, int npages, int flags) 899 { 900 int retval; 901 902 KASSERT(rw_write_held(uobj->vmobjlock)); 903 904 retval = uvn_io((struct uvm_vnode*)uobj, pps, npages, flags, UIO_WRITE); 905 906 return retval; 907 } 908 909 /* 910 * uvn_get: get pages (synchronously) from backing store 911 * 912 * => prefer map unlocked (not required) 913 * => flags: PGO_ALLPAGES: get all of the pages 914 * PGO_LOCKED: fault data structures are locked 915 * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx] 916 * => NOTE: caller must check for released pages!! 917 */ 918 int 919 uvn_get(struct uvm_object *uobj, voff_t offset, struct vm_page **pps, 920 int *npagesp, int centeridx, vm_prot_t access_type, int advice, int flags) 921 { 922 voff_t current_offset; 923 struct vm_page *ptmp; 924 int lcv, result, gotpages; 925 boolean_t done; 926 927 KASSERT(((flags & PGO_LOCKED) != 0 && rw_lock_held(uobj->vmobjlock)) || 928 (flags & PGO_LOCKED) == 0); 929 930 /* step 1: handled the case where fault data structures are locked. */ 931 if (flags & PGO_LOCKED) { 932 /* 933 * gotpages is the current number of pages we've gotten (which 934 * we pass back up to caller via *npagesp. 935 */ 936 gotpages = 0; 937 938 /* 939 * step 1a: get pages that are already resident. only do this 940 * if the data structures are locked (i.e. the first time 941 * through). 942 */ 943 done = TRUE; /* be optimistic */ 944 945 for (lcv = 0, current_offset = offset ; lcv < *npagesp ; 946 lcv++, current_offset += PAGE_SIZE) { 947 948 /* do we care about this page? if not, skip it */ 949 if (pps[lcv] == PGO_DONTCARE) 950 continue; 951 952 /* lookup page */ 953 ptmp = uvm_pagelookup(uobj, current_offset); 954 955 /* to be useful must get a non-busy, non-released pg */ 956 if (ptmp == NULL || 957 (ptmp->pg_flags & PG_BUSY) != 0) { 958 if (lcv == centeridx || (flags & PGO_ALLPAGES) 959 != 0) 960 done = FALSE; /* need to do a wait or I/O! */ 961 continue; 962 } 963 964 /* 965 * useful page: busy it and plug it in our 966 * result array 967 */ 968 atomic_setbits_int(&ptmp->pg_flags, PG_BUSY); 969 UVM_PAGE_OWN(ptmp, "uvn_get1"); 970 pps[lcv] = ptmp; 971 gotpages++; 972 973 } 974 975 /* 976 * XXX: given the "advice", should we consider async read-ahead? 977 * XXX: fault current does deactivate of pages behind us. is 978 * this good (other callers might now). 979 */ 980 /* 981 * XXX: read-ahead currently handled by buffer cache (bread) 982 * level. 983 * XXX: no async i/o available. 984 * XXX: so we don't do anything now. 985 */ 986 987 /* 988 * step 1c: now we've either done everything needed or we to 989 * unlock and do some waiting or I/O. 990 */ 991 992 *npagesp = gotpages; /* let caller know */ 993 if (done) 994 return VM_PAGER_OK; /* bingo! */ 995 else 996 return VM_PAGER_UNLOCK; 997 } 998 999 /* 1000 * step 2: get non-resident or busy pages. 1001 * data structures are unlocked. 1002 * 1003 * XXX: because we can't do async I/O at this level we get things 1004 * page at a time (otherwise we'd chunk). the VOP_READ() will do 1005 * async-read-ahead for us at a lower level. 1006 */ 1007 for (lcv = 0, current_offset = offset; 1008 lcv < *npagesp ; lcv++, current_offset += PAGE_SIZE) { 1009 1010 /* skip over pages we've already gotten or don't want */ 1011 /* skip over pages we don't _have_ to get */ 1012 if (pps[lcv] != NULL || (lcv != centeridx && 1013 (flags & PGO_ALLPAGES) == 0)) 1014 continue; 1015 1016 /* 1017 * we have yet to locate the current page (pps[lcv]). we first 1018 * look for a page that is already at the current offset. if 1019 * we fine a page, we check to see if it is busy or released. 1020 * if that is the case, then we sleep on the page until it is 1021 * no longer busy or released and repeat the lookup. if the 1022 * page we found is neither busy nor released, then we busy it 1023 * (so we own it) and plug it into pps[lcv]. this breaks the 1024 * following while loop and indicates we are ready to move on 1025 * to the next page in the "lcv" loop above. 1026 * 1027 * if we exit the while loop with pps[lcv] still set to NULL, 1028 * then it means that we allocated a new busy/fake/clean page 1029 * ptmp in the object and we need to do I/O to fill in the data. 1030 */ 1031 while (pps[lcv] == NULL) { /* top of "pps" while loop */ 1032 /* look for a current page */ 1033 ptmp = uvm_pagelookup(uobj, current_offset); 1034 1035 /* nope? allocate one now (if we can) */ 1036 if (ptmp == NULL) { 1037 ptmp = uvm_pagealloc(uobj, current_offset, 1038 NULL, 0); 1039 1040 /* out of RAM? */ 1041 if (ptmp == NULL) { 1042 uvm_wait("uvn_getpage"); 1043 1044 /* goto top of pps while loop */ 1045 continue; 1046 } 1047 1048 /* 1049 * got new page ready for I/O. break pps 1050 * while loop. pps[lcv] is still NULL. 1051 */ 1052 break; 1053 } 1054 1055 /* page is there, see if we need to wait on it */ 1056 if ((ptmp->pg_flags & PG_BUSY) != 0) { 1057 uvm_pagewait(ptmp, uobj->vmobjlock, "uvn_get"); 1058 rw_enter(uobj->vmobjlock, RW_WRITE); 1059 continue; /* goto top of pps while loop */ 1060 } 1061 1062 /* 1063 * if we get here then the page has become resident 1064 * and unbusy between steps 1 and 2. we busy it 1065 * now (so we own it) and set pps[lcv] (so that we 1066 * exit the while loop). 1067 */ 1068 atomic_setbits_int(&ptmp->pg_flags, PG_BUSY); 1069 UVM_PAGE_OWN(ptmp, "uvn_get2"); 1070 pps[lcv] = ptmp; 1071 } 1072 1073 /* 1074 * if we own the a valid page at the correct offset, pps[lcv] 1075 * will point to it. nothing more to do except go to the 1076 * next page. 1077 */ 1078 if (pps[lcv]) 1079 continue; /* next lcv */ 1080 1081 /* 1082 * we have a "fake/busy/clean" page that we just allocated. do 1083 * I/O to fill it with valid data. 1084 */ 1085 result = uvn_io((struct uvm_vnode *) uobj, &ptmp, 1, 1086 PGO_SYNCIO|PGO_NOWAIT, UIO_READ); 1087 1088 /* 1089 * I/O done. because we used syncio the result can not be 1090 * PEND or AGAIN. 1091 */ 1092 if (result != VM_PAGER_OK) { 1093 if (ptmp->pg_flags & PG_WANTED) 1094 wakeup(ptmp); 1095 1096 atomic_clearbits_int(&ptmp->pg_flags, 1097 PG_WANTED|PG_BUSY); 1098 UVM_PAGE_OWN(ptmp, NULL); 1099 uvm_lock_pageq(); 1100 uvm_pagefree(ptmp); 1101 uvm_unlock_pageq(); 1102 rw_exit(uobj->vmobjlock); 1103 return result; 1104 } 1105 1106 /* 1107 * we got the page! clear the fake flag (indicates valid 1108 * data now in page) and plug into our result array. note 1109 * that page is still busy. 1110 * 1111 * it is the callers job to: 1112 * => check if the page is released 1113 * => unbusy the page 1114 * => activate the page 1115 */ 1116 1117 /* data is valid ... */ 1118 atomic_clearbits_int(&ptmp->pg_flags, PG_FAKE); 1119 pmap_clear_modify(ptmp); /* ... and clean */ 1120 pps[lcv] = ptmp; 1121 1122 } 1123 1124 1125 rw_exit(uobj->vmobjlock); 1126 return (VM_PAGER_OK); 1127 } 1128 1129 /* 1130 * uvn_io: do I/O to a vnode 1131 * 1132 * => prefer map unlocked (not required) 1133 * => flags: PGO_SYNCIO -- use sync. I/O 1134 * => XXX: currently we use VOP_READ/VOP_WRITE which are only sync. 1135 * [thus we never do async i/o! see iodone comment] 1136 */ 1137 1138 int 1139 uvn_io(struct uvm_vnode *uvn, vm_page_t *pps, int npages, int flags, int rw) 1140 { 1141 struct uvm_object *uobj = &uvn->u_obj; 1142 struct vnode *vn; 1143 struct uio uio; 1144 struct iovec iov; 1145 vaddr_t kva; 1146 off_t file_offset; 1147 int waitf, result, mapinflags; 1148 size_t got, wanted; 1149 int netunlocked = 0; 1150 int lkflags = (flags & PGO_NOWAIT) ? LK_NOWAIT : 0; 1151 1152 KASSERT(rw_write_held(uobj->vmobjlock)); 1153 1154 /* init values */ 1155 waitf = (flags & PGO_SYNCIO) ? M_WAITOK : M_NOWAIT; 1156 vn = uvn->u_vnode; 1157 file_offset = pps[0]->offset; 1158 1159 /* check for sync'ing I/O. */ 1160 while (uvn->u_flags & UVM_VNODE_IOSYNC) { 1161 if (waitf == M_NOWAIT) { 1162 return VM_PAGER_AGAIN; 1163 } 1164 uvn->u_flags |= UVM_VNODE_IOSYNCWANTED; 1165 rwsleep_nsec(&uvn->u_flags, uobj->vmobjlock, PVM, "uvn_iosync", 1166 INFSLP); 1167 } 1168 1169 /* check size */ 1170 if (file_offset >= uvn->u_size) { 1171 return VM_PAGER_BAD; 1172 } 1173 1174 /* first try and map the pages in (without waiting) */ 1175 mapinflags = (rw == UIO_READ) ? 1176 UVMPAGER_MAPIN_READ : UVMPAGER_MAPIN_WRITE; 1177 1178 kva = uvm_pagermapin(pps, npages, mapinflags); 1179 if (kva == 0 && waitf == M_NOWAIT) { 1180 return VM_PAGER_AGAIN; 1181 } 1182 1183 /* 1184 * ok, now bump u_nio up. at this point we are done with uvn 1185 * and can unlock it. if we still don't have a kva, try again 1186 * (this time with sleep ok). 1187 */ 1188 uvn->u_nio++; /* we have an I/O in progress! */ 1189 rw_exit(uobj->vmobjlock); 1190 if (kva == 0) 1191 kva = uvm_pagermapin(pps, npages, 1192 mapinflags | UVMPAGER_MAPIN_WAITOK); 1193 1194 /* 1195 * ok, mapped in. our pages are PG_BUSY so they are not going to 1196 * get touched (so we can look at "offset" without having to lock 1197 * the object). set up for I/O. 1198 */ 1199 /* fill out uio/iov */ 1200 iov.iov_base = (caddr_t) kva; 1201 wanted = (size_t)npages << PAGE_SHIFT; 1202 if (file_offset + wanted > uvn->u_size) 1203 wanted = uvn->u_size - file_offset; /* XXX: needed? */ 1204 iov.iov_len = wanted; 1205 uio.uio_iov = &iov; 1206 uio.uio_iovcnt = 1; 1207 uio.uio_offset = file_offset; 1208 uio.uio_segflg = UIO_SYSSPACE; 1209 uio.uio_rw = rw; 1210 uio.uio_resid = wanted; 1211 uio.uio_procp = curproc; 1212 1213 /* 1214 * This process may already have the NET_LOCK(), if we 1215 * faulted in copyin() or copyout() in the network stack. 1216 */ 1217 if (rw_status(&netlock) == RW_WRITE) { 1218 NET_UNLOCK(); 1219 netunlocked = 1; 1220 } 1221 1222 /* do the I/O! (XXX: curproc?) */ 1223 /* 1224 * This process may already have this vnode locked, if we faulted in 1225 * copyin() or copyout() on a region backed by this vnode 1226 * while doing I/O to the vnode. If this is the case, don't 1227 * panic.. instead, return the error to the user. 1228 * 1229 * XXX this is a stopgap to prevent a panic. 1230 * Ideally, this kind of operation *should* work. 1231 */ 1232 result = 0; 1233 KERNEL_LOCK(); 1234 if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0) 1235 result = vn_lock(vn, LK_EXCLUSIVE | LK_RECURSEFAIL | lkflags); 1236 if (result == 0) { 1237 /* NOTE: vnode now locked! */ 1238 if (rw == UIO_READ) 1239 result = VOP_READ(vn, &uio, 0, curproc->p_ucred); 1240 else 1241 result = VOP_WRITE(vn, &uio, 1242 (flags & PGO_PDFREECLUST) ? IO_NOCACHE : 0, 1243 curproc->p_ucred); 1244 1245 if ((uvn->u_flags & UVM_VNODE_VNISLOCKED) == 0) 1246 VOP_UNLOCK(vn); 1247 1248 } 1249 KERNEL_UNLOCK(); 1250 1251 if (netunlocked) 1252 NET_LOCK(); 1253 1254 1255 /* NOTE: vnode now unlocked (unless vnislocked) */ 1256 /* 1257 * result == unix style errno (0 == OK!) 1258 * 1259 * zero out rest of buffer (if needed) 1260 */ 1261 if (result == 0) { 1262 got = wanted - uio.uio_resid; 1263 1264 if (wanted && got == 0) { 1265 result = EIO; /* XXX: error? */ 1266 } else if (got < PAGE_SIZE * npages && rw == UIO_READ) { 1267 memset((void *) (kva + got), 0, 1268 ((size_t)npages << PAGE_SHIFT) - got); 1269 } 1270 } 1271 1272 /* now remove pager mapping */ 1273 uvm_pagermapout(kva, npages); 1274 1275 /* now clean up the object (i.e. drop I/O count) */ 1276 rw_enter(uobj->vmobjlock, RW_WRITE); 1277 uvn->u_nio--; /* I/O DONE! */ 1278 if ((uvn->u_flags & UVM_VNODE_IOSYNC) != 0 && uvn->u_nio == 0) { 1279 wakeup(&uvn->u_nio); 1280 } 1281 1282 if (result == 0) { 1283 return VM_PAGER_OK; 1284 } else if (result == EBUSY) { 1285 KASSERT(flags & PGO_NOWAIT); 1286 return VM_PAGER_AGAIN; 1287 } else { 1288 if (rebooting) { 1289 KERNEL_LOCK(); 1290 while (rebooting) 1291 tsleep_nsec(&rebooting, PVM, "uvndead", INFSLP); 1292 KERNEL_UNLOCK(); 1293 } 1294 return VM_PAGER_ERROR; 1295 } 1296 } 1297 1298 /* 1299 * uvm_vnp_uncache: disable "persisting" in a vnode... when last reference 1300 * is gone we will kill the object (flushing dirty pages back to the vnode 1301 * if needed). 1302 * 1303 * => returns TRUE if there was no uvm_object attached or if there was 1304 * one and we killed it [i.e. if there is no active uvn] 1305 * => called with the vnode VOP_LOCK'd [we will unlock it for I/O, if 1306 * needed] 1307 * 1308 * => XXX: given that we now kill uvn's when a vnode is recycled (without 1309 * having to hold a reference on the vnode) and given a working 1310 * uvm_vnp_sync(), how does that effect the need for this function? 1311 * [XXXCDC: seems like it can die?] 1312 * 1313 * => XXX: this function should DIE once we merge the VM and buffer 1314 * cache. 1315 * 1316 * research shows that this is called in the following places: 1317 * ext2fs_truncate, ffs_truncate, detrunc[msdosfs]: called when vnode 1318 * changes sizes 1319 * ext2fs_write, WRITE [ufs_readwrite], msdosfs_write: called when we 1320 * are written to 1321 * ex2fs_chmod, ufs_chmod: called if VTEXT vnode and the sticky bit 1322 * is off 1323 * ffs_realloccg: when we can't extend the current block and have 1324 * to allocate a new one we call this [XXX: why?] 1325 * nfsrv_rename, rename_files: called when the target filename is there 1326 * and we want to remove it 1327 * nfsrv_remove, sys_unlink: called on file we are removing 1328 * nfsrv_access: if VTEXT and we want WRITE access and we don't uncache 1329 * then return "text busy" 1330 * nfs_open: seems to uncache any file opened with nfs 1331 * vn_writechk: if VTEXT vnode and can't uncache return "text busy" 1332 * fusefs_open: uncaches any file that is opened 1333 * fusefs_write: uncaches on every write 1334 */ 1335 1336 int 1337 uvm_vnp_uncache(struct vnode *vp) 1338 { 1339 struct uvm_vnode *uvn = vp->v_uvm; 1340 struct uvm_object *uobj = &uvn->u_obj; 1341 1342 /* lock uvn part of the vnode and check if we need to do anything */ 1343 1344 rw_enter(uobj->vmobjlock, RW_WRITE); 1345 if ((uvn->u_flags & UVM_VNODE_VALID) == 0 || 1346 (uvn->u_flags & UVM_VNODE_BLOCKED) != 0) { 1347 rw_exit(uobj->vmobjlock); 1348 return TRUE; 1349 } 1350 1351 /* 1352 * we have a valid, non-blocked uvn. clear persist flag. 1353 * if uvn is currently active we can return now. 1354 */ 1355 uvn->u_flags &= ~UVM_VNODE_CANPERSIST; 1356 if (uvn->u_obj.uo_refs) { 1357 rw_exit(uobj->vmobjlock); 1358 return FALSE; 1359 } 1360 1361 /* 1362 * uvn is currently persisting! we have to gain a reference to 1363 * it so that we can call uvn_detach to kill the uvn. 1364 */ 1365 vref(vp); /* seems ok, even with VOP_LOCK */ 1366 uvn->u_obj.uo_refs++; /* value is now 1 */ 1367 rw_exit(uobj->vmobjlock); 1368 1369 #ifdef VFSLCKDEBUG 1370 /* 1371 * carry over sanity check from old vnode pager: the vnode should 1372 * be VOP_LOCK'd, and we confirm it here. 1373 */ 1374 if ((vp->v_flag & VLOCKSWORK) && !VOP_ISLOCKED(vp)) 1375 panic("uvm_vnp_uncache: vnode not locked!"); 1376 #endif 1377 1378 /* 1379 * now drop our reference to the vnode. if we have the sole 1380 * reference to the vnode then this will cause it to die [as we 1381 * just cleared the persist flag]. we have to unlock the vnode 1382 * while we are doing this as it may trigger I/O. 1383 * 1384 * XXX: it might be possible for uvn to get reclaimed while we are 1385 * unlocked causing us to return TRUE when we should not. we ignore 1386 * this as a false-positive return value doesn't hurt us. 1387 */ 1388 VOP_UNLOCK(vp); 1389 uvn_detach(&uvn->u_obj); 1390 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 1391 1392 return TRUE; 1393 } 1394 1395 /* 1396 * uvm_vnp_setsize: grow or shrink a vnode uvn 1397 * 1398 * grow => just update size value 1399 * shrink => toss un-needed pages 1400 * 1401 * => we assume that the caller has a reference of some sort to the 1402 * vnode in question so that it will not be yanked out from under 1403 * us. 1404 * 1405 * called from: 1406 * => truncate fns (ext2fs_truncate, ffs_truncate, detrunc[msdos], 1407 * fusefs_setattr) 1408 * => "write" fns (ext2fs_write, WRITE [ufs/ufs], msdosfs_write, nfs_write 1409 * fusefs_write) 1410 * => ffs_balloc [XXX: why? doesn't WRITE handle?] 1411 * => NFS: nfs_loadattrcache, nfs_getattrcache, nfs_setattr 1412 * => union fs: union_newsize 1413 */ 1414 1415 void 1416 uvm_vnp_setsize(struct vnode *vp, off_t newsize) 1417 { 1418 struct uvm_vnode *uvn = vp->v_uvm; 1419 struct uvm_object *uobj = &uvn->u_obj; 1420 1421 KERNEL_ASSERT_LOCKED(); 1422 1423 rw_enter(uobj->vmobjlock, RW_WRITE); 1424 1425 /* lock uvn and check for valid object, and if valid: do it! */ 1426 if (uvn->u_flags & UVM_VNODE_VALID) { 1427 1428 /* 1429 * now check if the size has changed: if we shrink we had better 1430 * toss some pages... 1431 */ 1432 1433 if (uvn->u_size > newsize) { 1434 (void)uvn_flush(&uvn->u_obj, newsize, 1435 uvn->u_size, PGO_FREE); 1436 } 1437 uvn->u_size = newsize; 1438 } 1439 rw_exit(uobj->vmobjlock); 1440 } 1441 1442 /* 1443 * uvm_vnp_sync: flush all dirty VM pages back to their backing vnodes. 1444 * 1445 * => called from sys_sync with no VM structures locked 1446 * => only one process can do a sync at a time (because the uvn 1447 * structure only has one queue for sync'ing). we ensure this 1448 * by holding the uvn_sync_lock while the sync is in progress. 1449 * other processes attempting a sync will sleep on this lock 1450 * until we are done. 1451 */ 1452 void 1453 uvm_vnp_sync(struct mount *mp) 1454 { 1455 struct uvm_vnode *uvn; 1456 struct vnode *vp; 1457 1458 /* 1459 * step 1: ensure we are only ones using the uvn_sync_q by locking 1460 * our lock... 1461 */ 1462 rw_enter_write(&uvn_sync_lock); 1463 1464 /* 1465 * step 2: build up a simpleq of uvns of interest based on the 1466 * write list. we gain a reference to uvns of interest. 1467 */ 1468 SIMPLEQ_INIT(&uvn_sync_q); 1469 LIST_FOREACH(uvn, &uvn_wlist, u_wlist) { 1470 vp = uvn->u_vnode; 1471 if (mp && vp->v_mount != mp) 1472 continue; 1473 1474 /* 1475 * If the vnode is "blocked" it means it must be dying, which 1476 * in turn means its in the process of being flushed out so 1477 * we can safely skip it. 1478 * 1479 * note that uvn must already be valid because we found it on 1480 * the wlist (this also means it can't be ALOCK'd). 1481 */ 1482 if ((uvn->u_flags & UVM_VNODE_BLOCKED) != 0) 1483 continue; 1484 1485 /* 1486 * gain reference. watch out for persisting uvns (need to 1487 * regain vnode REF). 1488 */ 1489 if (uvn->u_obj.uo_refs == 0) 1490 vref(vp); 1491 uvn->u_obj.uo_refs++; 1492 1493 SIMPLEQ_INSERT_HEAD(&uvn_sync_q, uvn, u_syncq); 1494 } 1495 1496 /* step 3: we now have a list of uvn's that may need cleaning. */ 1497 SIMPLEQ_FOREACH(uvn, &uvn_sync_q, u_syncq) { 1498 rw_enter(uvn->u_obj.vmobjlock, RW_WRITE); 1499 #ifdef DEBUG 1500 if (uvn->u_flags & UVM_VNODE_DYING) { 1501 printf("uvm_vnp_sync: dying vnode on sync list\n"); 1502 } 1503 #endif 1504 uvn_flush(&uvn->u_obj, 0, 0, PGO_CLEANIT|PGO_ALLPAGES|PGO_DOACTCLUST); 1505 1506 /* 1507 * if we have the only reference and we just cleaned the uvn, 1508 * then we can pull it out of the UVM_VNODE_WRITEABLE state 1509 * thus allowing us to avoid thinking about flushing it again 1510 * on later sync ops. 1511 */ 1512 if (uvn->u_obj.uo_refs == 1 && 1513 (uvn->u_flags & UVM_VNODE_WRITEABLE)) { 1514 LIST_REMOVE(uvn, u_wlist); 1515 uvn->u_flags &= ~UVM_VNODE_WRITEABLE; 1516 } 1517 rw_exit(uvn->u_obj.vmobjlock); 1518 1519 /* now drop our reference to the uvn */ 1520 uvn_detach(&uvn->u_obj); 1521 } 1522 1523 rw_exit_write(&uvn_sync_lock); 1524 } 1525