1 /* $NetBSD: uvm_pager.c,v 1.42 2001/03/10 22:46:49 chs Exp $ */ 2 3 /* 4 * 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by Charles D. Cranor and 19 * Washington University. 20 * 4. The name of the author may not be used to endorse or promote products 21 * derived from this software without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 24 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 25 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 26 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 27 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 28 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 32 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 * 34 * from: Id: uvm_pager.c,v 1.1.2.23 1998/02/02 20:38:06 chuck Exp 35 */ 36 37 #include "opt_uvmhist.h" 38 39 /* 40 * uvm_pager.c: generic functions used to assist the pagers. 41 */ 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/proc.h> 46 #include <sys/malloc.h> 47 #include <sys/pool.h> 48 #include <sys/vnode.h> 49 50 #define UVM_PAGER 51 #include <uvm/uvm.h> 52 53 struct pool *uvm_aiobuf_pool; 54 55 /* 56 * list of uvm pagers in the system 57 */ 58 59 extern struct uvm_pagerops uvm_deviceops; 60 extern struct uvm_pagerops uvm_vnodeops; 61 extern struct uvm_pagerops ubc_pager; 62 63 struct uvm_pagerops *uvmpagerops[] = { 64 &aobj_pager, 65 &uvm_deviceops, 66 &uvm_vnodeops, 67 &ubc_pager, 68 }; 69 70 /* 71 * the pager map: provides KVA for I/O 72 */ 73 74 vm_map_t pager_map; /* XXX */ 75 simple_lock_data_t pager_map_wanted_lock; 76 boolean_t pager_map_wanted; /* locked by pager map */ 77 static vaddr_t emergva; 78 static boolean_t emerginuse; 79 80 /* 81 * uvm_pager_init: init pagers (at boot time) 82 */ 83 84 void 85 uvm_pager_init() 86 { 87 int lcv; 88 89 /* 90 * init pager map 91 */ 92 93 pager_map = uvm_km_suballoc(kernel_map, &uvm.pager_sva, &uvm.pager_eva, 94 PAGER_MAP_SIZE, 0, FALSE, NULL); 95 simple_lock_init(&pager_map_wanted_lock); 96 pager_map_wanted = FALSE; 97 emergva = uvm_km_valloc(kernel_map, MAXBSIZE); 98 emerginuse = FALSE; 99 100 /* 101 * init ASYNC I/O queue 102 */ 103 104 TAILQ_INIT(&uvm.aio_done); 105 106 /* 107 * call pager init functions 108 */ 109 for (lcv = 0 ; lcv < sizeof(uvmpagerops)/sizeof(struct uvm_pagerops *); 110 lcv++) { 111 if (uvmpagerops[lcv]->pgo_init) 112 uvmpagerops[lcv]->pgo_init(); 113 } 114 } 115 116 /* 117 * uvm_pagermapin: map pages into KVA (pager_map) for I/O that needs mappings 118 * 119 * we basically just map in a blank map entry to reserve the space in the 120 * map and then use pmap_enter() to put the mappings in by hand. 121 */ 122 123 vaddr_t 124 uvm_pagermapin(pps, npages, flags) 125 struct vm_page **pps; 126 int npages; 127 int flags; 128 { 129 vsize_t size; 130 vaddr_t kva; 131 vaddr_t cva; 132 struct vm_page *pp; 133 vm_prot_t prot; 134 UVMHIST_FUNC("uvm_pagermapin"); UVMHIST_CALLED(maphist); 135 136 UVMHIST_LOG(maphist,"(pps=0x%x, npages=%d)", pps, npages,0,0); 137 138 /* 139 * compute protection. outgoing I/O only needs read 140 * access to the page, whereas incoming needs read/write. 141 */ 142 143 prot = VM_PROT_READ; 144 if (flags & UVMPAGER_MAPIN_READ) 145 prot |= VM_PROT_WRITE; 146 147 ReStart: 148 size = npages << PAGE_SHIFT; 149 kva = 0; /* let system choose VA */ 150 151 if (uvm_map(pager_map, &kva, size, NULL, 152 UVM_UNKNOWN_OFFSET, 0, UVM_FLAG_NOMERGE) != KERN_SUCCESS) { 153 if (curproc == uvm.pagedaemon_proc) { 154 simple_lock(&pager_map_wanted_lock); 155 if (emerginuse) { 156 UVM_UNLOCK_AND_WAIT(&emergva, 157 &pager_map_wanted_lock, FALSE, 158 "emergva", 0); 159 goto ReStart; 160 } 161 emerginuse = TRUE; 162 simple_unlock(&pager_map_wanted_lock); 163 kva = emergva; 164 KASSERT(npages <= MAXBSIZE >> PAGE_SHIFT); 165 goto enter; 166 } 167 if ((flags & UVMPAGER_MAPIN_WAITOK) == 0) { 168 UVMHIST_LOG(maphist,"<- NOWAIT failed", 0,0,0,0); 169 return(0); 170 } 171 simple_lock(&pager_map_wanted_lock); 172 pager_map_wanted = TRUE; 173 UVMHIST_LOG(maphist, " SLEEPING on pager_map",0,0,0,0); 174 UVM_UNLOCK_AND_WAIT(pager_map, &pager_map_wanted_lock, FALSE, 175 "pager_map", 0); 176 goto ReStart; 177 } 178 179 enter: 180 /* got it */ 181 for (cva = kva ; size != 0 ; size -= PAGE_SIZE, cva += PAGE_SIZE) { 182 pp = *pps++; 183 KASSERT(pp); 184 KASSERT(pp->flags & PG_BUSY); 185 pmap_enter(vm_map_pmap(pager_map), cva, VM_PAGE_TO_PHYS(pp), 186 prot, PMAP_WIRED | ((pp->flags & PG_FAKE) ? prot : 187 VM_PROT_READ)); 188 } 189 190 UVMHIST_LOG(maphist, "<- done (KVA=0x%x)", kva,0,0,0); 191 return(kva); 192 } 193 194 /* 195 * uvm_pagermapout: remove pager_map mapping 196 * 197 * we remove our mappings by hand and then remove the mapping (waking 198 * up anyone wanting space). 199 */ 200 201 void 202 uvm_pagermapout(kva, npages) 203 vaddr_t kva; 204 int npages; 205 { 206 vsize_t size = npages << PAGE_SHIFT; 207 vm_map_entry_t entries; 208 UVMHIST_FUNC("uvm_pagermapout"); UVMHIST_CALLED(maphist); 209 210 UVMHIST_LOG(maphist, " (kva=0x%x, npages=%d)", kva, npages,0,0); 211 212 /* 213 * duplicate uvm_unmap, but add in pager_map_wanted handling. 214 */ 215 216 if (kva == emergva) { 217 simple_lock(&pager_map_wanted_lock); 218 emerginuse = FALSE; 219 wakeup(&emergva); 220 simple_unlock(&pager_map_wanted_lock); 221 entries = NULL; 222 goto remove; 223 } 224 225 vm_map_lock(pager_map); 226 (void) uvm_unmap_remove(pager_map, kva, kva + size, &entries); 227 simple_lock(&pager_map_wanted_lock); 228 if (pager_map_wanted) { 229 pager_map_wanted = FALSE; 230 wakeup(pager_map); 231 } 232 simple_unlock(&pager_map_wanted_lock); 233 vm_map_unlock(pager_map); 234 remove: 235 pmap_remove(pmap_kernel(), kva, kva + (npages << PAGE_SHIFT)); 236 if (entries) 237 uvm_unmap_detach(entries, 0); 238 239 UVMHIST_LOG(maphist,"<- done",0,0,0,0); 240 } 241 242 /* 243 * uvm_mk_pcluster 244 * 245 * generic "make 'pager put' cluster" function. a pager can either 246 * [1] set pgo_mk_pcluster to NULL (never cluster), [2] set it to this 247 * generic function, or [3] set it to a pager specific function. 248 * 249 * => caller must lock object _and_ pagequeues (since we need to look 250 * at active vs. inactive bits, etc.) 251 * => caller must make center page busy and write-protect it 252 * => we mark all cluster pages busy for the caller 253 * => the caller must unbusy all pages (and check wanted/released 254 * status if it drops the object lock) 255 * => flags: 256 * PGO_ALLPAGES: all pages in object are valid targets 257 * !PGO_ALLPAGES: use "lo" and "hi" to limit range of cluster 258 * PGO_DOACTCLUST: include active pages in cluster. 259 * NOTE: the caller should clear PG_CLEANCHK bits if PGO_DOACTCLUST. 260 * PG_CLEANCHK is only a hint, but clearing will help reduce 261 * the number of calls we make to the pmap layer. 262 */ 263 264 struct vm_page ** 265 uvm_mk_pcluster(uobj, pps, npages, center, flags, mlo, mhi) 266 struct uvm_object *uobj; /* IN */ 267 struct vm_page **pps, *center; /* IN/OUT, IN */ 268 int *npages, flags; /* IN/OUT, IN */ 269 voff_t mlo, mhi; /* IN (if !PGO_ALLPAGES) */ 270 { 271 struct vm_page **ppsp, *pclust; 272 voff_t lo, hi, curoff; 273 int center_idx, forward, incr; 274 UVMHIST_FUNC("uvm_mk_pcluster"); UVMHIST_CALLED(maphist); 275 276 /* 277 * center page should already be busy and write protected. XXX: 278 * suppose page is wired? if we lock, then a process could 279 * fault/block on it. if we don't lock, a process could write the 280 * pages in the middle of an I/O. (consider an msync()). let's 281 * lock it for now (better to delay than corrupt data?). 282 */ 283 284 /* 285 * get cluster boundaries, check sanity, and apply our limits as well. 286 */ 287 288 uobj->pgops->pgo_cluster(uobj, center->offset, &lo, &hi); 289 if ((flags & PGO_ALLPAGES) == 0) { 290 if (lo < mlo) 291 lo = mlo; 292 if (hi > mhi) 293 hi = mhi; 294 } 295 if ((hi - lo) >> PAGE_SHIFT > *npages) { /* pps too small, bail out! */ 296 pps[0] = center; 297 *npages = 1; 298 return(pps); 299 } 300 301 /* 302 * now determine the center and attempt to cluster around the 303 * edges 304 */ 305 306 center_idx = (center->offset - lo) >> PAGE_SHIFT; 307 pps[center_idx] = center; /* plug in the center page */ 308 ppsp = &pps[center_idx]; 309 *npages = 1; 310 311 /* 312 * attempt to cluster around the left [backward], and then 313 * the right side [forward]. 314 */ 315 316 for (forward = 0 ; forward <= 1 ; forward++) { 317 incr = forward ? PAGE_SIZE : -PAGE_SIZE; 318 curoff = center->offset + incr; 319 for ( ;(forward == 0 && curoff >= lo) || 320 (forward && curoff < hi); 321 curoff += incr) { 322 323 pclust = uvm_pagelookup(uobj, curoff); /* lookup page */ 324 if (pclust == NULL) { 325 break; /* no page */ 326 } 327 328 if ((flags & PGO_DOACTCLUST) == 0) { 329 /* dont want mapped pages at all */ 330 break; 331 } 332 333 /* 334 * get an up-to-date view of the "clean" bit. 335 * note this isn't 100% accurate, but it doesn't 336 * have to be. if it's not quite right, the 337 * worst that happens is we don't cluster as 338 * aggressively. we'll sync-it-for-sure before 339 * we free the page, and clean it if necessary. 340 */ 341 if ((pclust->flags & PG_CLEANCHK) == 0) { 342 if ((pclust->flags & (PG_CLEAN|PG_BUSY)) 343 == PG_CLEAN && 344 pmap_is_modified(pclust)) 345 pclust->flags &= ~PG_CLEAN; 346 347 /* now checked */ 348 pclust->flags |= PG_CLEANCHK; 349 } 350 351 /* is page available for cleaning and does it need it */ 352 if ((pclust->flags & (PG_CLEAN|PG_BUSY)) != 0) { 353 break; /* page is already clean or is busy */ 354 } 355 356 /* yes! enroll the page in our array */ 357 pclust->flags |= PG_BUSY; /* busy! */ 358 UVM_PAGE_OWN(pclust, "uvm_mk_pcluster"); 359 360 /* XXX: protect wired page? see above comment. */ 361 pmap_page_protect(pclust, VM_PROT_READ); 362 if (!forward) { 363 ppsp--; /* back up one page */ 364 *ppsp = pclust; 365 } else { 366 /* move forward one page */ 367 ppsp[*npages] = pclust; 368 } 369 (*npages)++; 370 } 371 } 372 373 /* 374 * done! return the cluster array to the caller!!! 375 */ 376 377 UVMHIST_LOG(maphist, "<- done",0,0,0,0); 378 return(ppsp); 379 } 380 381 /* 382 * uvm_pager_put: high level pageout routine 383 * 384 * we want to pageout page "pg" to backing store, clustering if 385 * possible. 386 * 387 * => page queues must be locked by caller 388 * => if page is not swap-backed, then "uobj" points to the object 389 * backing it. this object should be locked by the caller. 390 * => if page is swap-backed, then "uobj" should be NULL. 391 * => "pg" should be PG_BUSY (by caller), and !PG_CLEAN 392 * for swap-backed memory, "pg" can be NULL if there is no page 393 * of interest [sometimes the case for the pagedaemon] 394 * => "ppsp_ptr" should point to an array of npages vm_page pointers 395 * for possible cluster building 396 * => flags (first two for non-swap-backed pages) 397 * PGO_ALLPAGES: all pages in uobj are valid targets 398 * PGO_DOACTCLUST: include "PQ_ACTIVE" pages as valid targets 399 * PGO_SYNCIO: wait for i/o to complete 400 * PGO_PDFREECLUST: pagedaemon: drop cluster on successful I/O 401 * => start/stop: if (uobj && !PGO_ALLPAGES) limit targets to this range 402 * if (!uobj) start is the (daddr_t) of the starting swapblk 403 * => return state: 404 * 1. we return the error code of the pageout 405 * 2. we return with the page queues unlocked 406 * 3. if (uobj != NULL) [!swap_backed] we return with 407 * uobj locked _only_ if PGO_PDFREECLUST is set 408 * AND result == 0 AND async. in all other cases 409 * we return with uobj unlocked. [this is a hack 410 * that allows the pagedaemon to save one lock/unlock 411 * pair in the !swap_backed case since we have to 412 * lock the uobj to drop the cluster anyway] 413 * 4. on errors we always drop the cluster. thus, if we return 414 * an error, then the caller only has to worry about 415 * un-busying the main page (not the cluster pages). 416 * 5. on success, if !PGO_PDFREECLUST, we return the cluster 417 * with all pages busy (caller must un-busy and check 418 * wanted/released flags). 419 */ 420 421 int 422 uvm_pager_put(uobj, pg, ppsp_ptr, npages, flags, start, stop) 423 struct uvm_object *uobj; /* IN */ 424 struct vm_page *pg, ***ppsp_ptr;/* IN, IN/OUT */ 425 int *npages; /* IN/OUT */ 426 int flags; /* IN */ 427 voff_t start, stop; /* IN, IN */ 428 { 429 int result; 430 daddr_t swblk; 431 boolean_t async = (flags & PGO_SYNCIO) == 0; 432 struct vm_page **ppsp = *ppsp_ptr; 433 UVMHIST_FUNC("uvm_pager_put"); UVMHIST_CALLED(ubchist); 434 435 /* 436 * note that uobj is null if we are doing a swap-backed pageout. 437 * note that uobj is !null if we are doing normal object pageout. 438 * note that the page queues must be locked to cluster. 439 */ 440 441 if (uobj) { /* if !swap-backed */ 442 443 /* 444 * attempt to build a cluster for pageout using its 445 * make-put-cluster function (if it has one). 446 */ 447 448 if (uobj->pgops->pgo_mk_pcluster) { 449 ppsp = uobj->pgops->pgo_mk_pcluster(uobj, ppsp, 450 npages, pg, flags, start, stop); 451 *ppsp_ptr = ppsp; /* update caller's pointer */ 452 } else { 453 ppsp[0] = pg; 454 *npages = 1; 455 } 456 457 swblk = 0; /* XXX: keep gcc happy */ 458 459 } else { 460 461 /* 462 * for swap-backed pageout, the caller (the pagedaemon) has 463 * already built the cluster for us. the starting swap 464 * block we are writing to has been passed in as "start." 465 * "pg" could be NULL if there is no page we are especially 466 * interested in (in which case the whole cluster gets dropped 467 * in the event of an error or a sync "done"). 468 */ 469 swblk = (daddr_t) start; 470 /* ppsp and npages should be ok */ 471 } 472 473 /* now that we've clustered we can unlock the page queues */ 474 uvm_unlock_pageq(); 475 476 /* 477 * now attempt the I/O. if we have a failure and we are 478 * clustered, we will drop the cluster and try again. 479 */ 480 481 ReTry: 482 if (uobj) { 483 /* object is locked */ 484 result = uobj->pgops->pgo_put(uobj, ppsp, *npages, flags); 485 UVMHIST_LOG(ubchist, "put -> %d", result, 0,0,0); 486 /* object is now unlocked */ 487 } else { 488 /* nothing locked */ 489 result = uvm_swap_put(swblk, ppsp, *npages, flags); 490 /* nothing locked */ 491 } 492 493 /* 494 * we have attempted the I/O. 495 * 496 * if the I/O was a success then: 497 * if !PGO_PDFREECLUST, we return the cluster to the 498 * caller (who must un-busy all pages) 499 * else we un-busy cluster pages for the pagedaemon 500 * 501 * if I/O is pending (async i/o) then we return the pending code. 502 * [in this case the async i/o done function must clean up when 503 * i/o is done...] 504 */ 505 506 if (result == 0) { 507 if (flags & PGO_PDFREECLUST && !async) { 508 509 /* 510 * drop cluster and relock object for sync i/o. 511 */ 512 513 if (uobj) 514 /* required for dropcluster */ 515 simple_lock(&uobj->vmobjlock); 516 if (*npages > 1 || pg == NULL) 517 uvm_pager_dropcluster(uobj, pg, ppsp, npages, 518 PGO_PDFREECLUST); 519 520 /* if (uobj): object still locked, as per #3 */ 521 } 522 return (result); 523 } 524 525 /* 526 * a pager error occured. 527 * for transient errors, drop to a cluster of 1 page ("pg") 528 * and try again. for hard errors, don't bother retrying. 529 */ 530 531 if (*npages > 1 || pg == NULL) { 532 if (uobj) { 533 simple_lock(&uobj->vmobjlock); 534 } 535 uvm_pager_dropcluster(uobj, pg, ppsp, npages, PGO_REALLOCSWAP); 536 537 /* 538 * for failed swap-backed pageouts with a "pg", 539 * we need to reset pg's swslot to either: 540 * "swblk" (for transient errors, so we can retry), 541 * or 0 (for hard errors). 542 */ 543 544 if (uobj == NULL && pg != NULL) { 545 int nswblk = (result == EAGAIN) ? swblk : 0; 546 if (pg->pqflags & PQ_ANON) { 547 simple_lock(&pg->uanon->an_lock); 548 pg->uanon->an_swslot = nswblk; 549 simple_unlock(&pg->uanon->an_lock); 550 } else { 551 simple_lock(&pg->uobject->vmobjlock); 552 uao_set_swslot(pg->uobject, 553 pg->offset >> PAGE_SHIFT, 554 nswblk); 555 simple_unlock(&pg->uobject->vmobjlock); 556 } 557 } 558 if (result == EAGAIN) { 559 560 /* 561 * for transient failures, free all the swslots that 562 * we're not going to retry with. 563 */ 564 565 if (uobj == NULL) { 566 if (pg) { 567 uvm_swap_free(swblk + 1, *npages - 1); 568 } else { 569 uvm_swap_free(swblk, *npages); 570 } 571 } 572 if (pg) { 573 ppsp[0] = pg; 574 *npages = 1; 575 goto ReTry; 576 } 577 } else if (uobj == NULL) { 578 579 /* 580 * for hard errors on swap-backed pageouts, 581 * mark the swslots as bad. note that we do not 582 * free swslots that we mark bad. 583 */ 584 585 uvm_swap_markbad(swblk, *npages); 586 } 587 } 588 589 /* 590 * a pager error occured (even after dropping the cluster, if there 591 * was one). give up! the caller only has one page ("pg") 592 * to worry about. 593 */ 594 595 if (uobj && (flags & PGO_PDFREECLUST) != 0) 596 simple_lock(&uobj->vmobjlock); 597 return(result); 598 } 599 600 /* 601 * uvm_pager_dropcluster: drop a cluster we have built (because we 602 * got an error, or, if PGO_PDFREECLUST we are un-busying the 603 * cluster pages on behalf of the pagedaemon). 604 * 605 * => uobj, if non-null, is a non-swap-backed object that is 606 * locked by the caller. we return with this object still 607 * locked. 608 * => page queues are not locked 609 * => pg is our page of interest (the one we clustered around, can be null) 610 * => ppsp/npages is our current cluster 611 * => flags: PGO_PDFREECLUST: pageout was a success: un-busy cluster 612 * pages on behalf of the pagedaemon. 613 * PGO_REALLOCSWAP: drop previously allocated swap slots for 614 * clustered swap-backed pages (except for "pg" if !NULL) 615 * "swblk" is the start of swap alloc (e.g. for ppsp[0]) 616 * [only meaningful if swap-backed (uobj == NULL)] 617 */ 618 619 void 620 uvm_pager_dropcluster(uobj, pg, ppsp, npages, flags) 621 struct uvm_object *uobj; /* IN */ 622 struct vm_page *pg, **ppsp; /* IN, IN/OUT */ 623 int *npages; /* IN/OUT */ 624 int flags; 625 { 626 int lcv; 627 boolean_t obj_is_alive; 628 struct uvm_object *saved_uobj; 629 630 /* 631 * drop all pages but "pg" 632 */ 633 634 for (lcv = 0 ; lcv < *npages ; lcv++) { 635 636 /* skip "pg" or empty slot */ 637 if (ppsp[lcv] == pg || ppsp[lcv] == NULL) 638 continue; 639 640 /* 641 * if swap-backed, gain lock on object that owns page. note 642 * that PQ_ANON bit can't change as long as we are holding 643 * the PG_BUSY bit (so there is no need to lock the page 644 * queues to test it). 645 * 646 * once we have the lock, dispose of the pointer to swap, if 647 * requested 648 */ 649 if (!uobj) { 650 if (ppsp[lcv]->pqflags & PQ_ANON) { 651 simple_lock(&ppsp[lcv]->uanon->an_lock); 652 if (flags & PGO_REALLOCSWAP) 653 /* zap swap block */ 654 ppsp[lcv]->uanon->an_swslot = 0; 655 } else { 656 simple_lock(&ppsp[lcv]->uobject->vmobjlock); 657 if (flags & PGO_REALLOCSWAP) 658 uao_set_swslot(ppsp[lcv]->uobject, 659 ppsp[lcv]->offset >> PAGE_SHIFT, 0); 660 } 661 } 662 663 /* did someone want the page while we had it busy-locked? */ 664 if (ppsp[lcv]->flags & PG_WANTED) { 665 /* still holding obj lock */ 666 wakeup(ppsp[lcv]); 667 } 668 669 /* if page was released, release it. otherwise un-busy it */ 670 if (ppsp[lcv]->flags & PG_RELEASED) { 671 672 if (ppsp[lcv]->pqflags & PQ_ANON) { 673 /* so that anfree will free */ 674 ppsp[lcv]->flags &= ~(PG_BUSY); 675 UVM_PAGE_OWN(ppsp[lcv], NULL); 676 677 pmap_page_protect(ppsp[lcv], VM_PROT_NONE); 678 simple_unlock(&ppsp[lcv]->uanon->an_lock); 679 /* kills anon and frees pg */ 680 uvm_anfree(ppsp[lcv]->uanon); 681 682 continue; 683 } 684 685 /* 686 * pgo_releasepg will dump the page for us 687 */ 688 689 saved_uobj = ppsp[lcv]->uobject; 690 obj_is_alive = 691 saved_uobj->pgops->pgo_releasepg(ppsp[lcv], NULL); 692 693 /* for normal objects, "pg" is still PG_BUSY by us, 694 * so obj can't die */ 695 KASSERT(!uobj || obj_is_alive); 696 697 /* only unlock the object if it is still alive... */ 698 if (obj_is_alive && saved_uobj != uobj) 699 simple_unlock(&saved_uobj->vmobjlock); 700 701 /* 702 * XXXCDC: suppose uobj died in the pgo_releasepg? 703 * how pass that 704 * info up to caller. we are currently ignoring it... 705 */ 706 707 continue; /* next page */ 708 709 } else { 710 ppsp[lcv]->flags &= ~(PG_BUSY|PG_WANTED|PG_FAKE); 711 UVM_PAGE_OWN(ppsp[lcv], NULL); 712 } 713 714 /* 715 * if we are operating on behalf of the pagedaemon and we 716 * had a successful pageout update the page! 717 */ 718 if (flags & PGO_PDFREECLUST) { 719 pmap_clear_reference(ppsp[lcv]); 720 pmap_clear_modify(ppsp[lcv]); 721 ppsp[lcv]->flags |= PG_CLEAN; 722 } 723 724 /* if anonymous cluster, unlock object and move on */ 725 if (!uobj) { 726 if (ppsp[lcv]->pqflags & PQ_ANON) 727 simple_unlock(&ppsp[lcv]->uanon->an_lock); 728 else 729 simple_unlock(&ppsp[lcv]->uobject->vmobjlock); 730 } 731 } 732 } 733 734 /* 735 * interrupt-context iodone handler for nested i/o bufs. 736 * 737 * => must be at splbio(). 738 */ 739 740 void 741 uvm_aio_biodone1(bp) 742 struct buf *bp; 743 { 744 struct buf *mbp = bp->b_private; 745 746 KASSERT(mbp != bp); 747 if (bp->b_flags & B_ERROR) { 748 mbp->b_flags |= B_ERROR; 749 mbp->b_error = bp->b_error; 750 } 751 mbp->b_resid -= bp->b_bcount; 752 pool_put(&bufpool, bp); 753 if (mbp->b_resid == 0) { 754 biodone(mbp); 755 } 756 } 757 758 /* 759 * interrupt-context iodone handler for single-buf i/os 760 * or the top-level buf of a nested-buf i/o. 761 * 762 * => must be at splbio(). 763 */ 764 765 void 766 uvm_aio_biodone(bp) 767 struct buf *bp; 768 { 769 /* reset b_iodone for when this is a single-buf i/o. */ 770 bp->b_iodone = uvm_aio_aiodone; 771 772 simple_lock(&uvm.aiodoned_lock); /* locks uvm.aio_done */ 773 TAILQ_INSERT_TAIL(&uvm.aio_done, bp, b_freelist); 774 wakeup(&uvm.aiodoned); 775 simple_unlock(&uvm.aiodoned_lock); 776 } 777 778 /* 779 * uvm_aio_aiodone: do iodone processing for async i/os. 780 * this should be called in thread context, not interrupt context. 781 */ 782 783 void 784 uvm_aio_aiodone(bp) 785 struct buf *bp; 786 { 787 int npages = bp->b_bufsize >> PAGE_SHIFT; 788 struct vm_page *pg, *pgs[npages]; 789 struct uvm_object *uobj; 790 int s, i, error; 791 boolean_t write, swap; 792 UVMHIST_FUNC("uvm_aio_aiodone"); UVMHIST_CALLED(ubchist); 793 UVMHIST_LOG(ubchist, "bp %p", bp, 0,0,0); 794 795 error = (bp->b_flags & B_ERROR) ? (bp->b_error ? bp->b_error : EIO) : 0; 796 write = (bp->b_flags & B_READ) == 0; 797 /* XXXUBC B_NOCACHE is for swap pager, should be done differently */ 798 if (write && !(bp->b_flags & B_NOCACHE) && bioops.io_pageiodone) { 799 (*bioops.io_pageiodone)(bp); 800 } 801 802 uobj = NULL; 803 for (i = 0; i < npages; i++) { 804 pgs[i] = uvm_pageratop((vaddr_t)bp->b_data + (i << PAGE_SHIFT)); 805 UVMHIST_LOG(ubchist, "pgs[%d] = %p", i, pgs[i],0,0); 806 } 807 uvm_pagermapout((vaddr_t)bp->b_data, npages); 808 for (i = 0; i < npages; i++) { 809 pg = pgs[i]; 810 811 if (i == 0) { 812 swap = (pg->pqflags & PQ_SWAPBACKED) != 0; 813 if (!swap) { 814 uobj = pg->uobject; 815 simple_lock(&uobj->vmobjlock); 816 } 817 } 818 KASSERT(swap || pg->uobject == uobj); 819 if (swap) { 820 if (pg->pqflags & PQ_ANON) { 821 simple_lock(&pg->uanon->an_lock); 822 } else { 823 simple_lock(&pg->uobject->vmobjlock); 824 } 825 } 826 827 /* 828 * if this is a read and we got an error, mark the pages 829 * PG_RELEASED so that uvm_page_unbusy() will free them. 830 */ 831 832 if (!write && error) { 833 pg->flags |= PG_RELEASED; 834 continue; 835 } 836 KASSERT(!write || (pgs[i]->flags & PG_FAKE) == 0); 837 838 /* 839 * if this is a read and the page is PG_FAKE, 840 * or this was a successful write, 841 * mark the page PG_CLEAN and not PG_FAKE. 842 */ 843 844 if ((pgs[i]->flags & PG_FAKE) || (write && error != ENOMEM)) { 845 pmap_clear_reference(pgs[i]); 846 pmap_clear_modify(pgs[i]); 847 pgs[i]->flags |= PG_CLEAN; 848 pgs[i]->flags &= ~PG_FAKE; 849 } 850 uvm_pageactivate(pg); 851 if (swap) { 852 if (pg->pqflags & PQ_ANON) { 853 simple_unlock(&pg->uanon->an_lock); 854 } else { 855 simple_unlock(&pg->uobject->vmobjlock); 856 } 857 } 858 } 859 uvm_page_unbusy(pgs, npages); 860 if (!swap) { 861 simple_unlock(&uobj->vmobjlock); 862 } 863 864 s = splbio(); 865 if (write && (bp->b_flags & B_AGE) != 0) { 866 vwakeup(bp); 867 } 868 pool_put(&bufpool, bp); 869 splx(s); 870 } 871