1 /* $OpenBSD: uvm_fault.c,v 1.79 2014/11/16 12:31:00 deraadt Exp $ */ 2 /* $NetBSD: uvm_fault.c,v 1.51 2000/08/06 00:22:53 thorpej Exp $ */ 3 4 /* 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp 29 */ 30 31 /* 32 * uvm_fault.c: fault handler 33 */ 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/proc.h> 39 #include <sys/malloc.h> 40 #include <sys/mman.h> 41 #include <sys/user.h> 42 43 #include <uvm/uvm.h> 44 45 /* 46 * 47 * a word on page faults: 48 * 49 * types of page faults we handle: 50 * 51 * CASE 1: upper layer faults CASE 2: lower layer faults 52 * 53 * CASE 1A CASE 1B CASE 2A CASE 2B 54 * read/write1 write>1 read/write +-cow_write/zero 55 * | | | | 56 * +--|--+ +--|--+ +-----+ + | + | +-----+ 57 * amap | V | | ----------->new| | | | ^ | 58 * +-----+ +-----+ +-----+ + | + | +--|--+ 59 * | | | 60 * +-----+ +-----+ +--|--+ | +--|--+ 61 * uobj | d/c | | d/c | | V | +----| | 62 * +-----+ +-----+ +-----+ +-----+ 63 * 64 * d/c = don't care 65 * 66 * case [0]: layerless fault 67 * no amap or uobj is present. this is an error. 68 * 69 * case [1]: upper layer fault [anon active] 70 * 1A: [read] or [write with anon->an_ref == 1] 71 * I/O takes place in top level anon and uobj is not touched. 72 * 1B: [write with anon->an_ref > 1] 73 * new anon is alloc'd and data is copied off ["COW"] 74 * 75 * case [2]: lower layer fault [uobj] 76 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area] 77 * I/O takes place directly in object. 78 * 2B: [write to copy_on_write] or [read on NULL uobj] 79 * data is "promoted" from uobj to a new anon. 80 * if uobj is null, then we zero fill. 81 * 82 * we follow the standard UVM locking protocol ordering: 83 * 84 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ) 85 * we hold a PG_BUSY page if we unlock for I/O 86 * 87 * 88 * the code is structured as follows: 89 * 90 * - init the "IN" params in the ufi structure 91 * ReFault: 92 * - do lookups [locks maps], check protection, handle needs_copy 93 * - check for case 0 fault (error) 94 * - establish "range" of fault 95 * - if we have an amap lock it and extract the anons 96 * - if sequential advice deactivate pages behind us 97 * - at the same time check pmap for unmapped areas and anon for pages 98 * that we could map in (and do map it if found) 99 * - check object for resident pages that we could map in 100 * - if (case 2) goto Case2 101 * - >>> handle case 1 102 * - ensure source anon is resident in RAM 103 * - if case 1B alloc new anon and copy from source 104 * - map the correct page in 105 * Case2: 106 * - >>> handle case 2 107 * - ensure source page is resident (if uobj) 108 * - if case 2B alloc new anon and copy from source (could be zero 109 * fill if uobj == NULL) 110 * - map the correct page in 111 * - done! 112 * 113 * note on paging: 114 * if we have to do I/O we place a PG_BUSY page in the correct object, 115 * unlock everything, and do the I/O. when I/O is done we must reverify 116 * the state of the world before assuming that our data structures are 117 * valid. [because mappings could change while the map is unlocked] 118 * 119 * alternative 1: unbusy the page in question and restart the page fault 120 * from the top (ReFault). this is easy but does not take advantage 121 * of the information that we already have from our previous lookup, 122 * although it is possible that the "hints" in the vm_map will help here. 123 * 124 * alternative 2: the system already keeps track of a "version" number of 125 * a map. [i.e. every time you write-lock a map (e.g. to change a 126 * mapping) you bump the version number up by one...] so, we can save 127 * the version number of the map before we release the lock and start I/O. 128 * then when I/O is done we can relock and check the version numbers 129 * to see if anything changed. this might save us some over 1 because 130 * we don't have to unbusy the page and may be less compares(?). 131 * 132 * alternative 3: put in backpointers or a way to "hold" part of a map 133 * in place while I/O is in progress. this could be complex to 134 * implement (especially with structures like amap that can be referenced 135 * by multiple map entries, and figuring out what should wait could be 136 * complex as well...). 137 * 138 * given that we are not currently multiprocessor or multithreaded we might 139 * as well choose alternative 2 now. maybe alternative 3 would be useful 140 * in the future. XXX keep in mind for future consideration//rechecking. 141 */ 142 143 /* 144 * local data structures 145 */ 146 struct uvm_advice { 147 int nback; 148 int nforw; 149 }; 150 151 /* 152 * page range array: set up in uvmfault_init(). 153 */ 154 static struct uvm_advice uvmadvice[UVM_ADV_MASK + 1]; 155 156 #define UVM_MAXRANGE 16 /* must be max() of nback+nforw+1 */ 157 158 /* 159 * private prototypes 160 */ 161 static void uvmfault_amapcopy(struct uvm_faultinfo *); 162 static __inline void uvmfault_anonflush(struct vm_anon **, int); 163 void uvmfault_unlockmaps(struct uvm_faultinfo *, boolean_t); 164 void uvmfault_update_stats(struct uvm_faultinfo *); 165 166 /* 167 * inline functions 168 */ 169 /* 170 * uvmfault_anonflush: try and deactivate pages in specified anons 171 * 172 * => does not have to deactivate page if it is busy 173 */ 174 static __inline void 175 uvmfault_anonflush(struct vm_anon **anons, int n) 176 { 177 int lcv; 178 struct vm_page *pg; 179 180 for (lcv = 0 ; lcv < n ; lcv++) { 181 if (anons[lcv] == NULL) 182 continue; 183 pg = anons[lcv]->an_page; 184 if (pg && (pg->pg_flags & PG_BUSY) == 0 && pg->loan_count == 0) { 185 uvm_lock_pageq(); 186 if (pg->wire_count == 0) { 187 pmap_page_protect(pg, PROT_NONE); 188 uvm_pagedeactivate(pg); 189 } 190 uvm_unlock_pageq(); 191 } 192 } 193 } 194 195 /* 196 * normal functions 197 */ 198 /* 199 * uvmfault_init: compute proper values for the uvmadvice[] array. 200 */ 201 void 202 uvmfault_init() 203 { 204 int npages; 205 206 npages = atop(16384); 207 if (npages > 0) { 208 KASSERT(npages <= UVM_MAXRANGE / 2); 209 uvmadvice[POSIX_MADV_NORMAL].nforw = npages; 210 uvmadvice[POSIX_MADV_NORMAL].nback = npages - 1; 211 } 212 213 npages = atop(32768); 214 if (npages > 0) { 215 KASSERT(npages <= UVM_MAXRANGE / 2); 216 uvmadvice[POSIX_MADV_SEQUENTIAL].nforw = npages - 1; 217 uvmadvice[POSIX_MADV_SEQUENTIAL].nback = npages; 218 } 219 } 220 221 /* 222 * uvmfault_amapcopy: clear "needs_copy" in a map. 223 * 224 * => if we are out of RAM we sleep (waiting for more) 225 */ 226 static void 227 uvmfault_amapcopy(struct uvm_faultinfo *ufi) 228 { 229 230 /* while we haven't done the job */ 231 while (1) { 232 /* no mapping? give up. */ 233 if (uvmfault_lookup(ufi, TRUE) == FALSE) 234 return; 235 236 /* copy if needed. */ 237 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) 238 amap_copy(ufi->map, ufi->entry, M_NOWAIT, TRUE, 239 ufi->orig_rvaddr, ufi->orig_rvaddr + 1); 240 241 /* didn't work? must be out of RAM. sleep. */ 242 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 243 uvmfault_unlockmaps(ufi, TRUE); 244 uvm_wait("fltamapcopy"); 245 continue; 246 } 247 248 /* got it! */ 249 uvmfault_unlockmaps(ufi, TRUE); 250 return; 251 } 252 /*NOTREACHED*/ 253 } 254 255 /* 256 * uvmfault_anonget: get data in an anon into a non-busy, non-released 257 * page in that anon. 258 * 259 * => we don't move the page on the queues [gets moved later] 260 * => if we allocate a new page [we_own], it gets put on the queues. 261 * either way, the result is that the page is on the queues at return time 262 * => for pages which are on loan from a uvm_object (and thus are not 263 * owned by the anon): if successful, we return with the owning object 264 */ 265 int 266 uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap, 267 struct vm_anon *anon) 268 { 269 boolean_t we_own; /* we own anon's page? */ 270 boolean_t locked; /* did we relock? */ 271 struct vm_page *pg; 272 int result; 273 274 result = 0; /* XXX shut up gcc */ 275 uvmexp.fltanget++; 276 /* bump rusage counters */ 277 if (anon->an_page) 278 curproc->p_ru.ru_minflt++; 279 else 280 curproc->p_ru.ru_majflt++; 281 282 /* loop until we get it, or fail. */ 283 while (1) { 284 we_own = FALSE; /* TRUE if we set PG_BUSY on a page */ 285 pg = anon->an_page; 286 287 /* 288 * if there is a resident page and it is loaned, then anon 289 * may not own it. call out to uvm_anon_lockpage() to ensure 290 * the real owner of the page has been identified. 291 */ 292 if (pg && pg->loan_count) 293 pg = uvm_anon_lockloanpg(anon); 294 295 /* page there? make sure it is not busy/released. */ 296 if (pg) { 297 /* 298 * at this point, if the page has a uobject [meaning 299 * we have it on loan], then that uobject is locked 300 * by us! if the page is busy, we drop all the 301 * locks (including uobject) and try again. 302 */ 303 if ((pg->pg_flags & (PG_BUSY|PG_RELEASED)) == 0) { 304 return (VM_PAGER_OK); 305 } 306 atomic_setbits_int(&pg->pg_flags, PG_WANTED); 307 uvmexp.fltpgwait++; 308 309 /* 310 * the last unlock must be an atomic unlock+wait on 311 * the owner of page 312 */ 313 if (pg->uobject) { /* owner is uobject ? */ 314 uvmfault_unlockall(ufi, amap, NULL, anon); 315 UVM_WAIT(pg, FALSE, "anonget1",0); 316 } else { 317 /* anon owns page */ 318 uvmfault_unlockall(ufi, amap, NULL, NULL); 319 UVM_WAIT(pg, 0, "anonget2", 0); 320 } 321 /* ready to relock and try again */ 322 } else { 323 /* no page, we must try and bring it in. */ 324 pg = uvm_pagealloc(NULL, 0, anon, 0); 325 326 if (pg == NULL) { /* out of RAM. */ 327 uvmfault_unlockall(ufi, amap, NULL, anon); 328 uvmexp.fltnoram++; 329 uvm_wait("flt_noram1"); 330 /* ready to relock and try again */ 331 } else { 332 /* we set the PG_BUSY bit */ 333 we_own = TRUE; 334 uvmfault_unlockall(ufi, amap, NULL, anon); 335 336 /* 337 * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN 338 * page into the uvm_swap_get function with 339 * all data structures unlocked. note that 340 * it is ok to read an_swslot here because 341 * we hold PG_BUSY on the page. 342 */ 343 uvmexp.pageins++; 344 result = uvm_swap_get(pg, anon->an_swslot, 345 PGO_SYNCIO); 346 347 /* 348 * we clean up after the i/o below in the 349 * "we_own" case 350 */ 351 /* ready to relock and try again */ 352 } 353 } 354 355 /* now relock and try again */ 356 locked = uvmfault_relock(ufi); 357 358 /* 359 * if we own the page (i.e. we set PG_BUSY), then we need 360 * to clean up after the I/O. there are three cases to 361 * consider: 362 * [1] page released during I/O: free anon and ReFault. 363 * [2] I/O not OK. free the page and cause the fault 364 * to fail. 365 * [3] I/O OK! activate the page and sync with the 366 * non-we_own case (i.e. drop anon lock if not locked). 367 */ 368 if (we_own) { 369 if (pg->pg_flags & PG_WANTED) { 370 wakeup(pg); 371 } 372 /* un-busy! */ 373 atomic_clearbits_int(&pg->pg_flags, 374 PG_WANTED|PG_BUSY|PG_FAKE); 375 UVM_PAGE_OWN(pg, NULL); 376 377 /* 378 * if we were RELEASED during I/O, then our anon is 379 * no longer part of an amap. we need to free the 380 * anon and try again. 381 */ 382 if (pg->pg_flags & PG_RELEASED) { 383 pmap_page_protect(pg, PROT_NONE); 384 uvm_anfree(anon); /* frees page for us */ 385 if (locked) 386 uvmfault_unlockall(ufi, amap, NULL, 387 NULL); 388 uvmexp.fltpgrele++; 389 return (VM_PAGER_REFAULT); /* refault! */ 390 } 391 392 if (result != VM_PAGER_OK) { 393 KASSERT(result != VM_PAGER_PEND); 394 395 /* remove page from anon */ 396 anon->an_page = NULL; 397 398 /* 399 * remove the swap slot from the anon 400 * and mark the anon as having no real slot. 401 * don't free the swap slot, thus preventing 402 * it from being used again. 403 */ 404 uvm_swap_markbad(anon->an_swslot, 1); 405 anon->an_swslot = SWSLOT_BAD; 406 407 /* 408 * note: page was never !PG_BUSY, so it 409 * can't be mapped and thus no need to 410 * pmap_page_protect it... 411 */ 412 uvm_lock_pageq(); 413 uvm_pagefree(pg); 414 uvm_unlock_pageq(); 415 416 if (locked) 417 uvmfault_unlockall(ufi, amap, NULL, 418 anon); 419 return (VM_PAGER_ERROR); 420 } 421 422 /* 423 * must be OK, clear modify (already PG_CLEAN) 424 * and activate 425 */ 426 pmap_clear_modify(pg); 427 uvm_lock_pageq(); 428 uvm_pageactivate(pg); 429 uvm_unlock_pageq(); 430 } 431 432 /* we were not able to relock. restart fault. */ 433 if (!locked) 434 return (VM_PAGER_REFAULT); 435 436 /* verify no one touched the amap and moved the anon on us. */ 437 if (ufi != NULL && 438 amap_lookup(&ufi->entry->aref, 439 ufi->orig_rvaddr - ufi->entry->start) != anon) { 440 441 uvmfault_unlockall(ufi, amap, NULL, anon); 442 return (VM_PAGER_REFAULT); 443 } 444 445 /* try it again! */ 446 uvmexp.fltanretry++; 447 continue; 448 449 } /* while (1) */ 450 /*NOTREACHED*/ 451 } 452 453 /* 454 * Update statistics after fault resolution. 455 * - maxrss 456 */ 457 void 458 uvmfault_update_stats(struct uvm_faultinfo *ufi) 459 { 460 struct vm_map *map; 461 struct proc *p; 462 vsize_t res; 463 #ifndef pmap_resident_count 464 struct vm_space *vm; 465 #endif 466 467 map = ufi->orig_map; 468 469 /* Update the maxrss for the process. */ 470 if (map->flags & VM_MAP_ISVMSPACE) { 471 p = curproc; 472 KASSERT(p != NULL && &p->p_vmspace->vm_map == map); 473 474 #ifdef pmap_resident_count 475 res = pmap_resident_count(map->pmap); 476 #else 477 /* 478 * Rather inaccurate, but this is the current anon size 479 * of the vmspace. It's basically the resident size 480 * minus the mmapped in files/text. 481 */ 482 vm = (struct vmspace*)map; 483 res = vm->dsize; 484 #endif 485 486 /* Convert res from pages to kilobytes. */ 487 res <<= (PAGE_SHIFT - 10); 488 489 if (p->p_ru.ru_maxrss < res) 490 p->p_ru.ru_maxrss = res; 491 } 492 } 493 494 /* 495 * F A U L T - m a i n e n t r y p o i n t 496 */ 497 498 /* 499 * uvm_fault: page fault handler 500 * 501 * => called from MD code to resolve a page fault 502 * => VM data structures usually should be unlocked. however, it is 503 * possible to call here with the main map locked if the caller 504 * gets a write lock, sets it recursive, and then calls us (c.f. 505 * uvm_map_pageable). this should be avoided because it keeps 506 * the map locked off during I/O. 507 */ 508 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 509 ~PROT_WRITE : PROT_MASK) 510 int 511 uvm_fault(vm_map_t orig_map, vaddr_t vaddr, vm_fault_t fault_type, 512 vm_prot_t access_type) 513 { 514 struct uvm_faultinfo ufi; 515 vm_prot_t enter_prot; 516 boolean_t wired, narrow, promote, locked, shadowed; 517 int npages, nback, nforw, centeridx, result, lcv, gotpages; 518 vaddr_t startva, currva; 519 voff_t uoff; 520 paddr_t pa; 521 struct vm_amap *amap; 522 struct uvm_object *uobj; 523 struct vm_anon *anons_store[UVM_MAXRANGE], **anons, *anon, *oanon; 524 struct vm_page *pages[UVM_MAXRANGE], *pg, *uobjpage; 525 526 anon = NULL; 527 pg = NULL; 528 529 uvmexp.faults++; /* XXX: locking? */ 530 531 /* init the IN parameters in the ufi */ 532 ufi.orig_map = orig_map; 533 ufi.orig_rvaddr = trunc_page(vaddr); 534 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ 535 if (fault_type == VM_FAULT_WIRE) 536 narrow = TRUE; /* don't look for neighborhood 537 * pages on wire */ 538 else 539 narrow = FALSE; /* normal fault */ 540 541 /* "goto ReFault" means restart the page fault from ground zero. */ 542 ReFault: 543 /* lookup and lock the maps */ 544 if (uvmfault_lookup(&ufi, FALSE) == FALSE) { 545 return (EFAULT); 546 } 547 548 #ifdef DIAGNOSTIC 549 if ((ufi.map->flags & VM_MAP_PAGEABLE) == 0) 550 panic("uvm_fault: fault on non-pageable map (%p, 0x%lx)", 551 ufi.map, vaddr); 552 #endif 553 554 /* check protection */ 555 if ((ufi.entry->protection & access_type) != access_type) { 556 uvmfault_unlockmaps(&ufi, FALSE); 557 return (EACCES); 558 } 559 560 /* 561 * "enter_prot" is the protection we want to enter the page in at. 562 * for certain pages (e.g. copy-on-write pages) this protection can 563 * be more strict than ufi.entry->protection. "wired" means either 564 * the entry is wired or we are fault-wiring the pg. 565 */ 566 567 enter_prot = ufi.entry->protection; 568 wired = VM_MAPENT_ISWIRED(ufi.entry) || (fault_type == VM_FAULT_WIRE); 569 if (wired) 570 access_type = enter_prot; /* full access for wired */ 571 572 /* handle "needs_copy" case. */ 573 if (UVM_ET_ISNEEDSCOPY(ufi.entry)) { 574 if ((access_type & PROT_WRITE) || 575 (ufi.entry->object.uvm_obj == NULL)) { 576 /* need to clear */ 577 uvmfault_unlockmaps(&ufi, FALSE); 578 uvmfault_amapcopy(&ufi); 579 uvmexp.fltamcopy++; 580 goto ReFault; 581 } else { 582 /* 583 * ensure that we pmap_enter page R/O since 584 * needs_copy is still true 585 */ 586 enter_prot &= ~PROT_WRITE; 587 } 588 } 589 590 /* identify the players */ 591 amap = ufi.entry->aref.ar_amap; /* top layer */ 592 uobj = ufi.entry->object.uvm_obj; /* bottom layer */ 593 594 /* 595 * check for a case 0 fault. if nothing backing the entry then 596 * error now. 597 */ 598 if (amap == NULL && uobj == NULL) { 599 uvmfault_unlockmaps(&ufi, FALSE); 600 return (EFAULT); 601 } 602 603 /* 604 * establish range of interest based on advice from mapper 605 * and then clip to fit map entry. note that we only want 606 * to do this the first time through the fault. if we 607 * ReFault we will disable this by setting "narrow" to true. 608 */ 609 if (narrow == FALSE) { 610 611 /* wide fault (!narrow) */ 612 nback = min(uvmadvice[ufi.entry->advice].nback, 613 (ufi.orig_rvaddr - ufi.entry->start) >> PAGE_SHIFT); 614 startva = ufi.orig_rvaddr - ((vsize_t)nback << PAGE_SHIFT); 615 nforw = min(uvmadvice[ufi.entry->advice].nforw, 616 ((ufi.entry->end - ufi.orig_rvaddr) >> 617 PAGE_SHIFT) - 1); 618 /* 619 * note: "-1" because we don't want to count the 620 * faulting page as forw 621 */ 622 npages = nback + nforw + 1; 623 centeridx = nback; 624 625 narrow = TRUE; /* ensure only once per-fault */ 626 } else { 627 /* narrow fault! */ 628 nback = nforw = 0; 629 startva = ufi.orig_rvaddr; 630 npages = 1; 631 centeridx = 0; 632 } 633 634 /* if we've got an amap, extract current anons. */ 635 if (amap) { 636 anons = anons_store; 637 amap_lookups(&ufi.entry->aref, startva - ufi.entry->start, 638 anons, npages); 639 } else { 640 anons = NULL; /* to be safe */ 641 } 642 643 /* 644 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages 645 * now and then forget about them (for the rest of the fault). 646 */ 647 if (ufi.entry->advice == MADV_SEQUENTIAL && nback != 0) { 648 /* flush back-page anons? */ 649 if (amap) 650 uvmfault_anonflush(anons, nback); 651 652 /* flush object? */ 653 if (uobj) { 654 uoff = (startva - ufi.entry->start) + ufi.entry->offset; 655 (void) uobj->pgops->pgo_flush(uobj, uoff, uoff + 656 ((vsize_t)nback << PAGE_SHIFT), PGO_DEACTIVATE); 657 } 658 659 /* now forget about the backpages */ 660 if (amap) 661 anons += nback; 662 startva += ((vsize_t)nback << PAGE_SHIFT); 663 npages -= nback; 664 centeridx = 0; 665 } 666 667 /* 668 * map in the backpages and frontpages we found in the amap in hopes 669 * of preventing future faults. we also init the pages[] array as 670 * we go. 671 */ 672 currva = startva; 673 shadowed = FALSE; 674 for (lcv = 0 ; lcv < npages ; lcv++, currva += PAGE_SIZE) { 675 /* 676 * dont play with VAs that are already mapped 677 * except for center) 678 */ 679 if (lcv != centeridx && 680 pmap_extract(ufi.orig_map->pmap, currva, &pa)) { 681 pages[lcv] = PGO_DONTCARE; 682 continue; 683 } 684 685 /* unmapped or center page. check if any anon at this level. */ 686 if (amap == NULL || anons[lcv] == NULL) { 687 pages[lcv] = NULL; 688 continue; 689 } 690 691 /* check for present page and map if possible. re-activate it. */ 692 pages[lcv] = PGO_DONTCARE; 693 if (lcv == centeridx) { /* save center for later! */ 694 shadowed = TRUE; 695 continue; 696 } 697 anon = anons[lcv]; 698 /* ignore loaned pages */ 699 if (anon->an_page && anon->an_page->loan_count == 0 && 700 (anon->an_page->pg_flags & (PG_RELEASED|PG_BUSY)) == 0) { 701 uvm_lock_pageq(); 702 uvm_pageactivate(anon->an_page); /* reactivate */ 703 uvm_unlock_pageq(); 704 uvmexp.fltnamap++; 705 706 /* 707 * Since this isn't the page that's actually faulting, 708 * ignore pmap_enter() failures; it's not critical 709 * that we enter these right now. 710 */ 711 (void) pmap_enter(ufi.orig_map->pmap, currva, 712 VM_PAGE_TO_PHYS(anon->an_page), 713 (anon->an_ref > 1) ? (enter_prot & ~PROT_WRITE) : 714 enter_prot, 715 PMAP_CANFAIL | 716 (VM_MAPENT_ISWIRED(ufi.entry) ? PMAP_WIRED : 0)); 717 } 718 } 719 if (npages > 1) 720 pmap_update(ufi.orig_map->pmap); 721 722 /* (shadowed == TRUE) if there is an anon at the faulting address */ 723 /* 724 * note that if we are really short of RAM we could sleep in the above 725 * call to pmap_enter. bad? 726 * 727 * XXX Actually, that is bad; pmap_enter() should just fail in that 728 * XXX case. --thorpej 729 */ 730 /* 731 * if the desired page is not shadowed by the amap and we have a 732 * backing object, then we check to see if the backing object would 733 * prefer to handle the fault itself (rather than letting us do it 734 * with the usual pgo_get hook). the backing object signals this by 735 * providing a pgo_fault routine. 736 */ 737 if (uobj && shadowed == FALSE && uobj->pgops->pgo_fault != NULL) { 738 result = uobj->pgops->pgo_fault(&ufi, startva, pages, npages, 739 centeridx, fault_type, access_type, 740 PGO_LOCKED); 741 742 if (result == VM_PAGER_OK) 743 return (0); /* pgo_fault did pmap enter */ 744 else if (result == VM_PAGER_REFAULT) 745 goto ReFault; /* try again! */ 746 else 747 return (EACCES); 748 } 749 750 /* 751 * now, if the desired page is not shadowed by the amap and we have 752 * a backing object that does not have a special fault routine, then 753 * we ask (with pgo_get) the object for resident pages that we care 754 * about and attempt to map them in. we do not let pgo_get block 755 * (PGO_LOCKED). 756 * 757 * ("get" has the option of doing a pmap_enter for us) 758 */ 759 if (uobj && shadowed == FALSE) { 760 uvmexp.fltlget++; 761 gotpages = npages; 762 (void) uobj->pgops->pgo_get(uobj, ufi.entry->offset + 763 (startva - ufi.entry->start), 764 pages, &gotpages, centeridx, 765 access_type & MASK(ufi.entry), 766 ufi.entry->advice, PGO_LOCKED); 767 768 /* check for pages to map, if we got any */ 769 uobjpage = NULL; 770 if (gotpages) { 771 currva = startva; 772 for (lcv = 0 ; lcv < npages ; 773 lcv++, currva += PAGE_SIZE) { 774 if (pages[lcv] == NULL || 775 pages[lcv] == PGO_DONTCARE) 776 continue; 777 778 KASSERT((pages[lcv]->pg_flags & PG_RELEASED) == 0); 779 780 /* 781 * if center page is resident and not 782 * PG_BUSY, then pgo_get made it PG_BUSY 783 * for us and gave us a handle to it. 784 * remember this page as "uobjpage." 785 * (for later use). 786 */ 787 if (lcv == centeridx) { 788 uobjpage = pages[lcv]; 789 continue; 790 } 791 792 /* 793 * note: calling pgo_get with locked data 794 * structures returns us pages which are 795 * neither busy nor released, so we don't 796 * need to check for this. we can just 797 * directly enter the page (after moving it 798 * to the head of the active queue [useful?]). 799 */ 800 801 uvm_lock_pageq(); 802 uvm_pageactivate(pages[lcv]); /* reactivate */ 803 uvm_unlock_pageq(); 804 uvmexp.fltnomap++; 805 806 /* 807 * Since this page isn't the page that's 808 * actually faulting, ignore pmap_enter() 809 * failures; it's not critical that we 810 * enter these right now. 811 */ 812 (void) pmap_enter(ufi.orig_map->pmap, currva, 813 VM_PAGE_TO_PHYS(pages[lcv]), 814 enter_prot & MASK(ufi.entry), 815 PMAP_CANFAIL | 816 (wired ? PMAP_WIRED : 0)); 817 818 /* 819 * NOTE: page can't be PG_WANTED because 820 * we've held the lock the whole time 821 * we've had the handle. 822 */ 823 atomic_clearbits_int(&pages[lcv]->pg_flags, 824 PG_BUSY); 825 UVM_PAGE_OWN(pages[lcv], NULL); 826 } /* for "lcv" loop */ 827 pmap_update(ufi.orig_map->pmap); 828 } /* "gotpages" != 0 */ 829 /* note: object still _locked_ */ 830 } else { 831 uobjpage = NULL; 832 } 833 834 /* 835 * note that at this point we are done with any front or back pages. 836 * we are now going to focus on the center page (i.e. the one we've 837 * faulted on). if we have faulted on the top (anon) layer 838 * [i.e. case 1], then the anon we want is anons[centeridx] (we have 839 * not touched it yet). if we have faulted on the bottom (uobj) 840 * layer [i.e. case 2] and the page was both present and available, 841 * then we've got a pointer to it as "uobjpage" and we've already 842 * made it BUSY. 843 */ 844 /* 845 * there are four possible cases we must address: 1A, 1B, 2A, and 2B 846 */ 847 /* redirect case 2: if we are not shadowed, go to case 2. */ 848 if (shadowed == FALSE) 849 goto Case2; 850 851 /* handle case 1: fault on an anon in our amap */ 852 anon = anons[centeridx]; 853 854 /* 855 * no matter if we have case 1A or case 1B we are going to need to 856 * have the anon's memory resident. ensure that now. 857 */ 858 /* 859 * let uvmfault_anonget do the dirty work. 860 * also, if it is OK, then the anon's page is on the queues. 861 */ 862 result = uvmfault_anonget(&ufi, amap, anon); 863 switch (result) { 864 case VM_PAGER_OK: 865 break; 866 867 case VM_PAGER_REFAULT: 868 goto ReFault; 869 870 case VM_PAGER_ERROR: 871 /* 872 * An error occured while trying to bring in the 873 * page -- this is the only error we return right 874 * now. 875 */ 876 return (EACCES); /* XXX */ 877 default: 878 #ifdef DIAGNOSTIC 879 panic("uvm_fault: uvmfault_anonget -> %d", result); 880 #else 881 return (EACCES); 882 #endif 883 } 884 885 /* uobj is non null if the page is on loan from an object (i.e. uobj) */ 886 uobj = anon->an_page->uobject; 887 888 /* special handling for loaned pages */ 889 if (anon->an_page->loan_count) { 890 if ((access_type & PROT_WRITE) == 0) { 891 /* 892 * for read faults on loaned pages we just cap the 893 * protection at read-only. 894 */ 895 enter_prot = enter_prot & ~PROT_WRITE; 896 } else { 897 /* 898 * note that we can't allow writes into a loaned page! 899 * 900 * if we have a write fault on a loaned page in an 901 * anon then we need to look at the anon's ref count. 902 * if it is greater than one then we are going to do 903 * a normal copy-on-write fault into a new anon (this 904 * is not a problem). however, if the reference count 905 * is one (a case where we would normally allow a 906 * write directly to the page) then we need to kill 907 * the loan before we continue. 908 */ 909 910 /* >1 case is already ok */ 911 if (anon->an_ref == 1) { 912 /* get new un-owned replacement page */ 913 pg = uvm_pagealloc(NULL, 0, NULL, 0); 914 if (pg == NULL) { 915 uvmfault_unlockall(&ufi, amap, uobj, 916 anon); 917 uvm_wait("flt_noram2"); 918 goto ReFault; 919 } 920 921 /* copy data, kill loan */ 922 /* copy old -> new */ 923 uvm_pagecopy(anon->an_page, pg); 924 925 /* force reload */ 926 pmap_page_protect(anon->an_page, PROT_NONE); 927 uvm_lock_pageq(); /* KILL loan */ 928 if (uobj) 929 /* if we were loaning */ 930 anon->an_page->loan_count--; 931 anon->an_page->uanon = NULL; 932 /* in case we owned */ 933 atomic_clearbits_int( 934 &anon->an_page->pg_flags, PQ_ANON); 935 uvm_pageactivate(pg); 936 uvm_unlock_pageq(); 937 if (uobj) { 938 uobj = NULL; 939 } 940 941 /* install new page in anon */ 942 anon->an_page = pg; 943 pg->uanon = anon; 944 atomic_setbits_int(&pg->pg_flags, PQ_ANON); 945 atomic_clearbits_int(&pg->pg_flags, 946 PG_BUSY|PG_FAKE); 947 UVM_PAGE_OWN(pg, NULL); 948 } /* ref == 1 */ 949 } /* write fault */ 950 } /* loan count */ 951 952 /* 953 * if we are case 1B then we will need to allocate a new blank 954 * anon to transfer the data into. note that we have a lock 955 * on anon, so no one can busy or release the page until we are done. 956 * also note that the ref count can't drop to zero here because 957 * it is > 1 and we are only dropping one ref. 958 * 959 * in the (hopefully very rare) case that we are out of RAM we 960 * will wait for more RAM, and refault. 961 * 962 * if we are out of anon VM we kill the process (XXX: could wait?). 963 */ 964 965 if ((access_type & PROT_WRITE) != 0 && anon->an_ref > 1) { 966 uvmexp.flt_acow++; 967 oanon = anon; /* oanon = old */ 968 anon = uvm_analloc(); 969 if (anon) { 970 pg = uvm_pagealloc(NULL, 0, anon, 0); 971 } 972 973 /* check for out of RAM */ 974 if (anon == NULL || pg == NULL) { 975 if (anon) 976 uvm_anfree(anon); 977 uvmfault_unlockall(&ufi, amap, uobj, oanon); 978 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 979 if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) { 980 uvmexp.fltnoanon++; 981 return (ENOMEM); 982 } 983 984 uvmexp.fltnoram++; 985 uvm_wait("flt_noram3"); /* out of RAM, wait for more */ 986 goto ReFault; 987 } 988 989 /* got all resources, replace anon with nanon */ 990 uvm_pagecopy(oanon->an_page, pg); /* pg now !PG_CLEAN */ 991 /* un-busy! new page */ 992 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE); 993 UVM_PAGE_OWN(pg, NULL); 994 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start, 995 anon, 1); 996 997 /* deref: can not drop to zero here by defn! */ 998 oanon->an_ref--; 999 1000 /* 1001 * note: anon is _not_ locked, but we have the sole references 1002 * to in from amap. 1003 * thus, no one can get at it until we are done with it. 1004 */ 1005 } else { 1006 uvmexp.flt_anon++; 1007 oanon = anon; 1008 pg = anon->an_page; 1009 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */ 1010 enter_prot = enter_prot & ~PROT_WRITE; 1011 } 1012 1013 /* 1014 * now map the page in ... 1015 * XXX: old fault unlocks object before pmap_enter. this seems 1016 * suspect since some other thread could blast the page out from 1017 * under us between the unlock and the pmap_enter. 1018 */ 1019 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1020 enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) 1021 != 0) { 1022 /* 1023 * No need to undo what we did; we can simply think of 1024 * this as the pmap throwing away the mapping information. 1025 * 1026 * We do, however, have to go through the ReFault path, 1027 * as the map may change while we're asleep. 1028 */ 1029 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1030 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1031 if (uvmexp.swpgonly == uvmexp.swpages) { 1032 /* XXX instrumentation */ 1033 return (ENOMEM); 1034 } 1035 /* XXX instrumentation */ 1036 uvm_wait("flt_pmfail1"); 1037 goto ReFault; 1038 } 1039 1040 /* ... update the page queues. */ 1041 uvm_lock_pageq(); 1042 1043 if (fault_type == VM_FAULT_WIRE) { 1044 uvm_pagewire(pg); 1045 /* 1046 * since the now-wired page cannot be paged out, 1047 * release its swap resources for others to use. 1048 * since an anon with no swap cannot be PG_CLEAN, 1049 * clear its clean flag now. 1050 */ 1051 atomic_clearbits_int(&pg->pg_flags, PG_CLEAN); 1052 uvm_anon_dropswap(anon); 1053 } else { 1054 /* activate it */ 1055 uvm_pageactivate(pg); 1056 } 1057 1058 uvm_unlock_pageq(); 1059 1060 /* done case 1! finish up by unlocking everything and returning success */ 1061 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1062 pmap_update(ufi.orig_map->pmap); 1063 return (0); 1064 1065 1066 Case2: 1067 /* handle case 2: faulting on backing object or zero fill */ 1068 /* 1069 * note that uobjpage can not be PGO_DONTCARE at this point. we now 1070 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we 1071 * have a backing object, check and see if we are going to promote 1072 * the data up to an anon during the fault. 1073 */ 1074 if (uobj == NULL) { 1075 uobjpage = PGO_DONTCARE; 1076 promote = TRUE; /* always need anon here */ 1077 } else { 1078 KASSERT(uobjpage != PGO_DONTCARE); 1079 promote = (access_type & PROT_WRITE) && 1080 UVM_ET_ISCOPYONWRITE(ufi.entry); 1081 } 1082 1083 /* 1084 * if uobjpage is not null then we do not need to do I/O to get the 1085 * uobjpage. 1086 * 1087 * if uobjpage is null, then we need to ask the pager to 1088 * get the data for us. once we have the data, we need to reverify 1089 * the state the world. we are currently not holding any resources. 1090 */ 1091 if (uobjpage) { 1092 /* update rusage counters */ 1093 curproc->p_ru.ru_minflt++; 1094 } else { 1095 /* update rusage counters */ 1096 curproc->p_ru.ru_majflt++; 1097 1098 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1099 1100 uvmexp.fltget++; 1101 gotpages = 1; 1102 uoff = (ufi.orig_rvaddr - ufi.entry->start) + ufi.entry->offset; 1103 result = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages, 1104 0, access_type & MASK(ufi.entry), ufi.entry->advice, 1105 PGO_SYNCIO); 1106 1107 /* recover from I/O */ 1108 if (result != VM_PAGER_OK) { 1109 KASSERT(result != VM_PAGER_PEND); 1110 1111 if (result == VM_PAGER_AGAIN) { 1112 tsleep(&lbolt, PVM, "fltagain2", 0); 1113 goto ReFault; 1114 } 1115 1116 if (!UVM_ET_ISNOFAULT(ufi.entry)) 1117 return (EACCES); /* XXX i/o error */ 1118 1119 uobjpage = PGO_DONTCARE; 1120 promote = TRUE; 1121 } 1122 1123 /* re-verify the state of the world. */ 1124 locked = uvmfault_relock(&ufi); 1125 1126 /* 1127 * Re-verify that amap slot is still free. if there is 1128 * a problem, we clean up. 1129 */ 1130 if (locked && amap && amap_lookup(&ufi.entry->aref, 1131 ufi.orig_rvaddr - ufi.entry->start)) { 1132 if (locked) 1133 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1134 locked = FALSE; 1135 } 1136 1137 /* didn't get the lock? release the page and retry. */ 1138 if (locked == FALSE && uobjpage != PGO_DONTCARE) { 1139 uvm_lock_pageq(); 1140 /* make sure it is in queues */ 1141 uvm_pageactivate(uobjpage); 1142 uvm_unlock_pageq(); 1143 1144 if (uobjpage->pg_flags & PG_WANTED) 1145 /* still holding object lock */ 1146 wakeup(uobjpage); 1147 atomic_clearbits_int(&uobjpage->pg_flags, 1148 PG_BUSY|PG_WANTED); 1149 UVM_PAGE_OWN(uobjpage, NULL); 1150 goto ReFault; 1151 } 1152 1153 /* 1154 * we have the data in uobjpage which is PG_BUSY 1155 */ 1156 } 1157 1158 /* 1159 * notes: 1160 * - at this point uobjpage can not be NULL 1161 * - at this point uobjpage could be PG_WANTED (handle later) 1162 */ 1163 if (promote == FALSE) { 1164 /* 1165 * we are not promoting. if the mapping is COW ensure that we 1166 * don't give more access than we should (e.g. when doing a read 1167 * fault on a COPYONWRITE mapping we want to map the COW page in 1168 * R/O even though the entry protection could be R/W). 1169 * 1170 * set "pg" to the page we want to map in (uobjpage, usually) 1171 */ 1172 uvmexp.flt_obj++; 1173 if (UVM_ET_ISCOPYONWRITE(ufi.entry)) 1174 enter_prot &= ~PROT_WRITE; 1175 pg = uobjpage; /* map in the actual object */ 1176 1177 /* assert(uobjpage != PGO_DONTCARE) */ 1178 1179 /* 1180 * we are faulting directly on the page. be careful 1181 * about writing to loaned pages... 1182 */ 1183 if (uobjpage->loan_count) { 1184 1185 if ((access_type & PROT_WRITE) == 0) { 1186 /* read fault: cap the protection at readonly */ 1187 /* cap! */ 1188 enter_prot = enter_prot & ~PROT_WRITE; 1189 } else { 1190 /* write fault: must break the loan here */ 1191 /* alloc new un-owned page */ 1192 pg = uvm_pagealloc(NULL, 0, NULL, 0); 1193 1194 if (pg == NULL) { 1195 /* 1196 * drop ownership of page, it can't 1197 * be released 1198 */ 1199 if (uobjpage->pg_flags & PG_WANTED) 1200 wakeup(uobjpage); 1201 atomic_clearbits_int( 1202 &uobjpage->pg_flags, 1203 PG_BUSY|PG_WANTED); 1204 UVM_PAGE_OWN(uobjpage, NULL); 1205 1206 uvm_lock_pageq(); 1207 /* activate: we will need it later */ 1208 uvm_pageactivate(uobjpage); 1209 1210 uvm_unlock_pageq(); 1211 uvmfault_unlockall(&ufi, amap, uobj, 1212 NULL); 1213 uvmexp.fltnoram++; 1214 uvm_wait("flt_noram4"); 1215 goto ReFault; 1216 } 1217 1218 /* 1219 * copy the data from the old page to the new 1220 * one and clear the fake/clean flags on the 1221 * new page (keep it busy). force a reload 1222 * of the old page by clearing it from all 1223 * pmaps. then lock the page queues to 1224 * rename the pages. 1225 */ 1226 uvm_pagecopy(uobjpage, pg); /* old -> new */ 1227 atomic_clearbits_int(&pg->pg_flags, 1228 PG_FAKE|PG_CLEAN); 1229 pmap_page_protect(uobjpage, PROT_NONE); 1230 if (uobjpage->pg_flags & PG_WANTED) 1231 wakeup(uobjpage); 1232 atomic_clearbits_int(&uobjpage->pg_flags, 1233 PG_BUSY|PG_WANTED); 1234 UVM_PAGE_OWN(uobjpage, NULL); 1235 1236 uvm_lock_pageq(); 1237 uoff = uobjpage->offset; 1238 /* remove old page */ 1239 uvm_pagerealloc(uobjpage, NULL, 0); 1240 1241 /* 1242 * at this point we have absolutely no 1243 * control over uobjpage 1244 */ 1245 /* install new page */ 1246 uvm_pagerealloc(pg, uobj, uoff); 1247 uvm_unlock_pageq(); 1248 1249 /* 1250 * done! loan is broken and "pg" is 1251 * PG_BUSY. it can now replace uobjpage. 1252 */ 1253 1254 uobjpage = pg; 1255 1256 } /* write fault case */ 1257 } /* if loan_count */ 1258 } else { 1259 /* 1260 * if we are going to promote the data to an anon we 1261 * allocate a blank anon here and plug it into our amap. 1262 */ 1263 #ifdef DIAGNOSTIC 1264 if (amap == NULL) 1265 panic("uvm_fault: want to promote data, but no anon"); 1266 #endif 1267 1268 anon = uvm_analloc(); 1269 if (anon) { 1270 /* 1271 * In `Fill in data...' below, if 1272 * uobjpage == PGO_DONTCARE, we want 1273 * a zero'd, dirty page, so have 1274 * uvm_pagealloc() do that for us. 1275 */ 1276 pg = uvm_pagealloc(NULL, 0, anon, 1277 (uobjpage == PGO_DONTCARE) ? UVM_PGA_ZERO : 0); 1278 } 1279 1280 /* 1281 * out of memory resources? 1282 */ 1283 if (anon == NULL || pg == NULL) { 1284 /* arg! must unbusy our page and fail or sleep. */ 1285 if (uobjpage != PGO_DONTCARE) { 1286 uvm_lock_pageq(); 1287 uvm_pageactivate(uobjpage); 1288 uvm_unlock_pageq(); 1289 1290 if (uobjpage->pg_flags & PG_WANTED) 1291 wakeup(uobjpage); 1292 atomic_clearbits_int(&uobjpage->pg_flags, 1293 PG_BUSY|PG_WANTED); 1294 UVM_PAGE_OWN(uobjpage, NULL); 1295 } 1296 1297 /* unlock and fail ... */ 1298 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1299 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1300 if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) { 1301 uvmexp.fltnoanon++; 1302 return (ENOMEM); 1303 } 1304 1305 uvm_anfree(anon); 1306 uvmexp.fltnoram++; 1307 uvm_wait("flt_noram5"); 1308 goto ReFault; 1309 } 1310 1311 /* fill in the data */ 1312 if (uobjpage != PGO_DONTCARE) { 1313 uvmexp.flt_prcopy++; 1314 /* copy page [pg now dirty] */ 1315 uvm_pagecopy(uobjpage, pg); 1316 1317 /* 1318 * promote to shared amap? make sure all sharing 1319 * procs see it 1320 */ 1321 if ((amap_flags(amap) & AMAP_SHARED) != 0) { 1322 pmap_page_protect(uobjpage, PROT_NONE); 1323 } 1324 1325 /* dispose of uobjpage. drop handle to uobj as well. */ 1326 if (uobjpage->pg_flags & PG_WANTED) 1327 wakeup(uobjpage); 1328 atomic_clearbits_int(&uobjpage->pg_flags, 1329 PG_BUSY|PG_WANTED); 1330 UVM_PAGE_OWN(uobjpage, NULL); 1331 uvm_lock_pageq(); 1332 uvm_pageactivate(uobjpage); 1333 uvm_unlock_pageq(); 1334 uobj = NULL; 1335 } else { 1336 uvmexp.flt_przero++; 1337 /* 1338 * Page is zero'd and marked dirty by uvm_pagealloc() 1339 * above. 1340 */ 1341 } 1342 1343 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start, 1344 anon, 0); 1345 } 1346 1347 /* note: pg is either the uobjpage or the new page in the new anon */ 1348 /* 1349 * all resources are present. we can now map it in and free our 1350 * resources. 1351 */ 1352 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1353 enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) 1354 != 0) { 1355 /* 1356 * No need to undo what we did; we can simply think of 1357 * this as the pmap throwing away the mapping information. 1358 * 1359 * We do, however, have to go through the ReFault path, 1360 * as the map may change while we're asleep. 1361 */ 1362 if (pg->pg_flags & PG_WANTED) 1363 wakeup(pg); 1364 1365 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED); 1366 UVM_PAGE_OWN(pg, NULL); 1367 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1368 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1369 if (uvmexp.swpgonly == uvmexp.swpages) { 1370 /* XXX instrumentation */ 1371 return (ENOMEM); 1372 } 1373 /* XXX instrumentation */ 1374 uvm_wait("flt_pmfail2"); 1375 goto ReFault; 1376 } 1377 1378 uvm_lock_pageq(); 1379 1380 if (fault_type == VM_FAULT_WIRE) { 1381 uvm_pagewire(pg); 1382 if (pg->pg_flags & PQ_AOBJ) { 1383 /* 1384 * since the now-wired page cannot be paged out, 1385 * release its swap resources for others to use. 1386 * since an aobj page with no swap cannot be PG_CLEAN, 1387 * clear its clean flag now. 1388 */ 1389 atomic_clearbits_int(&pg->pg_flags, PG_CLEAN); 1390 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT); 1391 } 1392 } else { 1393 /* activate it */ 1394 uvm_pageactivate(pg); 1395 } 1396 uvm_unlock_pageq(); 1397 1398 if (pg->pg_flags & PG_WANTED) 1399 wakeup(pg); 1400 1401 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED); 1402 UVM_PAGE_OWN(pg, NULL); 1403 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1404 pmap_update(ufi.orig_map->pmap); 1405 1406 return (0); 1407 } 1408 1409 1410 /* 1411 * uvm_fault_wire: wire down a range of virtual addresses in a map. 1412 * 1413 * => map may be read-locked by caller, but MUST NOT be write-locked. 1414 * => if map is read-locked, any operations which may cause map to 1415 * be write-locked in uvm_fault() must be taken care of by 1416 * the caller. See uvm_map_pageable(). 1417 */ 1418 int 1419 uvm_fault_wire(vm_map_t map, vaddr_t start, vaddr_t end, vm_prot_t access_type) 1420 { 1421 vaddr_t va; 1422 pmap_t pmap; 1423 int rv; 1424 1425 pmap = vm_map_pmap(map); 1426 1427 /* 1428 * now fault it in a page at a time. if the fault fails then we have 1429 * to undo what we have done. note that in uvm_fault PROT_NONE 1430 * is replaced with the max protection if fault_type is VM_FAULT_WIRE. 1431 */ 1432 for (va = start ; va < end ; va += PAGE_SIZE) { 1433 rv = uvm_fault(map, va, VM_FAULT_WIRE, access_type); 1434 if (rv) { 1435 if (va != start) { 1436 uvm_fault_unwire(map, start, va); 1437 } 1438 return (rv); 1439 } 1440 } 1441 1442 return (0); 1443 } 1444 1445 /* 1446 * uvm_fault_unwire(): unwire range of virtual space. 1447 */ 1448 void 1449 uvm_fault_unwire(vm_map_t map, vaddr_t start, vaddr_t end) 1450 { 1451 1452 vm_map_lock_read(map); 1453 uvm_fault_unwire_locked(map, start, end); 1454 vm_map_unlock_read(map); 1455 } 1456 1457 /* 1458 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire(). 1459 * 1460 * => map must be at least read-locked. 1461 */ 1462 void 1463 uvm_fault_unwire_locked(vm_map_t map, vaddr_t start, vaddr_t end) 1464 { 1465 vm_map_entry_t entry, next; 1466 pmap_t pmap = vm_map_pmap(map); 1467 vaddr_t va; 1468 paddr_t pa; 1469 struct vm_page *pg; 1470 1471 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0); 1472 1473 /* 1474 * we assume that the area we are unwiring has actually been wired 1475 * in the first place. this means that we should be able to extract 1476 * the PAs from the pmap. we also lock out the page daemon so that 1477 * we can call uvm_pageunwire. 1478 */ 1479 uvm_lock_pageq(); 1480 1481 /* find the beginning map entry for the region. */ 1482 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map)); 1483 if (uvm_map_lookup_entry(map, start, &entry) == FALSE) 1484 panic("uvm_fault_unwire_locked: address not in map"); 1485 1486 for (va = start; va < end ; va += PAGE_SIZE) { 1487 if (pmap_extract(pmap, va, &pa) == FALSE) 1488 continue; 1489 1490 /* find the map entry for the current address. */ 1491 KASSERT(va >= entry->start); 1492 while (va >= entry->end) { 1493 next = RB_NEXT(uvm_map_addr, &map->addr, entry); 1494 KASSERT(next != NULL && next->start <= entry->end); 1495 entry = next; 1496 } 1497 1498 /* if the entry is no longer wired, tell the pmap. */ 1499 if (VM_MAPENT_ISWIRED(entry) == 0) 1500 pmap_unwire(pmap, va); 1501 1502 pg = PHYS_TO_VM_PAGE(pa); 1503 if (pg) 1504 uvm_pageunwire(pg); 1505 } 1506 1507 uvm_unlock_pageq(); 1508 } 1509 1510 /* 1511 * uvmfault_unlockmaps: unlock the maps 1512 */ 1513 void 1514 uvmfault_unlockmaps(struct uvm_faultinfo *ufi, boolean_t write_locked) 1515 { 1516 /* 1517 * ufi can be NULL when this isn't really a fault, 1518 * but merely paging in anon data. 1519 */ 1520 if (ufi == NULL) { 1521 return; 1522 } 1523 1524 uvmfault_update_stats(ufi); 1525 if (write_locked) { 1526 vm_map_unlock(ufi->map); 1527 } else { 1528 vm_map_unlock_read(ufi->map); 1529 } 1530 } 1531 1532 /* 1533 * uvmfault_unlockall: unlock everything passed in. 1534 * 1535 * => maps must be read-locked (not write-locked). 1536 */ 1537 void 1538 uvmfault_unlockall(struct uvm_faultinfo *ufi, struct vm_amap *amap, 1539 struct uvm_object *uobj, struct vm_anon *anon) 1540 { 1541 1542 uvmfault_unlockmaps(ufi, FALSE); 1543 } 1544 1545 /* 1546 * uvmfault_lookup: lookup a virtual address in a map 1547 * 1548 * => caller must provide a uvm_faultinfo structure with the IN 1549 * params properly filled in 1550 * => we will lookup the map entry (handling submaps) as we go 1551 * => if the lookup is a success we will return with the maps locked 1552 * => if "write_lock" is TRUE, we write_lock the map, otherwise we only 1553 * get a read lock. 1554 * => note that submaps can only appear in the kernel and they are 1555 * required to use the same virtual addresses as the map they 1556 * are referenced by (thus address translation between the main 1557 * map and the submap is unnecessary). 1558 */ 1559 1560 boolean_t 1561 uvmfault_lookup(struct uvm_faultinfo *ufi, boolean_t write_lock) 1562 { 1563 vm_map_t tmpmap; 1564 1565 /* init ufi values for lookup. */ 1566 ufi->map = ufi->orig_map; 1567 ufi->size = ufi->orig_size; 1568 1569 /* 1570 * keep going down levels until we are done. note that there can 1571 * only be two levels so we won't loop very long. 1572 */ 1573 while (1) { 1574 if (ufi->orig_rvaddr < ufi->map->min_offset || 1575 ufi->orig_rvaddr >= ufi->map->max_offset) 1576 return(FALSE); 1577 1578 /* lock map */ 1579 if (write_lock) { 1580 vm_map_lock(ufi->map); 1581 } else { 1582 vm_map_lock_read(ufi->map); 1583 } 1584 1585 /* lookup */ 1586 if (!uvm_map_lookup_entry(ufi->map, ufi->orig_rvaddr, 1587 &ufi->entry)) { 1588 uvmfault_unlockmaps(ufi, write_lock); 1589 return(FALSE); 1590 } 1591 1592 /* reduce size if necessary */ 1593 if (ufi->entry->end - ufi->orig_rvaddr < ufi->size) 1594 ufi->size = ufi->entry->end - ufi->orig_rvaddr; 1595 1596 /* 1597 * submap? replace map with the submap and lookup again. 1598 * note: VAs in submaps must match VAs in main map. 1599 */ 1600 if (UVM_ET_ISSUBMAP(ufi->entry)) { 1601 tmpmap = ufi->entry->object.sub_map; 1602 uvmfault_unlockmaps(ufi, write_lock); 1603 ufi->map = tmpmap; 1604 continue; 1605 } 1606 1607 /* got it! */ 1608 ufi->mapv = ufi->map->timestamp; 1609 return(TRUE); 1610 1611 } 1612 /*NOTREACHED*/ 1613 } 1614 1615 /* 1616 * uvmfault_relock: attempt to relock the same version of the map 1617 * 1618 * => fault data structures should be unlocked before calling. 1619 * => if a success (TRUE) maps will be locked after call. 1620 */ 1621 boolean_t 1622 uvmfault_relock(struct uvm_faultinfo *ufi) 1623 { 1624 /* 1625 * ufi can be NULL when this isn't really a fault, 1626 * but merely paging in anon data. 1627 */ 1628 if (ufi == NULL) { 1629 return TRUE; 1630 } 1631 1632 uvmexp.fltrelck++; 1633 1634 /* 1635 * relock map. fail if version mismatch (in which case nothing 1636 * gets locked). 1637 */ 1638 vm_map_lock_read(ufi->map); 1639 if (ufi->mapv != ufi->map->timestamp) { 1640 vm_map_unlock_read(ufi->map); 1641 return(FALSE); 1642 } 1643 1644 uvmexp.fltrelckok++; 1645 return(TRUE); /* got it! */ 1646 } 1647