1 /* $OpenBSD: uvm_fault.c,v 1.69 2013/05/30 18:02:04 tedu Exp $ */ 2 /* $NetBSD: uvm_fault.c,v 1.51 2000/08/06 00:22:53 thorpej Exp $ */ 3 4 /* 5 * 6 * Copyright (c) 1997 Charles D. Cranor and Washington University. 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed by Charles D. Cranor and 20 * Washington University. 21 * 4. The name of the author may not be used to endorse or promote products 22 * derived from this software without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 25 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 26 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 27 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 28 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 29 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 30 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 31 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 32 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 33 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 34 * 35 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp 36 */ 37 38 /* 39 * uvm_fault.c: fault handler 40 */ 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/kernel.h> 45 #include <sys/proc.h> 46 #include <sys/malloc.h> 47 #include <sys/mman.h> 48 #include <sys/user.h> 49 50 #include <uvm/uvm.h> 51 52 /* 53 * 54 * a word on page faults: 55 * 56 * types of page faults we handle: 57 * 58 * CASE 1: upper layer faults CASE 2: lower layer faults 59 * 60 * CASE 1A CASE 1B CASE 2A CASE 2B 61 * read/write1 write>1 read/write +-cow_write/zero 62 * | | | | 63 * +--|--+ +--|--+ +-----+ + | + | +-----+ 64 * amap | V | | ----------->new| | | | ^ | 65 * +-----+ +-----+ +-----+ + | + | +--|--+ 66 * | | | 67 * +-----+ +-----+ +--|--+ | +--|--+ 68 * uobj | d/c | | d/c | | V | +----| | 69 * +-----+ +-----+ +-----+ +-----+ 70 * 71 * d/c = don't care 72 * 73 * case [0]: layerless fault 74 * no amap or uobj is present. this is an error. 75 * 76 * case [1]: upper layer fault [anon active] 77 * 1A: [read] or [write with anon->an_ref == 1] 78 * I/O takes place in top level anon and uobj is not touched. 79 * 1B: [write with anon->an_ref > 1] 80 * new anon is alloc'd and data is copied off ["COW"] 81 * 82 * case [2]: lower layer fault [uobj] 83 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area] 84 * I/O takes place directly in object. 85 * 2B: [write to copy_on_write] or [read on NULL uobj] 86 * data is "promoted" from uobj to a new anon. 87 * if uobj is null, then we zero fill. 88 * 89 * we follow the standard UVM locking protocol ordering: 90 * 91 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ) 92 * we hold a PG_BUSY page if we unlock for I/O 93 * 94 * 95 * the code is structured as follows: 96 * 97 * - init the "IN" params in the ufi structure 98 * ReFault: 99 * - do lookups [locks maps], check protection, handle needs_copy 100 * - check for case 0 fault (error) 101 * - establish "range" of fault 102 * - if we have an amap lock it and extract the anons 103 * - if sequential advice deactivate pages behind us 104 * - at the same time check pmap for unmapped areas and anon for pages 105 * that we could map in (and do map it if found) 106 * - check object for resident pages that we could map in 107 * - if (case 2) goto Case2 108 * - >>> handle case 1 109 * - ensure source anon is resident in RAM 110 * - if case 1B alloc new anon and copy from source 111 * - map the correct page in 112 * Case2: 113 * - >>> handle case 2 114 * - ensure source page is resident (if uobj) 115 * - if case 2B alloc new anon and copy from source (could be zero 116 * fill if uobj == NULL) 117 * - map the correct page in 118 * - done! 119 * 120 * note on paging: 121 * if we have to do I/O we place a PG_BUSY page in the correct object, 122 * unlock everything, and do the I/O. when I/O is done we must reverify 123 * the state of the world before assuming that our data structures are 124 * valid. [because mappings could change while the map is unlocked] 125 * 126 * alternative 1: unbusy the page in question and restart the page fault 127 * from the top (ReFault). this is easy but does not take advantage 128 * of the information that we already have from our previous lookup, 129 * although it is possible that the "hints" in the vm_map will help here. 130 * 131 * alternative 2: the system already keeps track of a "version" number of 132 * a map. [i.e. every time you write-lock a map (e.g. to change a 133 * mapping) you bump the version number up by one...] so, we can save 134 * the version number of the map before we release the lock and start I/O. 135 * then when I/O is done we can relock and check the version numbers 136 * to see if anything changed. this might save us some over 1 because 137 * we don't have to unbusy the page and may be less compares(?). 138 * 139 * alternative 3: put in backpointers or a way to "hold" part of a map 140 * in place while I/O is in progress. this could be complex to 141 * implement (especially with structures like amap that can be referenced 142 * by multiple map entries, and figuring out what should wait could be 143 * complex as well...). 144 * 145 * given that we are not currently multiprocessor or multithreaded we might 146 * as well choose alternative 2 now. maybe alternative 3 would be useful 147 * in the future. XXX keep in mind for future consideration//rechecking. 148 */ 149 150 /* 151 * local data structures 152 */ 153 154 struct uvm_advice { 155 int advice; 156 int nback; 157 int nforw; 158 }; 159 160 /* 161 * page range array: 162 * note: index in array must match "advice" value 163 * XXX: borrowed numbers from freebsd. do they work well for us? 164 */ 165 166 static struct uvm_advice uvmadvice[] = { 167 { MADV_NORMAL, 3, 4 }, 168 { MADV_RANDOM, 0, 0 }, 169 { MADV_SEQUENTIAL, 8, 7}, 170 }; 171 172 #define UVM_MAXRANGE 16 /* must be max() of nback+nforw+1 */ 173 174 /* 175 * private prototypes 176 */ 177 178 static void uvmfault_amapcopy(struct uvm_faultinfo *); 179 static __inline void uvmfault_anonflush(struct vm_anon **, int); 180 void uvmfault_unlockmaps(struct uvm_faultinfo *, boolean_t); 181 void uvmfault_update_stats(struct uvm_faultinfo *); 182 183 /* 184 * inline functions 185 */ 186 187 /* 188 * uvmfault_anonflush: try and deactivate pages in specified anons 189 * 190 * => does not have to deactivate page if it is busy 191 */ 192 193 static __inline void 194 uvmfault_anonflush(struct vm_anon **anons, int n) 195 { 196 int lcv; 197 struct vm_page *pg; 198 199 for (lcv = 0 ; lcv < n ; lcv++) { 200 if (anons[lcv] == NULL) 201 continue; 202 pg = anons[lcv]->an_page; 203 if (pg && (pg->pg_flags & PG_BUSY) == 0 && pg->loan_count == 0) { 204 uvm_lock_pageq(); 205 if (pg->wire_count == 0) { 206 #ifdef UBC 207 pmap_clear_reference(pg); 208 #else 209 pmap_page_protect(pg, VM_PROT_NONE); 210 #endif 211 uvm_pagedeactivate(pg); 212 } 213 uvm_unlock_pageq(); 214 } 215 } 216 } 217 218 /* 219 * normal functions 220 */ 221 222 /* 223 * uvmfault_amapcopy: clear "needs_copy" in a map. 224 * 225 * => if we are out of RAM we sleep (waiting for more) 226 */ 227 228 static void 229 uvmfault_amapcopy(struct uvm_faultinfo *ufi) 230 { 231 232 /* 233 * while we haven't done the job 234 */ 235 236 while (1) { 237 238 /* 239 * no mapping? give up. 240 */ 241 242 if (uvmfault_lookup(ufi, TRUE) == FALSE) 243 return; 244 245 /* 246 * copy if needed. 247 */ 248 249 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) 250 amap_copy(ufi->map, ufi->entry, M_NOWAIT, TRUE, 251 ufi->orig_rvaddr, ufi->orig_rvaddr + 1); 252 253 /* 254 * didn't work? must be out of RAM. sleep. 255 */ 256 257 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 258 uvmfault_unlockmaps(ufi, TRUE); 259 uvm_wait("fltamapcopy"); 260 continue; 261 } 262 263 /* 264 * got it! 265 */ 266 267 uvmfault_unlockmaps(ufi, TRUE); 268 return; 269 } 270 /*NOTREACHED*/ 271 } 272 273 /* 274 * uvmfault_anonget: get data in an anon into a non-busy, non-released 275 * page in that anon. 276 * 277 * => we don't move the page on the queues [gets moved later] 278 * => if we allocate a new page [we_own], it gets put on the queues. 279 * either way, the result is that the page is on the queues at return time 280 * => for pages which are on loan from a uvm_object (and thus are not 281 * owned by the anon): if successful, we return with the owning object 282 */ 283 284 int 285 uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap, 286 struct vm_anon *anon) 287 { 288 boolean_t we_own; /* we own anon's page? */ 289 boolean_t locked; /* did we relock? */ 290 struct vm_page *pg; 291 int result; 292 293 result = 0; /* XXX shut up gcc */ 294 uvmexp.fltanget++; 295 /* bump rusage counters */ 296 if (anon->an_page) 297 curproc->p_ru.ru_minflt++; 298 else 299 curproc->p_ru.ru_majflt++; 300 301 /* 302 * loop until we get it, or fail. 303 */ 304 305 while (1) { 306 307 we_own = FALSE; /* TRUE if we set PG_BUSY on a page */ 308 pg = anon->an_page; 309 310 /* 311 * if there is a resident page and it is loaned, then anon 312 * may not own it. call out to uvm_anon_lockpage() to ensure 313 * the real owner of the page has been identified. 314 */ 315 316 if (pg && pg->loan_count) 317 pg = uvm_anon_lockloanpg(anon); 318 319 /* 320 * page there? make sure it is not busy/released. 321 */ 322 323 if (pg) { 324 325 /* 326 * at this point, if the page has a uobject [meaning 327 * we have it on loan], then that uobject is locked 328 * by us! if the page is busy, we drop all the 329 * locks (including uobject) and try again. 330 */ 331 332 if ((pg->pg_flags & (PG_BUSY|PG_RELEASED)) == 0) { 333 return (VM_PAGER_OK); 334 } 335 atomic_setbits_int(&pg->pg_flags, PG_WANTED); 336 uvmexp.fltpgwait++; 337 338 /* 339 * the last unlock must be an atomic unlock+wait on 340 * the owner of page 341 */ 342 if (pg->uobject) { /* owner is uobject ? */ 343 uvmfault_unlockall(ufi, amap, NULL, anon); 344 UVM_WAIT(pg, FALSE, "anonget1",0); 345 } else { 346 /* anon owns page */ 347 uvmfault_unlockall(ufi, amap, NULL, NULL); 348 UVM_WAIT(pg, 0, "anonget2", 0); 349 } 350 /* ready to relock and try again */ 351 352 } else { 353 354 /* 355 * no page, we must try and bring it in. 356 */ 357 pg = uvm_pagealloc(NULL, 0, anon, 0); 358 359 if (pg == NULL) { /* out of RAM. */ 360 361 uvmfault_unlockall(ufi, amap, NULL, anon); 362 uvmexp.fltnoram++; 363 uvm_wait("flt_noram1"); 364 /* ready to relock and try again */ 365 366 } else { 367 368 /* we set the PG_BUSY bit */ 369 we_own = TRUE; 370 uvmfault_unlockall(ufi, amap, NULL, anon); 371 372 /* 373 * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN 374 * page into the uvm_swap_get function with 375 * all data structures unlocked. note that 376 * it is ok to read an_swslot here because 377 * we hold PG_BUSY on the page. 378 */ 379 uvmexp.pageins++; 380 result = uvm_swap_get(pg, anon->an_swslot, 381 PGO_SYNCIO); 382 383 /* 384 * we clean up after the i/o below in the 385 * "we_own" case 386 */ 387 /* ready to relock and try again */ 388 } 389 } 390 391 /* 392 * now relock and try again 393 */ 394 395 locked = uvmfault_relock(ufi); 396 397 /* 398 * if we own the page (i.e. we set PG_BUSY), then we need 399 * to clean up after the I/O. there are three cases to 400 * consider: 401 * [1] page released during I/O: free anon and ReFault. 402 * [2] I/O not OK. free the page and cause the fault 403 * to fail. 404 * [3] I/O OK! activate the page and sync with the 405 * non-we_own case (i.e. drop anon lock if not locked). 406 */ 407 408 if (we_own) { 409 410 if (pg->pg_flags & PG_WANTED) { 411 wakeup(pg); 412 } 413 /* un-busy! */ 414 atomic_clearbits_int(&pg->pg_flags, 415 PG_WANTED|PG_BUSY|PG_FAKE); 416 UVM_PAGE_OWN(pg, NULL); 417 418 /* 419 * if we were RELEASED during I/O, then our anon is 420 * no longer part of an amap. we need to free the 421 * anon and try again. 422 */ 423 if (pg->pg_flags & PG_RELEASED) { 424 pmap_page_protect(pg, VM_PROT_NONE); 425 uvm_anfree(anon); /* frees page for us */ 426 if (locked) 427 uvmfault_unlockall(ufi, amap, NULL, 428 NULL); 429 uvmexp.fltpgrele++; 430 return (VM_PAGER_REFAULT); /* refault! */ 431 } 432 433 if (result != VM_PAGER_OK) { 434 KASSERT(result != VM_PAGER_PEND); 435 436 /* remove page from anon */ 437 anon->an_page = NULL; 438 439 /* 440 * remove the swap slot from the anon 441 * and mark the anon as having no real slot. 442 * don't free the swap slot, thus preventing 443 * it from being used again. 444 */ 445 uvm_swap_markbad(anon->an_swslot, 1); 446 anon->an_swslot = SWSLOT_BAD; 447 448 /* 449 * note: page was never !PG_BUSY, so it 450 * can't be mapped and thus no need to 451 * pmap_page_protect it... 452 */ 453 uvm_lock_pageq(); 454 uvm_pagefree(pg); 455 uvm_unlock_pageq(); 456 457 if (locked) 458 uvmfault_unlockall(ufi, amap, NULL, 459 anon); 460 return (VM_PAGER_ERROR); 461 } 462 463 /* 464 * must be OK, clear modify (already PG_CLEAN) 465 * and activate 466 */ 467 pmap_clear_modify(pg); 468 uvm_lock_pageq(); 469 uvm_pageactivate(pg); 470 uvm_unlock_pageq(); 471 } 472 473 /* 474 * we were not able to relock. restart fault. 475 */ 476 477 if (!locked) 478 return (VM_PAGER_REFAULT); 479 480 /* 481 * verify no one has touched the amap and moved the anon on us. 482 */ 483 484 if (ufi != NULL && 485 amap_lookup(&ufi->entry->aref, 486 ufi->orig_rvaddr - ufi->entry->start) != anon) { 487 488 uvmfault_unlockall(ufi, amap, NULL, anon); 489 return (VM_PAGER_REFAULT); 490 } 491 492 /* 493 * try it again! 494 */ 495 496 uvmexp.fltanretry++; 497 continue; 498 499 } /* while (1) */ 500 501 /*NOTREACHED*/ 502 } 503 504 /* 505 * Update statistics after fault resolution. 506 * - maxrss 507 */ 508 void 509 uvmfault_update_stats(struct uvm_faultinfo *ufi) 510 { 511 struct vm_map *map; 512 struct proc *p; 513 vsize_t res; 514 #ifndef pmap_resident_count 515 struct vm_space *vm; 516 #endif 517 518 map = ufi->orig_map; 519 520 /* 521 * Update the maxrss for the process. 522 */ 523 if (map->flags & VM_MAP_ISVMSPACE) { 524 p = curproc; 525 KASSERT(p != NULL && &p->p_vmspace->vm_map == map); 526 527 #ifdef pmap_resident_count 528 res = pmap_resident_count(map->pmap); 529 #else 530 /* 531 * Rather inaccurate, but this is the current anon size 532 * of the vmspace. It's basically the resident size 533 * minus the mmapped in files/text. 534 */ 535 vm = (struct vmspace*)map; 536 res = vm->dsize; 537 #endif 538 539 /* Convert res from pages to kilobytes. */ 540 res <<= (PAGE_SHIFT - 10); 541 542 if (p->p_ru.ru_maxrss < res) 543 p->p_ru.ru_maxrss = res; 544 } 545 } 546 547 /* 548 * F A U L T - m a i n e n t r y p o i n t 549 */ 550 551 /* 552 * uvm_fault: page fault handler 553 * 554 * => called from MD code to resolve a page fault 555 * => VM data structures usually should be unlocked. however, it is 556 * possible to call here with the main map locked if the caller 557 * gets a write lock, sets it recursive, and then calls us (c.f. 558 * uvm_map_pageable). this should be avoided because it keeps 559 * the map locked off during I/O. 560 */ 561 562 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 563 ~VM_PROT_WRITE : VM_PROT_ALL) 564 565 int 566 uvm_fault(vm_map_t orig_map, vaddr_t vaddr, vm_fault_t fault_type, 567 vm_prot_t access_type) 568 { 569 struct uvm_faultinfo ufi; 570 vm_prot_t enter_prot; 571 boolean_t wired, narrow, promote, locked, shadowed; 572 int npages, nback, nforw, centeridx, result, lcv, gotpages; 573 vaddr_t startva, currva; 574 voff_t uoff; 575 paddr_t pa; 576 struct vm_amap *amap; 577 struct uvm_object *uobj; 578 struct vm_anon *anons_store[UVM_MAXRANGE], **anons, *anon, *oanon; 579 struct vm_page *pages[UVM_MAXRANGE], *pg, *uobjpage; 580 581 anon = NULL; 582 pg = NULL; 583 584 uvmexp.faults++; /* XXX: locking? */ 585 586 /* 587 * init the IN parameters in the ufi 588 */ 589 590 ufi.orig_map = orig_map; 591 ufi.orig_rvaddr = trunc_page(vaddr); 592 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ 593 if (fault_type == VM_FAULT_WIRE) 594 narrow = TRUE; /* don't look for neighborhood 595 * pages on wire */ 596 else 597 narrow = FALSE; /* normal fault */ 598 599 /* 600 * "goto ReFault" means restart the page fault from ground zero. 601 */ 602 ReFault: 603 604 /* 605 * lookup and lock the maps 606 */ 607 608 if (uvmfault_lookup(&ufi, FALSE) == FALSE) { 609 return (EFAULT); 610 } 611 612 #ifdef DIAGNOSTIC 613 if ((ufi.map->flags & VM_MAP_PAGEABLE) == 0) 614 panic("uvm_fault: fault on non-pageable map (%p, 0x%lx)", 615 ufi.map, vaddr); 616 #endif 617 618 /* 619 * check protection 620 */ 621 622 if ((ufi.entry->protection & access_type) != access_type) { 623 uvmfault_unlockmaps(&ufi, FALSE); 624 return (EACCES); 625 } 626 627 /* 628 * "enter_prot" is the protection we want to enter the page in at. 629 * for certain pages (e.g. copy-on-write pages) this protection can 630 * be more strict than ufi.entry->protection. "wired" means either 631 * the entry is wired or we are fault-wiring the pg. 632 */ 633 634 enter_prot = ufi.entry->protection; 635 wired = VM_MAPENT_ISWIRED(ufi.entry) || (fault_type == VM_FAULT_WIRE); 636 if (wired) 637 access_type = enter_prot; /* full access for wired */ 638 639 /* 640 * handle "needs_copy" case. 641 */ 642 643 if (UVM_ET_ISNEEDSCOPY(ufi.entry)) { 644 if ((access_type & VM_PROT_WRITE) || 645 (ufi.entry->object.uvm_obj == NULL)) { 646 /* need to clear */ 647 uvmfault_unlockmaps(&ufi, FALSE); 648 uvmfault_amapcopy(&ufi); 649 uvmexp.fltamcopy++; 650 goto ReFault; 651 652 } else { 653 654 /* 655 * ensure that we pmap_enter page R/O since 656 * needs_copy is still true 657 */ 658 enter_prot &= ~VM_PROT_WRITE; 659 660 } 661 } 662 663 /* 664 * identify the players 665 */ 666 667 amap = ufi.entry->aref.ar_amap; /* top layer */ 668 uobj = ufi.entry->object.uvm_obj; /* bottom layer */ 669 670 /* 671 * check for a case 0 fault. if nothing backing the entry then 672 * error now. 673 */ 674 675 if (amap == NULL && uobj == NULL) { 676 uvmfault_unlockmaps(&ufi, FALSE); 677 return (EFAULT); 678 } 679 680 /* 681 * establish range of interest based on advice from mapper 682 * and then clip to fit map entry. note that we only want 683 * to do this the first time through the fault. if we 684 * ReFault we will disable this by setting "narrow" to true. 685 */ 686 687 if (narrow == FALSE) { 688 689 /* wide fault (!narrow) */ 690 KASSERT(uvmadvice[ufi.entry->advice].advice == 691 ufi.entry->advice); 692 nback = min(uvmadvice[ufi.entry->advice].nback, 693 (ufi.orig_rvaddr - ufi.entry->start) >> PAGE_SHIFT); 694 startva = ufi.orig_rvaddr - (nback << PAGE_SHIFT); 695 nforw = min(uvmadvice[ufi.entry->advice].nforw, 696 ((ufi.entry->end - ufi.orig_rvaddr) >> 697 PAGE_SHIFT) - 1); 698 /* 699 * note: "-1" because we don't want to count the 700 * faulting page as forw 701 */ 702 npages = nback + nforw + 1; 703 centeridx = nback; 704 705 narrow = TRUE; /* ensure only once per-fault */ 706 707 } else { 708 709 /* narrow fault! */ 710 nback = nforw = 0; 711 startva = ufi.orig_rvaddr; 712 npages = 1; 713 centeridx = 0; 714 715 } 716 717 /* 718 * if we've got an amap, extract current anons. 719 */ 720 721 if (amap) { 722 anons = anons_store; 723 amap_lookups(&ufi.entry->aref, startva - ufi.entry->start, 724 anons, npages); 725 } else { 726 anons = NULL; /* to be safe */ 727 } 728 729 /* 730 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages 731 * now and then forget about them (for the rest of the fault). 732 */ 733 734 if (ufi.entry->advice == MADV_SEQUENTIAL && nback != 0) { 735 736 /* flush back-page anons? */ 737 if (amap) 738 uvmfault_anonflush(anons, nback); 739 740 /* flush object? */ 741 if (uobj) { 742 uoff = (startva - ufi.entry->start) + ufi.entry->offset; 743 (void) uobj->pgops->pgo_flush(uobj, uoff, uoff + 744 (nback << PAGE_SHIFT), PGO_DEACTIVATE); 745 } 746 747 /* now forget about the backpages */ 748 if (amap) 749 anons += nback; 750 startva += (nback << PAGE_SHIFT); 751 npages -= nback; 752 centeridx = 0; 753 } 754 755 /* 756 * map in the backpages and frontpages we found in the amap in hopes 757 * of preventing future faults. we also init the pages[] array as 758 * we go. 759 */ 760 761 currva = startva; 762 shadowed = FALSE; 763 for (lcv = 0 ; lcv < npages ; lcv++, currva += PAGE_SIZE) { 764 765 /* 766 * dont play with VAs that are already mapped 767 * except for center) 768 */ 769 if (lcv != centeridx && 770 pmap_extract(ufi.orig_map->pmap, currva, &pa)) { 771 pages[lcv] = PGO_DONTCARE; 772 continue; 773 } 774 775 /* 776 * unmapped or center page. check if any anon at this level. 777 */ 778 if (amap == NULL || anons[lcv] == NULL) { 779 pages[lcv] = NULL; 780 continue; 781 } 782 783 /* 784 * check for present page and map if possible. re-activate it. 785 */ 786 787 pages[lcv] = PGO_DONTCARE; 788 if (lcv == centeridx) { /* save center for later! */ 789 shadowed = TRUE; 790 continue; 791 } 792 anon = anons[lcv]; 793 /* ignore loaned pages */ 794 if (anon->an_page && anon->an_page->loan_count == 0 && 795 (anon->an_page->pg_flags & (PG_RELEASED|PG_BUSY)) == 0) { 796 uvm_lock_pageq(); 797 uvm_pageactivate(anon->an_page); /* reactivate */ 798 uvm_unlock_pageq(); 799 uvmexp.fltnamap++; 800 801 /* 802 * Since this isn't the page that's actually faulting, 803 * ignore pmap_enter() failures; it's not critical 804 * that we enter these right now. 805 */ 806 807 (void) pmap_enter(ufi.orig_map->pmap, currva, 808 VM_PAGE_TO_PHYS(anon->an_page), 809 (anon->an_ref > 1) ? (enter_prot & ~VM_PROT_WRITE) : 810 enter_prot, 811 PMAP_CANFAIL | 812 (VM_MAPENT_ISWIRED(ufi.entry) ? PMAP_WIRED : 0)); 813 } 814 pmap_update(ufi.orig_map->pmap); 815 } 816 817 /* (shadowed == TRUE) if there is an anon at the faulting address */ 818 819 /* 820 * note that if we are really short of RAM we could sleep in the above 821 * call to pmap_enter. bad? 822 * 823 * XXX Actually, that is bad; pmap_enter() should just fail in that 824 * XXX case. --thorpej 825 */ 826 827 /* 828 * if the desired page is not shadowed by the amap and we have a 829 * backing object, then we check to see if the backing object would 830 * prefer to handle the fault itself (rather than letting us do it 831 * with the usual pgo_get hook). the backing object signals this by 832 * providing a pgo_fault routine. 833 */ 834 835 if (uobj && shadowed == FALSE && uobj->pgops->pgo_fault != NULL) { 836 result = uobj->pgops->pgo_fault(&ufi, startva, pages, npages, 837 centeridx, fault_type, access_type, 838 PGO_LOCKED); 839 840 if (result == VM_PAGER_OK) 841 return (0); /* pgo_fault did pmap enter */ 842 else if (result == VM_PAGER_REFAULT) 843 goto ReFault; /* try again! */ 844 else 845 return (EACCES); 846 } 847 848 /* 849 * now, if the desired page is not shadowed by the amap and we have 850 * a backing object that does not have a special fault routine, then 851 * we ask (with pgo_get) the object for resident pages that we care 852 * about and attempt to map them in. we do not let pgo_get block 853 * (PGO_LOCKED). 854 * 855 * ("get" has the option of doing a pmap_enter for us) 856 */ 857 858 if (uobj && shadowed == FALSE) { 859 uvmexp.fltlget++; 860 gotpages = npages; 861 (void) uobj->pgops->pgo_get(uobj, ufi.entry->offset + 862 (startva - ufi.entry->start), 863 pages, &gotpages, centeridx, 864 access_type & MASK(ufi.entry), 865 ufi.entry->advice, PGO_LOCKED); 866 867 /* 868 * check for pages to map, if we got any 869 */ 870 871 uobjpage = NULL; 872 873 if (gotpages) { 874 currva = startva; 875 for (lcv = 0 ; lcv < npages ; 876 lcv++, currva += PAGE_SIZE) { 877 878 if (pages[lcv] == NULL || 879 pages[lcv] == PGO_DONTCARE) 880 continue; 881 882 KASSERT((pages[lcv]->pg_flags & PG_RELEASED) == 0); 883 884 /* 885 * if center page is resident and not 886 * PG_BUSY, then pgo_get made it PG_BUSY 887 * for us and gave us a handle to it. 888 * remember this page as "uobjpage." 889 * (for later use). 890 */ 891 892 if (lcv == centeridx) { 893 uobjpage = pages[lcv]; 894 continue; 895 } 896 897 /* 898 * note: calling pgo_get with locked data 899 * structures returns us pages which are 900 * neither busy nor released, so we don't 901 * need to check for this. we can just 902 * directly enter the page (after moving it 903 * to the head of the active queue [useful?]). 904 */ 905 906 uvm_lock_pageq(); 907 uvm_pageactivate(pages[lcv]); /* reactivate */ 908 uvm_unlock_pageq(); 909 uvmexp.fltnomap++; 910 911 /* 912 * Since this page isn't the page that's 913 * actually fauling, ignore pmap_enter() 914 * failures; it's not critical that we 915 * enter these right now. 916 */ 917 918 (void) pmap_enter(ufi.orig_map->pmap, currva, 919 VM_PAGE_TO_PHYS(pages[lcv]), 920 enter_prot & MASK(ufi.entry), 921 PMAP_CANFAIL | 922 (wired ? PMAP_WIRED : 0)); 923 924 /* 925 * NOTE: page can't be PG_WANTED because 926 * we've held the lock the whole time 927 * we've had the handle. 928 */ 929 930 atomic_clearbits_int(&pages[lcv]->pg_flags, 931 PG_BUSY); 932 UVM_PAGE_OWN(pages[lcv], NULL); 933 } /* for "lcv" loop */ 934 pmap_update(ufi.orig_map->pmap); 935 } /* "gotpages" != 0 */ 936 /* note: object still _locked_ */ 937 } else { 938 uobjpage = NULL; 939 } 940 941 /* 942 * note that at this point we are done with any front or back pages. 943 * we are now going to focus on the center page (i.e. the one we've 944 * faulted on). if we have faulted on the top (anon) layer 945 * [i.e. case 1], then the anon we want is anons[centeridx] (we have 946 * not touched it yet). if we have faulted on the bottom (uobj) 947 * layer [i.e. case 2] and the page was both present and available, 948 * then we've got a pointer to it as "uobjpage" and we've already 949 * made it BUSY. 950 */ 951 952 /* 953 * there are four possible cases we must address: 1A, 1B, 2A, and 2B 954 */ 955 956 /* 957 * redirect case 2: if we are not shadowed, go to case 2. 958 */ 959 960 if (shadowed == FALSE) 961 goto Case2; 962 963 /* 964 * handle case 1: fault on an anon in our amap 965 */ 966 967 anon = anons[centeridx]; 968 969 /* 970 * no matter if we have case 1A or case 1B we are going to need to 971 * have the anon's memory resident. ensure that now. 972 */ 973 974 /* 975 * let uvmfault_anonget do the dirty work. 976 * also, if it is OK, then the anon's page is on the queues. 977 */ 978 979 result = uvmfault_anonget(&ufi, amap, anon); 980 switch (result) { 981 case VM_PAGER_OK: 982 break; 983 984 case VM_PAGER_REFAULT: 985 goto ReFault; 986 987 case VM_PAGER_ERROR: 988 /* 989 * An error occured while trying to bring in the 990 * page -- this is the only error we return right 991 * now. 992 */ 993 return (EACCES); /* XXX */ 994 995 default: 996 #ifdef DIAGNOSTIC 997 panic("uvm_fault: uvmfault_anonget -> %d", result); 998 #else 999 return (EACCES); 1000 #endif 1001 } 1002 1003 /* 1004 * uobj is non null if the page is on loan from an object (i.e. uobj) 1005 */ 1006 1007 uobj = anon->an_page->uobject; 1008 1009 /* 1010 * special handling for loaned pages 1011 */ 1012 1013 if (anon->an_page->loan_count) { 1014 1015 if ((access_type & VM_PROT_WRITE) == 0) { 1016 1017 /* 1018 * for read faults on loaned pages we just cap the 1019 * protection at read-only. 1020 */ 1021 1022 enter_prot = enter_prot & ~VM_PROT_WRITE; 1023 1024 } else { 1025 /* 1026 * note that we can't allow writes into a loaned page! 1027 * 1028 * if we have a write fault on a loaned page in an 1029 * anon then we need to look at the anon's ref count. 1030 * if it is greater than one then we are going to do 1031 * a normal copy-on-write fault into a new anon (this 1032 * is not a problem). however, if the reference count 1033 * is one (a case where we would normally allow a 1034 * write directly to the page) then we need to kill 1035 * the loan before we continue. 1036 */ 1037 1038 /* >1 case is already ok */ 1039 if (anon->an_ref == 1) { 1040 1041 /* get new un-owned replacement page */ 1042 pg = uvm_pagealloc(NULL, 0, NULL, 0); 1043 if (pg == NULL) { 1044 uvmfault_unlockall(&ufi, amap, uobj, 1045 anon); 1046 uvm_wait("flt_noram2"); 1047 goto ReFault; 1048 } 1049 1050 /* 1051 * copy data, kill loan 1052 */ 1053 /* copy old -> new */ 1054 uvm_pagecopy(anon->an_page, pg); 1055 1056 /* force reload */ 1057 pmap_page_protect(anon->an_page, 1058 VM_PROT_NONE); 1059 uvm_lock_pageq(); /* KILL loan */ 1060 if (uobj) 1061 /* if we were loaning */ 1062 anon->an_page->loan_count--; 1063 anon->an_page->uanon = NULL; 1064 /* in case we owned */ 1065 atomic_clearbits_int( 1066 &anon->an_page->pg_flags, PQ_ANON); 1067 uvm_pageactivate(pg); 1068 uvm_unlock_pageq(); 1069 if (uobj) { 1070 uobj = NULL; 1071 } 1072 1073 /* install new page in anon */ 1074 anon->an_page = pg; 1075 pg->uanon = anon; 1076 atomic_setbits_int(&pg->pg_flags, PQ_ANON); 1077 atomic_clearbits_int(&pg->pg_flags, 1078 PG_BUSY|PG_FAKE); 1079 UVM_PAGE_OWN(pg, NULL); 1080 1081 /* done! */ 1082 } /* ref == 1 */ 1083 } /* write fault */ 1084 } /* loan count */ 1085 1086 /* 1087 * if we are case 1B then we will need to allocate a new blank 1088 * anon to transfer the data into. note that we have a lock 1089 * on anon, so no one can busy or release the page until we are done. 1090 * also note that the ref count can't drop to zero here because 1091 * it is > 1 and we are only dropping one ref. 1092 * 1093 * in the (hopefully very rare) case that we are out of RAM we 1094 * will wait for more RAM, and refault. 1095 * 1096 * if we are out of anon VM we kill the process (XXX: could wait?). 1097 */ 1098 1099 if ((access_type & VM_PROT_WRITE) != 0 && anon->an_ref > 1) { 1100 uvmexp.flt_acow++; 1101 oanon = anon; /* oanon = old */ 1102 anon = uvm_analloc(); 1103 if (anon) { 1104 pg = uvm_pagealloc(NULL, 0, anon, 0); 1105 } 1106 1107 /* check for out of RAM */ 1108 if (anon == NULL || pg == NULL) { 1109 if (anon) 1110 uvm_anfree(anon); 1111 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1112 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1113 if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) { 1114 uvmexp.fltnoanon++; 1115 return (ENOMEM); 1116 } 1117 1118 uvmexp.fltnoram++; 1119 uvm_wait("flt_noram3"); /* out of RAM, wait for more */ 1120 goto ReFault; 1121 } 1122 1123 /* got all resources, replace anon with nanon */ 1124 1125 uvm_pagecopy(oanon->an_page, pg); /* pg now !PG_CLEAN */ 1126 /* un-busy! new page */ 1127 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE); 1128 UVM_PAGE_OWN(pg, NULL); 1129 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start, 1130 anon, 1); 1131 1132 /* deref: can not drop to zero here by defn! */ 1133 oanon->an_ref--; 1134 1135 /* 1136 * note: anon is _not_ locked, but we have the sole references 1137 * to in from amap. 1138 * thus, no one can get at it until we are done with it. 1139 */ 1140 1141 } else { 1142 1143 uvmexp.flt_anon++; 1144 oanon = anon; 1145 pg = anon->an_page; 1146 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */ 1147 enter_prot = enter_prot & ~VM_PROT_WRITE; 1148 1149 } 1150 1151 /* 1152 * now map the page in ... 1153 * XXX: old fault unlocks object before pmap_enter. this seems 1154 * suspect since some other thread could blast the page out from 1155 * under us between the unlock and the pmap_enter. 1156 */ 1157 1158 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1159 enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) 1160 != 0) { 1161 /* 1162 * No need to undo what we did; we can simply think of 1163 * this as the pmap throwing away the mapping information. 1164 * 1165 * We do, however, have to go through the ReFault path, 1166 * as the map may change while we're asleep. 1167 */ 1168 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1169 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1170 if (uvmexp.swpgonly == uvmexp.swpages) { 1171 /* XXX instrumentation */ 1172 return (ENOMEM); 1173 } 1174 /* XXX instrumentation */ 1175 uvm_wait("flt_pmfail1"); 1176 goto ReFault; 1177 } 1178 1179 /* 1180 * ... update the page queues. 1181 */ 1182 1183 uvm_lock_pageq(); 1184 1185 if (fault_type == VM_FAULT_WIRE) { 1186 uvm_pagewire(pg); 1187 1188 /* 1189 * since the now-wired page cannot be paged out, 1190 * release its swap resources for others to use. 1191 * since an anon with no swap cannot be PG_CLEAN, 1192 * clear its clean flag now. 1193 */ 1194 atomic_clearbits_int(&pg->pg_flags, PG_CLEAN); 1195 uvm_anon_dropswap(anon); 1196 } else { 1197 /* activate it */ 1198 uvm_pageactivate(pg); 1199 } 1200 1201 uvm_unlock_pageq(); 1202 1203 /* 1204 * done case 1! finish up by unlocking everything and returning success 1205 */ 1206 1207 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1208 pmap_update(ufi.orig_map->pmap); 1209 return (0); 1210 1211 1212 Case2: 1213 /* 1214 * handle case 2: faulting on backing object or zero fill 1215 */ 1216 1217 /* 1218 * note that uobjpage can not be PGO_DONTCARE at this point. we now 1219 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we 1220 * have a backing object, check and see if we are going to promote 1221 * the data up to an anon during the fault. 1222 */ 1223 1224 if (uobj == NULL) { 1225 uobjpage = PGO_DONTCARE; 1226 promote = TRUE; /* always need anon here */ 1227 } else { 1228 KASSERT(uobjpage != PGO_DONTCARE); 1229 promote = (access_type & VM_PROT_WRITE) && 1230 UVM_ET_ISCOPYONWRITE(ufi.entry); 1231 } 1232 1233 /* 1234 * if uobjpage is not null then we do not need to do I/O to get the 1235 * uobjpage. 1236 * 1237 * if uobjpage is null, then we need to ask the pager to 1238 * get the data for us. once we have the data, we need to reverify 1239 * the state the world. we are currently not holding any resources. 1240 */ 1241 1242 if (uobjpage) { 1243 /* update rusage counters */ 1244 curproc->p_ru.ru_minflt++; 1245 } else { 1246 /* update rusage counters */ 1247 curproc->p_ru.ru_majflt++; 1248 1249 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1250 1251 uvmexp.fltget++; 1252 gotpages = 1; 1253 uoff = (ufi.orig_rvaddr - ufi.entry->start) + ufi.entry->offset; 1254 result = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages, 1255 0, access_type & MASK(ufi.entry), ufi.entry->advice, 1256 PGO_SYNCIO); 1257 1258 /* 1259 * recover from I/O 1260 */ 1261 1262 if (result != VM_PAGER_OK) { 1263 KASSERT(result != VM_PAGER_PEND); 1264 1265 if (result == VM_PAGER_AGAIN) { 1266 tsleep(&lbolt, PVM, "fltagain2", 0); 1267 goto ReFault; 1268 } 1269 1270 return (EACCES); /* XXX i/o error */ 1271 } 1272 1273 /* 1274 * re-verify the state of the world. 1275 */ 1276 1277 locked = uvmfault_relock(&ufi); 1278 1279 /* 1280 * Re-verify that amap slot is still free. if there is 1281 * a problem, we clean up. 1282 */ 1283 1284 if (locked && amap && amap_lookup(&ufi.entry->aref, 1285 ufi.orig_rvaddr - ufi.entry->start)) { 1286 if (locked) 1287 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1288 locked = FALSE; 1289 } 1290 1291 /* 1292 * didn't get the lock? release the page and retry. 1293 */ 1294 1295 if (locked == FALSE) { 1296 if (uobjpage->pg_flags & PG_WANTED) 1297 /* still holding object lock */ 1298 wakeup(uobjpage); 1299 1300 uvm_lock_pageq(); 1301 /* make sure it is in queues */ 1302 uvm_pageactivate(uobjpage); 1303 1304 uvm_unlock_pageq(); 1305 atomic_clearbits_int(&uobjpage->pg_flags, 1306 PG_BUSY|PG_WANTED); 1307 UVM_PAGE_OWN(uobjpage, NULL); 1308 goto ReFault; 1309 1310 } 1311 1312 /* 1313 * we have the data in uobjpage which is PG_BUSY 1314 */ 1315 1316 } 1317 1318 /* 1319 * notes: 1320 * - at this point uobjpage can not be NULL 1321 * - at this point uobjpage could be PG_WANTED (handle later) 1322 */ 1323 1324 if (promote == FALSE) { 1325 1326 /* 1327 * we are not promoting. if the mapping is COW ensure that we 1328 * don't give more access than we should (e.g. when doing a read 1329 * fault on a COPYONWRITE mapping we want to map the COW page in 1330 * R/O even though the entry protection could be R/W). 1331 * 1332 * set "pg" to the page we want to map in (uobjpage, usually) 1333 */ 1334 1335 uvmexp.flt_obj++; 1336 if (UVM_ET_ISCOPYONWRITE(ufi.entry)) 1337 enter_prot &= ~VM_PROT_WRITE; 1338 pg = uobjpage; /* map in the actual object */ 1339 1340 /* assert(uobjpage != PGO_DONTCARE) */ 1341 1342 /* 1343 * we are faulting directly on the page. be careful 1344 * about writing to loaned pages... 1345 */ 1346 if (uobjpage->loan_count) { 1347 1348 if ((access_type & VM_PROT_WRITE) == 0) { 1349 /* read fault: cap the protection at readonly */ 1350 /* cap! */ 1351 enter_prot = enter_prot & ~VM_PROT_WRITE; 1352 } else { 1353 /* write fault: must break the loan here */ 1354 1355 /* alloc new un-owned page */ 1356 pg = uvm_pagealloc(NULL, 0, NULL, 0); 1357 1358 if (pg == NULL) { 1359 /* 1360 * drop ownership of page, it can't 1361 * be released 1362 */ 1363 if (uobjpage->pg_flags & PG_WANTED) 1364 wakeup(uobjpage); 1365 atomic_clearbits_int( 1366 &uobjpage->pg_flags, 1367 PG_BUSY|PG_WANTED); 1368 UVM_PAGE_OWN(uobjpage, NULL); 1369 1370 uvm_lock_pageq(); 1371 /* activate: we will need it later */ 1372 uvm_pageactivate(uobjpage); 1373 1374 uvm_unlock_pageq(); 1375 uvmfault_unlockall(&ufi, amap, uobj, 1376 NULL); 1377 uvmexp.fltnoram++; 1378 uvm_wait("flt_noram4"); 1379 goto ReFault; 1380 } 1381 1382 /* 1383 * copy the data from the old page to the new 1384 * one and clear the fake/clean flags on the 1385 * new page (keep it busy). force a reload 1386 * of the old page by clearing it from all 1387 * pmaps. then lock the page queues to 1388 * rename the pages. 1389 */ 1390 uvm_pagecopy(uobjpage, pg); /* old -> new */ 1391 atomic_clearbits_int(&pg->pg_flags, 1392 PG_FAKE|PG_CLEAN); 1393 pmap_page_protect(uobjpage, VM_PROT_NONE); 1394 if (uobjpage->pg_flags & PG_WANTED) 1395 wakeup(uobjpage); 1396 atomic_clearbits_int(&uobjpage->pg_flags, 1397 PG_BUSY|PG_WANTED); 1398 UVM_PAGE_OWN(uobjpage, NULL); 1399 1400 uvm_lock_pageq(); 1401 uoff = uobjpage->offset; 1402 /* remove old page */ 1403 uvm_pagerealloc(uobjpage, NULL, 0); 1404 1405 /* 1406 * at this point we have absolutely no 1407 * control over uobjpage 1408 */ 1409 /* install new page */ 1410 uvm_pagerealloc(pg, uobj, uoff); 1411 uvm_unlock_pageq(); 1412 1413 /* 1414 * done! loan is broken and "pg" is 1415 * PG_BUSY. it can now replace uobjpage. 1416 */ 1417 1418 uobjpage = pg; 1419 1420 } /* write fault case */ 1421 } /* if loan_count */ 1422 1423 } else { 1424 1425 /* 1426 * if we are going to promote the data to an anon we 1427 * allocate a blank anon here and plug it into our amap. 1428 */ 1429 #ifdef DIAGNOSTIC 1430 if (amap == NULL) 1431 panic("uvm_fault: want to promote data, but no anon"); 1432 #endif 1433 1434 anon = uvm_analloc(); 1435 if (anon) { 1436 /* 1437 * In `Fill in data...' below, if 1438 * uobjpage == PGO_DONTCARE, we want 1439 * a zero'd, dirty page, so have 1440 * uvm_pagealloc() do that for us. 1441 */ 1442 pg = uvm_pagealloc(NULL, 0, anon, 1443 (uobjpage == PGO_DONTCARE) ? UVM_PGA_ZERO : 0); 1444 } 1445 1446 /* 1447 * out of memory resources? 1448 */ 1449 if (anon == NULL || pg == NULL) { 1450 1451 /* 1452 * arg! must unbusy our page and fail or sleep. 1453 */ 1454 if (uobjpage != PGO_DONTCARE) { 1455 if (uobjpage->pg_flags & PG_WANTED) 1456 wakeup(uobjpage); 1457 1458 uvm_lock_pageq(); 1459 uvm_pageactivate(uobjpage); 1460 uvm_unlock_pageq(); 1461 atomic_clearbits_int(&uobjpage->pg_flags, 1462 PG_BUSY|PG_WANTED); 1463 UVM_PAGE_OWN(uobjpage, NULL); 1464 } 1465 1466 /* unlock and fail ... */ 1467 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1468 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1469 if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) { 1470 uvmexp.fltnoanon++; 1471 return (ENOMEM); 1472 } 1473 1474 uvm_anfree(anon); 1475 uvmexp.fltnoram++; 1476 uvm_wait("flt_noram5"); 1477 goto ReFault; 1478 } 1479 1480 /* 1481 * fill in the data 1482 */ 1483 1484 if (uobjpage != PGO_DONTCARE) { 1485 uvmexp.flt_prcopy++; 1486 /* copy page [pg now dirty] */ 1487 uvm_pagecopy(uobjpage, pg); 1488 1489 /* 1490 * promote to shared amap? make sure all sharing 1491 * procs see it 1492 */ 1493 if ((amap_flags(amap) & AMAP_SHARED) != 0) { 1494 pmap_page_protect(uobjpage, VM_PROT_NONE); 1495 } 1496 1497 /* 1498 * dispose of uobjpage. drop handle to uobj as well. 1499 */ 1500 1501 if (uobjpage->pg_flags & PG_WANTED) 1502 wakeup(uobjpage); 1503 atomic_clearbits_int(&uobjpage->pg_flags, 1504 PG_BUSY|PG_WANTED); 1505 UVM_PAGE_OWN(uobjpage, NULL); 1506 uvm_lock_pageq(); 1507 uvm_pageactivate(uobjpage); 1508 uvm_unlock_pageq(); 1509 uobj = NULL; 1510 } else { 1511 uvmexp.flt_przero++; 1512 /* 1513 * Page is zero'd and marked dirty by uvm_pagealloc() 1514 * above. 1515 */ 1516 } 1517 1518 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start, 1519 anon, 0); 1520 } 1521 1522 /* 1523 * note: pg is either the uobjpage or the new page in the new anon 1524 */ 1525 1526 /* 1527 * all resources are present. we can now map it in and free our 1528 * resources. 1529 */ 1530 1531 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1532 enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) 1533 != 0) { 1534 1535 /* 1536 * No need to undo what we did; we can simply think of 1537 * this as the pmap throwing away the mapping information. 1538 * 1539 * We do, however, have to go through the ReFault path, 1540 * as the map may change while we're asleep. 1541 */ 1542 1543 if (pg->pg_flags & PG_WANTED) 1544 wakeup(pg); 1545 1546 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED); 1547 UVM_PAGE_OWN(pg, NULL); 1548 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1549 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1550 if (uvmexp.swpgonly == uvmexp.swpages) { 1551 /* XXX instrumentation */ 1552 return (ENOMEM); 1553 } 1554 /* XXX instrumentation */ 1555 uvm_wait("flt_pmfail2"); 1556 goto ReFault; 1557 } 1558 1559 uvm_lock_pageq(); 1560 1561 if (fault_type == VM_FAULT_WIRE) { 1562 uvm_pagewire(pg); 1563 if (pg->pg_flags & PQ_AOBJ) { 1564 1565 /* 1566 * since the now-wired page cannot be paged out, 1567 * release its swap resources for others to use. 1568 * since an aobj page with no swap cannot be PG_CLEAN, 1569 * clear its clean flag now. 1570 */ 1571 atomic_clearbits_int(&pg->pg_flags, PG_CLEAN); 1572 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT); 1573 } 1574 } else { 1575 /* activate it */ 1576 uvm_pageactivate(pg); 1577 } 1578 uvm_unlock_pageq(); 1579 1580 if (pg->pg_flags & PG_WANTED) 1581 wakeup(pg); 1582 1583 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED); 1584 UVM_PAGE_OWN(pg, NULL); 1585 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1586 pmap_update(ufi.orig_map->pmap); 1587 1588 return (0); 1589 } 1590 1591 1592 /* 1593 * uvm_fault_wire: wire down a range of virtual addresses in a map. 1594 * 1595 * => map may be read-locked by caller, but MUST NOT be write-locked. 1596 * => if map is read-locked, any operations which may cause map to 1597 * be write-locked in uvm_fault() must be taken care of by 1598 * the caller. See uvm_map_pageable(). 1599 */ 1600 1601 int 1602 uvm_fault_wire(vm_map_t map, vaddr_t start, vaddr_t end, vm_prot_t access_type) 1603 { 1604 vaddr_t va; 1605 pmap_t pmap; 1606 int rv; 1607 1608 pmap = vm_map_pmap(map); 1609 1610 /* 1611 * now fault it in a page at a time. if the fault fails then we have 1612 * to undo what we have done. note that in uvm_fault VM_PROT_NONE 1613 * is replaced with the max protection if fault_type is VM_FAULT_WIRE. 1614 */ 1615 1616 for (va = start ; va < end ; va += PAGE_SIZE) { 1617 rv = uvm_fault(map, va, VM_FAULT_WIRE, access_type); 1618 if (rv) { 1619 if (va != start) { 1620 uvm_fault_unwire(map, start, va); 1621 } 1622 return (rv); 1623 } 1624 } 1625 1626 return (0); 1627 } 1628 1629 /* 1630 * uvm_fault_unwire(): unwire range of virtual space. 1631 */ 1632 1633 void 1634 uvm_fault_unwire(vm_map_t map, vaddr_t start, vaddr_t end) 1635 { 1636 1637 vm_map_lock_read(map); 1638 uvm_fault_unwire_locked(map, start, end); 1639 vm_map_unlock_read(map); 1640 } 1641 1642 /* 1643 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire(). 1644 * 1645 * => map must be at least read-locked. 1646 */ 1647 1648 void 1649 uvm_fault_unwire_locked(vm_map_t map, vaddr_t start, vaddr_t end) 1650 { 1651 vm_map_entry_t entry, next; 1652 pmap_t pmap = vm_map_pmap(map); 1653 vaddr_t va; 1654 paddr_t pa; 1655 struct vm_page *pg; 1656 1657 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0); 1658 1659 /* 1660 * we assume that the area we are unwiring has actually been wired 1661 * in the first place. this means that we should be able to extract 1662 * the PAs from the pmap. we also lock out the page daemon so that 1663 * we can call uvm_pageunwire. 1664 */ 1665 1666 uvm_lock_pageq(); 1667 1668 /* 1669 * find the beginning map entry for the region. 1670 */ 1671 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map)); 1672 if (uvm_map_lookup_entry(map, start, &entry) == FALSE) 1673 panic("uvm_fault_unwire_locked: address not in map"); 1674 1675 for (va = start; va < end ; va += PAGE_SIZE) { 1676 if (pmap_extract(pmap, va, &pa) == FALSE) 1677 continue; 1678 1679 /* 1680 * find the map entry for the current address. 1681 */ 1682 KASSERT(va >= entry->start); 1683 while (va >= entry->end) { 1684 next = RB_NEXT(uvm_map_addr, &map->addr, entry); 1685 KASSERT(next != NULL && next->start <= entry->end); 1686 entry = next; 1687 } 1688 1689 /* 1690 * if the entry is no longer wired, tell the pmap. 1691 */ 1692 if (VM_MAPENT_ISWIRED(entry) == 0) 1693 pmap_unwire(pmap, va); 1694 1695 pg = PHYS_TO_VM_PAGE(pa); 1696 if (pg) 1697 uvm_pageunwire(pg); 1698 } 1699 1700 uvm_unlock_pageq(); 1701 } 1702 1703 /* 1704 * uvmfault_unlockmaps: unlock the maps 1705 */ 1706 void 1707 uvmfault_unlockmaps(struct uvm_faultinfo *ufi, boolean_t write_locked) 1708 { 1709 /* 1710 * ufi can be NULL when this isn't really a fault, 1711 * but merely paging in anon data. 1712 */ 1713 1714 if (ufi == NULL) { 1715 return; 1716 } 1717 1718 uvmfault_update_stats(ufi); 1719 if (write_locked) { 1720 vm_map_unlock(ufi->map); 1721 } else { 1722 vm_map_unlock_read(ufi->map); 1723 } 1724 } 1725 1726 /* 1727 * uvmfault_unlockall: unlock everything passed in. 1728 * 1729 * => maps must be read-locked (not write-locked). 1730 */ 1731 void 1732 uvmfault_unlockall(struct uvm_faultinfo *ufi, struct vm_amap *amap, 1733 struct uvm_object *uobj, struct vm_anon *anon) 1734 { 1735 1736 uvmfault_unlockmaps(ufi, FALSE); 1737 } 1738 1739 /* 1740 * uvmfault_lookup: lookup a virtual address in a map 1741 * 1742 * => caller must provide a uvm_faultinfo structure with the IN 1743 * params properly filled in 1744 * => we will lookup the map entry (handling submaps) as we go 1745 * => if the lookup is a success we will return with the maps locked 1746 * => if "write_lock" is TRUE, we write_lock the map, otherwise we only 1747 * get a read lock. 1748 * => note that submaps can only appear in the kernel and they are 1749 * required to use the same virtual addresses as the map they 1750 * are referenced by (thus address translation between the main 1751 * map and the submap is unnecessary). 1752 */ 1753 1754 boolean_t 1755 uvmfault_lookup(struct uvm_faultinfo *ufi, boolean_t write_lock) 1756 { 1757 vm_map_t tmpmap; 1758 1759 /* 1760 * init ufi values for lookup. 1761 */ 1762 1763 ufi->map = ufi->orig_map; 1764 ufi->size = ufi->orig_size; 1765 1766 /* 1767 * keep going down levels until we are done. note that there can 1768 * only be two levels so we won't loop very long. 1769 */ 1770 1771 while (1) { 1772 if (ufi->orig_rvaddr < ufi->map->min_offset || 1773 ufi->orig_rvaddr >= ufi->map->max_offset) 1774 return(FALSE); 1775 1776 /* 1777 * lock map 1778 */ 1779 if (write_lock) { 1780 vm_map_lock(ufi->map); 1781 } else { 1782 vm_map_lock_read(ufi->map); 1783 } 1784 1785 /* 1786 * lookup 1787 */ 1788 if (!uvm_map_lookup_entry(ufi->map, ufi->orig_rvaddr, 1789 &ufi->entry)) { 1790 uvmfault_unlockmaps(ufi, write_lock); 1791 return(FALSE); 1792 } 1793 1794 /* 1795 * reduce size if necessary 1796 */ 1797 if (ufi->entry->end - ufi->orig_rvaddr < ufi->size) 1798 ufi->size = ufi->entry->end - ufi->orig_rvaddr; 1799 1800 /* 1801 * submap? replace map with the submap and lookup again. 1802 * note: VAs in submaps must match VAs in main map. 1803 */ 1804 if (UVM_ET_ISSUBMAP(ufi->entry)) { 1805 tmpmap = ufi->entry->object.sub_map; 1806 uvmfault_unlockmaps(ufi, write_lock); 1807 ufi->map = tmpmap; 1808 continue; 1809 } 1810 1811 /* 1812 * got it! 1813 */ 1814 1815 ufi->mapv = ufi->map->timestamp; 1816 return(TRUE); 1817 1818 } /* while loop */ 1819 1820 /*NOTREACHED*/ 1821 } 1822 1823 /* 1824 * uvmfault_relock: attempt to relock the same version of the map 1825 * 1826 * => fault data structures should be unlocked before calling. 1827 * => if a success (TRUE) maps will be locked after call. 1828 */ 1829 boolean_t 1830 uvmfault_relock(struct uvm_faultinfo *ufi) 1831 { 1832 /* 1833 * ufi can be NULL when this isn't really a fault, 1834 * but merely paging in anon data. 1835 */ 1836 1837 if (ufi == NULL) { 1838 return TRUE; 1839 } 1840 1841 uvmexp.fltrelck++; 1842 1843 /* 1844 * relock map. fail if version mismatch (in which case nothing 1845 * gets locked). 1846 */ 1847 1848 vm_map_lock_read(ufi->map); 1849 if (ufi->mapv != ufi->map->timestamp) { 1850 vm_map_unlock_read(ufi->map); 1851 return(FALSE); 1852 } 1853 1854 uvmexp.fltrelckok++; 1855 return(TRUE); /* got it! */ 1856 } 1857