1 /* $OpenBSD: uvm_fault.c,v 1.65 2012/04/12 11:55:43 ariane Exp $ */ 2 /* $NetBSD: uvm_fault.c,v 1.51 2000/08/06 00:22:53 thorpej Exp $ */ 3 4 /* 5 * 6 * Copyright (c) 1997 Charles D. Cranor and Washington University. 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed by Charles D. Cranor and 20 * Washington University. 21 * 4. The name of the author may not be used to endorse or promote products 22 * derived from this software without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 25 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 26 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 27 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 28 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 29 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 30 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 31 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 32 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 33 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 34 * 35 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp 36 */ 37 38 /* 39 * uvm_fault.c: fault handler 40 */ 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/kernel.h> 45 #include <sys/proc.h> 46 #include <sys/malloc.h> 47 #include <sys/mman.h> 48 #include <sys/user.h> 49 50 #include <uvm/uvm.h> 51 52 /* 53 * 54 * a word on page faults: 55 * 56 * types of page faults we handle: 57 * 58 * CASE 1: upper layer faults CASE 2: lower layer faults 59 * 60 * CASE 1A CASE 1B CASE 2A CASE 2B 61 * read/write1 write>1 read/write +-cow_write/zero 62 * | | | | 63 * +--|--+ +--|--+ +-----+ + | + | +-----+ 64 * amap | V | | ----------->new| | | | ^ | 65 * +-----+ +-----+ +-----+ + | + | +--|--+ 66 * | | | 67 * +-----+ +-----+ +--|--+ | +--|--+ 68 * uobj | d/c | | d/c | | V | +----| | 69 * +-----+ +-----+ +-----+ +-----+ 70 * 71 * d/c = don't care 72 * 73 * case [0]: layerless fault 74 * no amap or uobj is present. this is an error. 75 * 76 * case [1]: upper layer fault [anon active] 77 * 1A: [read] or [write with anon->an_ref == 1] 78 * I/O takes place in top level anon and uobj is not touched. 79 * 1B: [write with anon->an_ref > 1] 80 * new anon is alloc'd and data is copied off ["COW"] 81 * 82 * case [2]: lower layer fault [uobj] 83 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area] 84 * I/O takes place directly in object. 85 * 2B: [write to copy_on_write] or [read on NULL uobj] 86 * data is "promoted" from uobj to a new anon. 87 * if uobj is null, then we zero fill. 88 * 89 * we follow the standard UVM locking protocol ordering: 90 * 91 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ) 92 * we hold a PG_BUSY page if we unlock for I/O 93 * 94 * 95 * the code is structured as follows: 96 * 97 * - init the "IN" params in the ufi structure 98 * ReFault: 99 * - do lookups [locks maps], check protection, handle needs_copy 100 * - check for case 0 fault (error) 101 * - establish "range" of fault 102 * - if we have an amap lock it and extract the anons 103 * - if sequential advice deactivate pages behind us 104 * - at the same time check pmap for unmapped areas and anon for pages 105 * that we could map in (and do map it if found) 106 * - check object for resident pages that we could map in 107 * - if (case 2) goto Case2 108 * - >>> handle case 1 109 * - ensure source anon is resident in RAM 110 * - if case 1B alloc new anon and copy from source 111 * - map the correct page in 112 * Case2: 113 * - >>> handle case 2 114 * - ensure source page is resident (if uobj) 115 * - if case 2B alloc new anon and copy from source (could be zero 116 * fill if uobj == NULL) 117 * - map the correct page in 118 * - done! 119 * 120 * note on paging: 121 * if we have to do I/O we place a PG_BUSY page in the correct object, 122 * unlock everything, and do the I/O. when I/O is done we must reverify 123 * the state of the world before assuming that our data structures are 124 * valid. [because mappings could change while the map is unlocked] 125 * 126 * alternative 1: unbusy the page in question and restart the page fault 127 * from the top (ReFault). this is easy but does not take advantage 128 * of the information that we already have from our previous lookup, 129 * although it is possible that the "hints" in the vm_map will help here. 130 * 131 * alternative 2: the system already keeps track of a "version" number of 132 * a map. [i.e. every time you write-lock a map (e.g. to change a 133 * mapping) you bump the version number up by one...] so, we can save 134 * the version number of the map before we release the lock and start I/O. 135 * then when I/O is done we can relock and check the version numbers 136 * to see if anything changed. this might save us some over 1 because 137 * we don't have to unbusy the page and may be less compares(?). 138 * 139 * alternative 3: put in backpointers or a way to "hold" part of a map 140 * in place while I/O is in progress. this could be complex to 141 * implement (especially with structures like amap that can be referenced 142 * by multiple map entries, and figuring out what should wait could be 143 * complex as well...). 144 * 145 * given that we are not currently multiprocessor or multithreaded we might 146 * as well choose alternative 2 now. maybe alternative 3 would be useful 147 * in the future. XXX keep in mind for future consideration//rechecking. 148 */ 149 150 /* 151 * local data structures 152 */ 153 154 struct uvm_advice { 155 int advice; 156 int nback; 157 int nforw; 158 }; 159 160 /* 161 * page range array: 162 * note: index in array must match "advice" value 163 * XXX: borrowed numbers from freebsd. do they work well for us? 164 */ 165 166 static struct uvm_advice uvmadvice[] = { 167 { MADV_NORMAL, 3, 4 }, 168 { MADV_RANDOM, 0, 0 }, 169 { MADV_SEQUENTIAL, 8, 7}, 170 }; 171 172 #define UVM_MAXRANGE 16 /* must be max() of nback+nforw+1 */ 173 174 /* 175 * private prototypes 176 */ 177 178 static void uvmfault_amapcopy(struct uvm_faultinfo *); 179 static __inline void uvmfault_anonflush(struct vm_anon **, int); 180 void uvmfault_unlockmaps(struct uvm_faultinfo *, boolean_t); 181 void uvmfault_update_stats(struct uvm_faultinfo *); 182 183 /* 184 * inline functions 185 */ 186 187 /* 188 * uvmfault_anonflush: try and deactivate pages in specified anons 189 * 190 * => does not have to deactivate page if it is busy 191 */ 192 193 static __inline void 194 uvmfault_anonflush(struct vm_anon **anons, int n) 195 { 196 int lcv; 197 struct vm_page *pg; 198 199 for (lcv = 0 ; lcv < n ; lcv++) { 200 if (anons[lcv] == NULL) 201 continue; 202 simple_lock(&anons[lcv]->an_lock); 203 pg = anons[lcv]->an_page; 204 if (pg && (pg->pg_flags & PG_BUSY) == 0 && pg->loan_count == 0) { 205 uvm_lock_pageq(); 206 if (pg->wire_count == 0) { 207 #ifdef UBC 208 pmap_clear_reference(pg); 209 #else 210 pmap_page_protect(pg, VM_PROT_NONE); 211 #endif 212 uvm_pagedeactivate(pg); 213 } 214 uvm_unlock_pageq(); 215 } 216 simple_unlock(&anons[lcv]->an_lock); 217 } 218 } 219 220 /* 221 * normal functions 222 */ 223 224 /* 225 * uvmfault_amapcopy: clear "needs_copy" in a map. 226 * 227 * => called with VM data structures unlocked (usually, see below) 228 * => we get a write lock on the maps and clear needs_copy for a VA 229 * => if we are out of RAM we sleep (waiting for more) 230 */ 231 232 static void 233 uvmfault_amapcopy(struct uvm_faultinfo *ufi) 234 { 235 236 /* 237 * while we haven't done the job 238 */ 239 240 while (1) { 241 242 /* 243 * no mapping? give up. 244 */ 245 246 if (uvmfault_lookup(ufi, TRUE) == FALSE) 247 return; 248 249 /* 250 * copy if needed. 251 */ 252 253 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) 254 amap_copy(ufi->map, ufi->entry, M_NOWAIT, TRUE, 255 ufi->orig_rvaddr, ufi->orig_rvaddr + 1); 256 257 /* 258 * didn't work? must be out of RAM. unlock and sleep. 259 */ 260 261 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 262 uvmfault_unlockmaps(ufi, TRUE); 263 uvm_wait("fltamapcopy"); 264 continue; 265 } 266 267 /* 268 * got it! unlock and return. 269 */ 270 271 uvmfault_unlockmaps(ufi, TRUE); 272 return; 273 } 274 /*NOTREACHED*/ 275 } 276 277 /* 278 * uvmfault_anonget: get data in an anon into a non-busy, non-released 279 * page in that anon. 280 * 281 * => maps, amap, and anon locked by caller. 282 * => if we fail (result != VM_PAGER_OK) we unlock everything. 283 * => if we are successful, we return with everything still locked. 284 * => we don't move the page on the queues [gets moved later] 285 * => if we allocate a new page [we_own], it gets put on the queues. 286 * either way, the result is that the page is on the queues at return time 287 * => for pages which are on loan from a uvm_object (and thus are not 288 * owned by the anon): if successful, we return with the owning object 289 * locked. the caller must unlock this object when it unlocks everything 290 * else. 291 */ 292 293 int 294 uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap, 295 struct vm_anon *anon) 296 { 297 boolean_t we_own; /* we own anon's page? */ 298 boolean_t locked; /* did we relock? */ 299 struct vm_page *pg; 300 int result; 301 302 result = 0; /* XXX shut up gcc */ 303 uvmexp.fltanget++; 304 /* bump rusage counters */ 305 if (anon->an_page) 306 curproc->p_ru.ru_minflt++; 307 else 308 curproc->p_ru.ru_majflt++; 309 310 /* 311 * loop until we get it, or fail. 312 */ 313 314 while (1) { 315 316 we_own = FALSE; /* TRUE if we set PG_BUSY on a page */ 317 pg = anon->an_page; 318 319 /* 320 * if there is a resident page and it is loaned, then anon 321 * may not own it. call out to uvm_anon_lockpage() to ensure 322 * the real owner of the page has been identified and locked. 323 */ 324 325 if (pg && pg->loan_count) 326 pg = uvm_anon_lockloanpg(anon); 327 328 /* 329 * page there? make sure it is not busy/released. 330 */ 331 332 if (pg) { 333 334 /* 335 * at this point, if the page has a uobject [meaning 336 * we have it on loan], then that uobject is locked 337 * by us! if the page is busy, we drop all the 338 * locks (including uobject) and try again. 339 */ 340 341 if ((pg->pg_flags & (PG_BUSY|PG_RELEASED)) == 0) { 342 return (VM_PAGER_OK); 343 } 344 atomic_setbits_int(&pg->pg_flags, PG_WANTED); 345 uvmexp.fltpgwait++; 346 347 /* 348 * the last unlock must be an atomic unlock+wait on 349 * the owner of page 350 */ 351 if (pg->uobject) { /* owner is uobject ? */ 352 uvmfault_unlockall(ufi, amap, NULL, anon); 353 UVM_UNLOCK_AND_WAIT(pg, 354 &pg->uobject->vmobjlock, 355 FALSE, "anonget1",0); 356 } else { 357 /* anon owns page */ 358 uvmfault_unlockall(ufi, amap, NULL, NULL); 359 UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0, 360 "anonget2",0); 361 } 362 /* ready to relock and try again */ 363 364 } else { 365 366 /* 367 * no page, we must try and bring it in. 368 */ 369 pg = uvm_pagealloc(NULL, 0, anon, 0); 370 371 if (pg == NULL) { /* out of RAM. */ 372 373 uvmfault_unlockall(ufi, amap, NULL, anon); 374 uvmexp.fltnoram++; 375 uvm_wait("flt_noram1"); 376 /* ready to relock and try again */ 377 378 } else { 379 380 /* we set the PG_BUSY bit */ 381 we_own = TRUE; 382 uvmfault_unlockall(ufi, amap, NULL, anon); 383 384 /* 385 * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN 386 * page into the uvm_swap_get function with 387 * all data structures unlocked. note that 388 * it is ok to read an_swslot here because 389 * we hold PG_BUSY on the page. 390 */ 391 uvmexp.pageins++; 392 result = uvm_swap_get(pg, anon->an_swslot, 393 PGO_SYNCIO); 394 395 /* 396 * we clean up after the i/o below in the 397 * "we_own" case 398 */ 399 /* ready to relock and try again */ 400 } 401 } 402 403 /* 404 * now relock and try again 405 */ 406 407 locked = uvmfault_relock(ufi); 408 if (locked || we_own) 409 simple_lock(&anon->an_lock); 410 411 /* 412 * if we own the page (i.e. we set PG_BUSY), then we need 413 * to clean up after the I/O. there are three cases to 414 * consider: 415 * [1] page released during I/O: free anon and ReFault. 416 * [2] I/O not OK. free the page and cause the fault 417 * to fail. 418 * [3] I/O OK! activate the page and sync with the 419 * non-we_own case (i.e. drop anon lock if not locked). 420 */ 421 422 if (we_own) { 423 424 if (pg->pg_flags & PG_WANTED) { 425 /* still holding object lock */ 426 wakeup(pg); 427 } 428 /* un-busy! */ 429 atomic_clearbits_int(&pg->pg_flags, 430 PG_WANTED|PG_BUSY|PG_FAKE); 431 UVM_PAGE_OWN(pg, NULL); 432 433 /* 434 * if we were RELEASED during I/O, then our anon is 435 * no longer part of an amap. we need to free the 436 * anon and try again. 437 */ 438 if (pg->pg_flags & PG_RELEASED) { 439 pmap_page_protect(pg, VM_PROT_NONE); 440 simple_unlock(&anon->an_lock); 441 uvm_anfree(anon); /* frees page for us */ 442 if (locked) 443 uvmfault_unlockall(ufi, amap, NULL, 444 NULL); 445 uvmexp.fltpgrele++; 446 return (VM_PAGER_REFAULT); /* refault! */ 447 } 448 449 if (result != VM_PAGER_OK) { 450 KASSERT(result != VM_PAGER_PEND); 451 452 /* remove page from anon */ 453 anon->an_page = NULL; 454 455 /* 456 * remove the swap slot from the anon 457 * and mark the anon as having no real slot. 458 * don't free the swap slot, thus preventing 459 * it from being used again. 460 */ 461 uvm_swap_markbad(anon->an_swslot, 1); 462 anon->an_swslot = SWSLOT_BAD; 463 464 /* 465 * note: page was never !PG_BUSY, so it 466 * can't be mapped and thus no need to 467 * pmap_page_protect it... 468 */ 469 uvm_lock_pageq(); 470 uvm_pagefree(pg); 471 uvm_unlock_pageq(); 472 473 if (locked) 474 uvmfault_unlockall(ufi, amap, NULL, 475 anon); 476 else 477 simple_unlock(&anon->an_lock); 478 return (VM_PAGER_ERROR); 479 } 480 481 /* 482 * must be OK, clear modify (already PG_CLEAN) 483 * and activate 484 */ 485 pmap_clear_modify(pg); 486 uvm_lock_pageq(); 487 uvm_pageactivate(pg); 488 uvm_unlock_pageq(); 489 if (!locked) 490 simple_unlock(&anon->an_lock); 491 } 492 493 /* 494 * we were not able to relock. restart fault. 495 */ 496 497 if (!locked) 498 return (VM_PAGER_REFAULT); 499 500 /* 501 * verify no one has touched the amap and moved the anon on us. 502 */ 503 504 if (ufi != NULL && 505 amap_lookup(&ufi->entry->aref, 506 ufi->orig_rvaddr - ufi->entry->start) != anon) { 507 508 uvmfault_unlockall(ufi, amap, NULL, anon); 509 return (VM_PAGER_REFAULT); 510 } 511 512 /* 513 * try it again! 514 */ 515 516 uvmexp.fltanretry++; 517 continue; 518 519 } /* while (1) */ 520 521 /*NOTREACHED*/ 522 } 523 524 /* 525 * Update statistics after fault resolution. 526 * - maxrss 527 */ 528 void 529 uvmfault_update_stats(struct uvm_faultinfo *ufi) 530 { 531 struct vm_map *map; 532 struct proc *p; 533 vsize_t res; 534 #ifndef pmap_resident_count 535 struct vm_space *vm; 536 #endif 537 538 map = ufi->orig_map; 539 540 /* 541 * Update the maxrss for the process. 542 */ 543 if (map->flags & VM_MAP_ISVMSPACE) { 544 p = curproc; 545 KASSERT(p != NULL && &p->p_vmspace->vm_map == map); 546 547 #ifdef pmap_resident_count 548 res = pmap_resident_count(map->pmap); 549 #else 550 /* 551 * Rather inaccurate, but this is the current anon size 552 * of the vmspace. It's basically the resident size 553 * minus the mmapped in files/text. 554 */ 555 vm = (struct vmspace*)map; 556 res = vm->dsize; 557 #endif 558 559 /* Convert res from pages to kilobytes. */ 560 res <<= (PAGE_SHIFT - 10); 561 562 if (p->p_ru.ru_maxrss < res) 563 p->p_ru.ru_maxrss = res; 564 } 565 } 566 567 /* 568 * F A U L T - m a i n e n t r y p o i n t 569 */ 570 571 /* 572 * uvm_fault: page fault handler 573 * 574 * => called from MD code to resolve a page fault 575 * => VM data structures usually should be unlocked. however, it is 576 * possible to call here with the main map locked if the caller 577 * gets a write lock, sets it recursive, and then calls us (c.f. 578 * uvm_map_pageable). this should be avoided because it keeps 579 * the map locked off during I/O. 580 */ 581 582 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 583 ~VM_PROT_WRITE : VM_PROT_ALL) 584 585 int 586 uvm_fault(vm_map_t orig_map, vaddr_t vaddr, vm_fault_t fault_type, 587 vm_prot_t access_type) 588 { 589 struct uvm_faultinfo ufi; 590 vm_prot_t enter_prot; 591 boolean_t wired, narrow, promote, locked, shadowed; 592 int npages, nback, nforw, centeridx, result, lcv, gotpages; 593 vaddr_t startva, currva; 594 voff_t uoff; 595 paddr_t pa; 596 struct vm_amap *amap; 597 struct uvm_object *uobj; 598 struct vm_anon *anons_store[UVM_MAXRANGE], **anons, *anon, *oanon; 599 struct vm_page *pages[UVM_MAXRANGE], *pg, *uobjpage; 600 601 anon = NULL; 602 pg = NULL; 603 604 uvmexp.faults++; /* XXX: locking? */ 605 606 /* 607 * init the IN parameters in the ufi 608 */ 609 610 ufi.orig_map = orig_map; 611 ufi.orig_rvaddr = trunc_page(vaddr); 612 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ 613 if (fault_type == VM_FAULT_WIRE) 614 narrow = TRUE; /* don't look for neighborhood 615 * pages on wire */ 616 else 617 narrow = FALSE; /* normal fault */ 618 619 /* 620 * "goto ReFault" means restart the page fault from ground zero. 621 */ 622 ReFault: 623 624 /* 625 * lookup and lock the maps 626 */ 627 628 if (uvmfault_lookup(&ufi, FALSE) == FALSE) { 629 return (EFAULT); 630 } 631 /* locked: maps(read) */ 632 633 #ifdef DIAGNOSTIC 634 if ((ufi.map->flags & VM_MAP_PAGEABLE) == 0) 635 panic("uvm_fault: fault on non-pageable map (%p, 0x%lx)", 636 ufi.map, vaddr); 637 #endif 638 639 /* 640 * check protection 641 */ 642 643 if ((ufi.entry->protection & access_type) != access_type) { 644 uvmfault_unlockmaps(&ufi, FALSE); 645 return (EACCES); 646 } 647 648 /* 649 * "enter_prot" is the protection we want to enter the page in at. 650 * for certain pages (e.g. copy-on-write pages) this protection can 651 * be more strict than ufi.entry->protection. "wired" means either 652 * the entry is wired or we are fault-wiring the pg. 653 */ 654 655 enter_prot = ufi.entry->protection; 656 wired = VM_MAPENT_ISWIRED(ufi.entry) || (fault_type == VM_FAULT_WIRE); 657 if (wired) 658 access_type = enter_prot; /* full access for wired */ 659 660 /* 661 * handle "needs_copy" case. if we need to copy the amap we will 662 * have to drop our readlock and relock it with a write lock. (we 663 * need a write lock to change anything in a map entry [e.g. 664 * needs_copy]). 665 */ 666 667 if (UVM_ET_ISNEEDSCOPY(ufi.entry)) { 668 if ((access_type & VM_PROT_WRITE) || 669 (ufi.entry->object.uvm_obj == NULL)) { 670 /* need to clear */ 671 uvmfault_unlockmaps(&ufi, FALSE); 672 uvmfault_amapcopy(&ufi); 673 uvmexp.fltamcopy++; 674 goto ReFault; 675 676 } else { 677 678 /* 679 * ensure that we pmap_enter page R/O since 680 * needs_copy is still true 681 */ 682 enter_prot &= ~VM_PROT_WRITE; 683 684 } 685 } 686 687 /* 688 * identify the players 689 */ 690 691 amap = ufi.entry->aref.ar_amap; /* top layer */ 692 uobj = ufi.entry->object.uvm_obj; /* bottom layer */ 693 694 /* 695 * check for a case 0 fault. if nothing backing the entry then 696 * error now. 697 */ 698 699 if (amap == NULL && uobj == NULL) { 700 uvmfault_unlockmaps(&ufi, FALSE); 701 return (EFAULT); 702 } 703 704 /* 705 * establish range of interest based on advice from mapper 706 * and then clip to fit map entry. note that we only want 707 * to do this the first time through the fault. if we 708 * ReFault we will disable this by setting "narrow" to true. 709 */ 710 711 if (narrow == FALSE) { 712 713 /* wide fault (!narrow) */ 714 KASSERT(uvmadvice[ufi.entry->advice].advice == 715 ufi.entry->advice); 716 nback = min(uvmadvice[ufi.entry->advice].nback, 717 (ufi.orig_rvaddr - ufi.entry->start) >> PAGE_SHIFT); 718 startva = ufi.orig_rvaddr - (nback << PAGE_SHIFT); 719 nforw = min(uvmadvice[ufi.entry->advice].nforw, 720 ((ufi.entry->end - ufi.orig_rvaddr) >> 721 PAGE_SHIFT) - 1); 722 /* 723 * note: "-1" because we don't want to count the 724 * faulting page as forw 725 */ 726 npages = nback + nforw + 1; 727 centeridx = nback; 728 729 narrow = TRUE; /* ensure only once per-fault */ 730 731 } else { 732 733 /* narrow fault! */ 734 nback = nforw = 0; 735 startva = ufi.orig_rvaddr; 736 npages = 1; 737 centeridx = 0; 738 739 } 740 741 /* locked: maps(read) */ 742 743 /* 744 * if we've got an amap, lock it and extract current anons. 745 */ 746 747 if (amap) { 748 anons = anons_store; 749 amap_lookups(&ufi.entry->aref, startva - ufi.entry->start, 750 anons, npages); 751 } else { 752 anons = NULL; /* to be safe */ 753 } 754 755 /* locked: maps(read), amap(if there) */ 756 757 /* 758 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages 759 * now and then forget about them (for the rest of the fault). 760 */ 761 762 if (ufi.entry->advice == MADV_SEQUENTIAL && nback != 0) { 763 764 /* flush back-page anons? */ 765 if (amap) 766 uvmfault_anonflush(anons, nback); 767 768 /* flush object? */ 769 if (uobj) { 770 uoff = (startva - ufi.entry->start) + ufi.entry->offset; 771 simple_lock(&uobj->vmobjlock); 772 (void) uobj->pgops->pgo_flush(uobj, uoff, uoff + 773 (nback << PAGE_SHIFT), PGO_DEACTIVATE); 774 simple_unlock(&uobj->vmobjlock); 775 } 776 777 /* now forget about the backpages */ 778 if (amap) 779 anons += nback; 780 startva += (nback << PAGE_SHIFT); 781 npages -= nback; 782 centeridx = 0; 783 } 784 785 /* locked: maps(read), amap(if there) */ 786 787 /* 788 * map in the backpages and frontpages we found in the amap in hopes 789 * of preventing future faults. we also init the pages[] array as 790 * we go. 791 */ 792 793 currva = startva; 794 shadowed = FALSE; 795 for (lcv = 0 ; lcv < npages ; lcv++, currva += PAGE_SIZE) { 796 797 /* 798 * dont play with VAs that are already mapped 799 * except for center) 800 */ 801 if (lcv != centeridx && 802 pmap_extract(ufi.orig_map->pmap, currva, &pa)) { 803 pages[lcv] = PGO_DONTCARE; 804 continue; 805 } 806 807 /* 808 * unmapped or center page. check if any anon at this level. 809 */ 810 if (amap == NULL || anons[lcv] == NULL) { 811 pages[lcv] = NULL; 812 continue; 813 } 814 815 /* 816 * check for present page and map if possible. re-activate it. 817 */ 818 819 pages[lcv] = PGO_DONTCARE; 820 if (lcv == centeridx) { /* save center for later! */ 821 shadowed = TRUE; 822 continue; 823 } 824 anon = anons[lcv]; 825 simple_lock(&anon->an_lock); 826 /* ignore loaned pages */ 827 if (anon->an_page && anon->an_page->loan_count == 0 && 828 (anon->an_page->pg_flags & (PG_RELEASED|PG_BUSY)) == 0) { 829 uvm_lock_pageq(); 830 uvm_pageactivate(anon->an_page); /* reactivate */ 831 uvm_unlock_pageq(); 832 uvmexp.fltnamap++; 833 834 /* 835 * Since this isn't the page that's actually faulting, 836 * ignore pmap_enter() failures; it's not critical 837 * that we enter these right now. 838 */ 839 840 (void) pmap_enter(ufi.orig_map->pmap, currva, 841 VM_PAGE_TO_PHYS(anon->an_page), 842 (anon->an_ref > 1) ? (enter_prot & ~VM_PROT_WRITE) : 843 enter_prot, 844 PMAP_CANFAIL | 845 (VM_MAPENT_ISWIRED(ufi.entry) ? PMAP_WIRED : 0)); 846 } 847 simple_unlock(&anon->an_lock); 848 pmap_update(ufi.orig_map->pmap); 849 } 850 851 /* locked: maps(read), amap(if there) */ 852 /* (shadowed == TRUE) if there is an anon at the faulting address */ 853 854 /* 855 * note that if we are really short of RAM we could sleep in the above 856 * call to pmap_enter with everything locked. bad? 857 * 858 * XXX Actually, that is bad; pmap_enter() should just fail in that 859 * XXX case. --thorpej 860 */ 861 862 /* 863 * if the desired page is not shadowed by the amap and we have a 864 * backing object, then we check to see if the backing object would 865 * prefer to handle the fault itself (rather than letting us do it 866 * with the usual pgo_get hook). the backing object signals this by 867 * providing a pgo_fault routine. 868 */ 869 870 if (uobj && shadowed == FALSE && uobj->pgops->pgo_fault != NULL) { 871 simple_lock(&uobj->vmobjlock); 872 873 /* locked: maps(read), amap (if there), uobj */ 874 result = uobj->pgops->pgo_fault(&ufi, startva, pages, npages, 875 centeridx, fault_type, access_type, 876 PGO_LOCKED); 877 878 /* locked: nothing, pgo_fault has unlocked everything */ 879 880 if (result == VM_PAGER_OK) 881 return (0); /* pgo_fault did pmap enter */ 882 else if (result == VM_PAGER_REFAULT) 883 goto ReFault; /* try again! */ 884 else 885 return (EACCES); 886 } 887 888 /* 889 * now, if the desired page is not shadowed by the amap and we have 890 * a backing object that does not have a special fault routine, then 891 * we ask (with pgo_get) the object for resident pages that we care 892 * about and attempt to map them in. we do not let pgo_get block 893 * (PGO_LOCKED). 894 * 895 * ("get" has the option of doing a pmap_enter for us) 896 */ 897 898 if (uobj && shadowed == FALSE) { 899 simple_lock(&uobj->vmobjlock); 900 901 /* locked (!shadowed): maps(read), amap (if there), uobj */ 902 /* 903 * the following call to pgo_get does _not_ change locking state 904 */ 905 906 uvmexp.fltlget++; 907 gotpages = npages; 908 (void) uobj->pgops->pgo_get(uobj, ufi.entry->offset + 909 (startva - ufi.entry->start), 910 pages, &gotpages, centeridx, 911 access_type & MASK(ufi.entry), 912 ufi.entry->advice, PGO_LOCKED); 913 914 /* 915 * check for pages to map, if we got any 916 */ 917 918 uobjpage = NULL; 919 920 if (gotpages) { 921 currva = startva; 922 for (lcv = 0 ; lcv < npages ; 923 lcv++, currva += PAGE_SIZE) { 924 925 if (pages[lcv] == NULL || 926 pages[lcv] == PGO_DONTCARE) 927 continue; 928 929 KASSERT((pages[lcv]->pg_flags & PG_RELEASED) == 0); 930 931 /* 932 * if center page is resident and not 933 * PG_BUSY, then pgo_get made it PG_BUSY 934 * for us and gave us a handle to it. 935 * remember this page as "uobjpage." 936 * (for later use). 937 */ 938 939 if (lcv == centeridx) { 940 uobjpage = pages[lcv]; 941 continue; 942 } 943 944 /* 945 * note: calling pgo_get with locked data 946 * structures returns us pages which are 947 * neither busy nor released, so we don't 948 * need to check for this. we can just 949 * directly enter the page (after moving it 950 * to the head of the active queue [useful?]). 951 */ 952 953 uvm_lock_pageq(); 954 uvm_pageactivate(pages[lcv]); /* reactivate */ 955 uvm_unlock_pageq(); 956 uvmexp.fltnomap++; 957 958 /* 959 * Since this page isn't the page that's 960 * actually fauling, ignore pmap_enter() 961 * failures; it's not critical that we 962 * enter these right now. 963 */ 964 965 (void) pmap_enter(ufi.orig_map->pmap, currva, 966 VM_PAGE_TO_PHYS(pages[lcv]), 967 enter_prot & MASK(ufi.entry), 968 PMAP_CANFAIL | 969 (wired ? PMAP_WIRED : 0)); 970 971 /* 972 * NOTE: page can't be PG_WANTED because 973 * we've held the lock the whole time 974 * we've had the handle. 975 */ 976 977 atomic_clearbits_int(&pages[lcv]->pg_flags, 978 PG_BUSY); 979 UVM_PAGE_OWN(pages[lcv], NULL); 980 } /* for "lcv" loop */ 981 pmap_update(ufi.orig_map->pmap); 982 } /* "gotpages" != 0 */ 983 /* note: object still _locked_ */ 984 } else { 985 uobjpage = NULL; 986 } 987 988 /* locked (shadowed): maps(read), amap */ 989 /* locked (!shadowed): maps(read), amap(if there), 990 uobj(if !null), uobjpage(if !null) */ 991 992 /* 993 * note that at this point we are done with any front or back pages. 994 * we are now going to focus on the center page (i.e. the one we've 995 * faulted on). if we have faulted on the top (anon) layer 996 * [i.e. case 1], then the anon we want is anons[centeridx] (we have 997 * not touched it yet). if we have faulted on the bottom (uobj) 998 * layer [i.e. case 2] and the page was both present and available, 999 * then we've got a pointer to it as "uobjpage" and we've already 1000 * made it BUSY. 1001 */ 1002 1003 /* 1004 * there are four possible cases we must address: 1A, 1B, 2A, and 2B 1005 */ 1006 1007 /* 1008 * redirect case 2: if we are not shadowed, go to case 2. 1009 */ 1010 1011 if (shadowed == FALSE) 1012 goto Case2; 1013 1014 /* locked: maps(read), amap */ 1015 1016 /* 1017 * handle case 1: fault on an anon in our amap 1018 */ 1019 1020 anon = anons[centeridx]; 1021 simple_lock(&anon->an_lock); 1022 1023 /* locked: maps(read), amap, anon */ 1024 1025 /* 1026 * no matter if we have case 1A or case 1B we are going to need to 1027 * have the anon's memory resident. ensure that now. 1028 */ 1029 1030 /* 1031 * let uvmfault_anonget do the dirty work. 1032 * if it fails (!OK) it will unlock everything for us. 1033 * if it succeeds, locks are still valid and locked. 1034 * also, if it is OK, then the anon's page is on the queues. 1035 * if the page is on loan from a uvm_object, then anonget will 1036 * lock that object for us if it does not fail. 1037 */ 1038 1039 result = uvmfault_anonget(&ufi, amap, anon); 1040 switch (result) { 1041 case VM_PAGER_OK: 1042 break; 1043 1044 case VM_PAGER_REFAULT: 1045 goto ReFault; 1046 1047 case VM_PAGER_ERROR: 1048 /* 1049 * An error occured while trying to bring in the 1050 * page -- this is the only error we return right 1051 * now. 1052 */ 1053 return (EACCES); /* XXX */ 1054 1055 default: 1056 #ifdef DIAGNOSTIC 1057 panic("uvm_fault: uvmfault_anonget -> %d", result); 1058 #else 1059 return (EACCES); 1060 #endif 1061 } 1062 1063 /* 1064 * uobj is non null if the page is on loan from an object (i.e. uobj) 1065 */ 1066 1067 uobj = anon->an_page->uobject; /* locked by anonget if !NULL */ 1068 1069 /* locked: maps(read), amap, anon, uobj(if one) */ 1070 1071 /* 1072 * special handling for loaned pages 1073 */ 1074 1075 if (anon->an_page->loan_count) { 1076 1077 if ((access_type & VM_PROT_WRITE) == 0) { 1078 1079 /* 1080 * for read faults on loaned pages we just cap the 1081 * protection at read-only. 1082 */ 1083 1084 enter_prot = enter_prot & ~VM_PROT_WRITE; 1085 1086 } else { 1087 /* 1088 * note that we can't allow writes into a loaned page! 1089 * 1090 * if we have a write fault on a loaned page in an 1091 * anon then we need to look at the anon's ref count. 1092 * if it is greater than one then we are going to do 1093 * a normal copy-on-write fault into a new anon (this 1094 * is not a problem). however, if the reference count 1095 * is one (a case where we would normally allow a 1096 * write directly to the page) then we need to kill 1097 * the loan before we continue. 1098 */ 1099 1100 /* >1 case is already ok */ 1101 if (anon->an_ref == 1) { 1102 1103 /* get new un-owned replacement page */ 1104 pg = uvm_pagealloc(NULL, 0, NULL, 0); 1105 if (pg == NULL) { 1106 uvmfault_unlockall(&ufi, amap, uobj, 1107 anon); 1108 uvm_wait("flt_noram2"); 1109 goto ReFault; 1110 } 1111 1112 /* 1113 * copy data, kill loan, and drop uobj lock 1114 * (if any) 1115 */ 1116 /* copy old -> new */ 1117 uvm_pagecopy(anon->an_page, pg); 1118 1119 /* force reload */ 1120 pmap_page_protect(anon->an_page, 1121 VM_PROT_NONE); 1122 uvm_lock_pageq(); /* KILL loan */ 1123 if (uobj) 1124 /* if we were loaning */ 1125 anon->an_page->loan_count--; 1126 anon->an_page->uanon = NULL; 1127 /* in case we owned */ 1128 atomic_clearbits_int( 1129 &anon->an_page->pg_flags, PQ_ANON); 1130 uvm_pageactivate(pg); 1131 uvm_unlock_pageq(); 1132 if (uobj) { 1133 simple_unlock(&uobj->vmobjlock); 1134 uobj = NULL; 1135 } 1136 1137 /* install new page in anon */ 1138 anon->an_page = pg; 1139 pg->uanon = anon; 1140 atomic_setbits_int(&pg->pg_flags, PQ_ANON); 1141 atomic_clearbits_int(&pg->pg_flags, 1142 PG_BUSY|PG_FAKE); 1143 UVM_PAGE_OWN(pg, NULL); 1144 1145 /* done! */ 1146 } /* ref == 1 */ 1147 } /* write fault */ 1148 } /* loan count */ 1149 1150 /* 1151 * if we are case 1B then we will need to allocate a new blank 1152 * anon to transfer the data into. note that we have a lock 1153 * on anon, so no one can busy or release the page until we are done. 1154 * also note that the ref count can't drop to zero here because 1155 * it is > 1 and we are only dropping one ref. 1156 * 1157 * in the (hopefully very rare) case that we are out of RAM we 1158 * will unlock, wait for more RAM, and refault. 1159 * 1160 * if we are out of anon VM we kill the process (XXX: could wait?). 1161 */ 1162 1163 if ((access_type & VM_PROT_WRITE) != 0 && anon->an_ref > 1) { 1164 uvmexp.flt_acow++; 1165 oanon = anon; /* oanon = old, locked anon */ 1166 anon = uvm_analloc(); 1167 if (anon) { 1168 pg = uvm_pagealloc(NULL, 0, anon, 0); 1169 } 1170 1171 /* check for out of RAM */ 1172 if (anon == NULL || pg == NULL) { 1173 if (anon) 1174 uvm_anfree(anon); 1175 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1176 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1177 if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) { 1178 uvmexp.fltnoanon++; 1179 return (ENOMEM); 1180 } 1181 1182 uvmexp.fltnoram++; 1183 uvm_wait("flt_noram3"); /* out of RAM, wait for more */ 1184 goto ReFault; 1185 } 1186 1187 /* got all resources, replace anon with nanon */ 1188 1189 uvm_pagecopy(oanon->an_page, pg); /* pg now !PG_CLEAN */ 1190 /* un-busy! new page */ 1191 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE); 1192 UVM_PAGE_OWN(pg, NULL); 1193 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start, 1194 anon, 1); 1195 1196 /* deref: can not drop to zero here by defn! */ 1197 oanon->an_ref--; 1198 1199 /* 1200 * note: oanon still locked. anon is _not_ locked, but we 1201 * have the sole references to in from amap which _is_ locked. 1202 * thus, no one can get at it until we are done with it. 1203 */ 1204 1205 } else { 1206 1207 uvmexp.flt_anon++; 1208 oanon = anon; /* old, locked anon is same as anon */ 1209 pg = anon->an_page; 1210 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */ 1211 enter_prot = enter_prot & ~VM_PROT_WRITE; 1212 1213 } 1214 1215 /* locked: maps(read), amap, oanon */ 1216 1217 /* 1218 * now map the page in ... 1219 * XXX: old fault unlocks object before pmap_enter. this seems 1220 * suspect since some other thread could blast the page out from 1221 * under us between the unlock and the pmap_enter. 1222 */ 1223 1224 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1225 enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) 1226 != 0) { 1227 /* 1228 * No need to undo what we did; we can simply think of 1229 * this as the pmap throwing away the mapping information. 1230 * 1231 * We do, however, have to go through the ReFault path, 1232 * as the map may change while we're asleep. 1233 */ 1234 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1235 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1236 if (uvmexp.swpgonly == uvmexp.swpages) { 1237 /* XXX instrumentation */ 1238 return (ENOMEM); 1239 } 1240 /* XXX instrumentation */ 1241 uvm_wait("flt_pmfail1"); 1242 goto ReFault; 1243 } 1244 1245 /* 1246 * ... update the page queues. 1247 */ 1248 1249 uvm_lock_pageq(); 1250 1251 if (fault_type == VM_FAULT_WIRE) { 1252 uvm_pagewire(pg); 1253 1254 /* 1255 * since the now-wired page cannot be paged out, 1256 * release its swap resources for others to use. 1257 * since an anon with no swap cannot be PG_CLEAN, 1258 * clear its clean flag now. 1259 */ 1260 atomic_clearbits_int(&pg->pg_flags, PG_CLEAN); 1261 uvm_anon_dropswap(anon); 1262 } else { 1263 /* activate it */ 1264 uvm_pageactivate(pg); 1265 } 1266 1267 uvm_unlock_pageq(); 1268 1269 /* 1270 * done case 1! finish up by unlocking everything and returning success 1271 */ 1272 1273 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1274 pmap_update(ufi.orig_map->pmap); 1275 return (0); 1276 1277 1278 Case2: 1279 /* 1280 * handle case 2: faulting on backing object or zero fill 1281 */ 1282 1283 /* 1284 * locked: 1285 * maps(read), amap(if there), uobj(if !null), uobjpage(if !null) 1286 */ 1287 1288 /* 1289 * note that uobjpage can not be PGO_DONTCARE at this point. we now 1290 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we 1291 * have a backing object, check and see if we are going to promote 1292 * the data up to an anon during the fault. 1293 */ 1294 1295 if (uobj == NULL) { 1296 uobjpage = PGO_DONTCARE; 1297 promote = TRUE; /* always need anon here */ 1298 } else { 1299 KASSERT(uobjpage != PGO_DONTCARE); 1300 promote = (access_type & VM_PROT_WRITE) && 1301 UVM_ET_ISCOPYONWRITE(ufi.entry); 1302 } 1303 1304 /* 1305 * if uobjpage is not null then we do not need to do I/O to get the 1306 * uobjpage. 1307 * 1308 * if uobjpage is null, then we need to unlock and ask the pager to 1309 * get the data for us. once we have the data, we need to reverify 1310 * the state the world. we are currently not holding any resources. 1311 */ 1312 1313 if (uobjpage) { 1314 /* update rusage counters */ 1315 curproc->p_ru.ru_minflt++; 1316 } else { 1317 /* update rusage counters */ 1318 curproc->p_ru.ru_majflt++; 1319 1320 /* locked: maps(read), amap(if there), uobj */ 1321 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1322 /* locked: uobj */ 1323 1324 uvmexp.fltget++; 1325 gotpages = 1; 1326 uoff = (ufi.orig_rvaddr - ufi.entry->start) + ufi.entry->offset; 1327 result = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages, 1328 0, access_type & MASK(ufi.entry), ufi.entry->advice, 1329 PGO_SYNCIO); 1330 1331 /* locked: uobjpage(if result OK) */ 1332 1333 /* 1334 * recover from I/O 1335 */ 1336 1337 if (result != VM_PAGER_OK) { 1338 KASSERT(result != VM_PAGER_PEND); 1339 1340 if (result == VM_PAGER_AGAIN) { 1341 tsleep((caddr_t)&lbolt, PVM, "fltagain2", 0); 1342 goto ReFault; 1343 } 1344 1345 return (EACCES); /* XXX i/o error */ 1346 } 1347 1348 /* locked: uobjpage */ 1349 1350 /* 1351 * re-verify the state of the world by first trying to relock 1352 * the maps. always relock the object. 1353 */ 1354 1355 locked = uvmfault_relock(&ufi); 1356 simple_lock(&uobj->vmobjlock); 1357 1358 /* locked(locked): maps(read), amap(if !null), uobj, uobjpage */ 1359 /* locked(!locked): uobj, uobjpage */ 1360 1361 /* 1362 * Re-verify that amap slot is still free. if there is 1363 * a problem, we unlock and clean up. 1364 */ 1365 1366 if (locked && amap && amap_lookup(&ufi.entry->aref, 1367 ufi.orig_rvaddr - ufi.entry->start)) { 1368 if (locked) 1369 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1370 locked = FALSE; 1371 } 1372 1373 /* 1374 * didn't get the lock? release the page and retry. 1375 */ 1376 1377 if (locked == FALSE) { 1378 if (uobjpage->pg_flags & PG_WANTED) 1379 /* still holding object lock */ 1380 wakeup(uobjpage); 1381 1382 uvm_lock_pageq(); 1383 /* make sure it is in queues */ 1384 uvm_pageactivate(uobjpage); 1385 1386 uvm_unlock_pageq(); 1387 atomic_clearbits_int(&uobjpage->pg_flags, 1388 PG_BUSY|PG_WANTED); 1389 UVM_PAGE_OWN(uobjpage, NULL); 1390 simple_unlock(&uobj->vmobjlock); 1391 goto ReFault; 1392 1393 } 1394 1395 /* 1396 * we have the data in uobjpage which is PG_BUSY and we are 1397 * holding object lock. 1398 */ 1399 1400 /* locked: maps(read), amap(if !null), uobj, uobjpage */ 1401 } 1402 1403 /* 1404 * locked: 1405 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj) 1406 */ 1407 1408 /* 1409 * notes: 1410 * - at this point uobjpage can not be NULL 1411 * - at this point uobjpage could be PG_WANTED (handle later) 1412 */ 1413 1414 if (promote == FALSE) { 1415 1416 /* 1417 * we are not promoting. if the mapping is COW ensure that we 1418 * don't give more access than we should (e.g. when doing a read 1419 * fault on a COPYONWRITE mapping we want to map the COW page in 1420 * R/O even though the entry protection could be R/W). 1421 * 1422 * set "pg" to the page we want to map in (uobjpage, usually) 1423 */ 1424 1425 uvmexp.flt_obj++; 1426 if (UVM_ET_ISCOPYONWRITE(ufi.entry)) 1427 enter_prot &= ~VM_PROT_WRITE; 1428 pg = uobjpage; /* map in the actual object */ 1429 1430 /* assert(uobjpage != PGO_DONTCARE) */ 1431 1432 /* 1433 * we are faulting directly on the page. be careful 1434 * about writing to loaned pages... 1435 */ 1436 if (uobjpage->loan_count) { 1437 1438 if ((access_type & VM_PROT_WRITE) == 0) { 1439 /* read fault: cap the protection at readonly */ 1440 /* cap! */ 1441 enter_prot = enter_prot & ~VM_PROT_WRITE; 1442 } else { 1443 /* write fault: must break the loan here */ 1444 1445 /* alloc new un-owned page */ 1446 pg = uvm_pagealloc(NULL, 0, NULL, 0); 1447 1448 if (pg == NULL) { 1449 /* 1450 * drop ownership of page, it can't 1451 * be released 1452 */ 1453 if (uobjpage->pg_flags & PG_WANTED) 1454 wakeup(uobjpage); 1455 atomic_clearbits_int( 1456 &uobjpage->pg_flags, 1457 PG_BUSY|PG_WANTED); 1458 UVM_PAGE_OWN(uobjpage, NULL); 1459 1460 uvm_lock_pageq(); 1461 /* activate: we will need it later */ 1462 uvm_pageactivate(uobjpage); 1463 1464 uvm_unlock_pageq(); 1465 uvmfault_unlockall(&ufi, amap, uobj, 1466 NULL); 1467 uvmexp.fltnoram++; 1468 uvm_wait("flt_noram4"); 1469 goto ReFault; 1470 } 1471 1472 /* 1473 * copy the data from the old page to the new 1474 * one and clear the fake/clean flags on the 1475 * new page (keep it busy). force a reload 1476 * of the old page by clearing it from all 1477 * pmaps. then lock the page queues to 1478 * rename the pages. 1479 */ 1480 uvm_pagecopy(uobjpage, pg); /* old -> new */ 1481 atomic_clearbits_int(&pg->pg_flags, 1482 PG_FAKE|PG_CLEAN); 1483 pmap_page_protect(uobjpage, VM_PROT_NONE); 1484 if (uobjpage->pg_flags & PG_WANTED) 1485 wakeup(uobjpage); 1486 /* uobj still locked */ 1487 atomic_clearbits_int(&uobjpage->pg_flags, 1488 PG_BUSY|PG_WANTED); 1489 UVM_PAGE_OWN(uobjpage, NULL); 1490 1491 uvm_lock_pageq(); 1492 uoff = uobjpage->offset; 1493 /* remove old page */ 1494 uvm_pagerealloc(uobjpage, NULL, 0); 1495 1496 /* 1497 * at this point we have absolutely no 1498 * control over uobjpage 1499 */ 1500 /* install new page */ 1501 uvm_pagerealloc(pg, uobj, uoff); 1502 uvm_unlock_pageq(); 1503 1504 /* 1505 * done! loan is broken and "pg" is 1506 * PG_BUSY. it can now replace uobjpage. 1507 */ 1508 1509 uobjpage = pg; 1510 1511 } /* write fault case */ 1512 } /* if loan_count */ 1513 1514 } else { 1515 1516 /* 1517 * if we are going to promote the data to an anon we 1518 * allocate a blank anon here and plug it into our amap. 1519 */ 1520 #ifdef DIAGNOSTIC 1521 if (amap == NULL) 1522 panic("uvm_fault: want to promote data, but no anon"); 1523 #endif 1524 1525 anon = uvm_analloc(); 1526 if (anon) { 1527 /* 1528 * In `Fill in data...' below, if 1529 * uobjpage == PGO_DONTCARE, we want 1530 * a zero'd, dirty page, so have 1531 * uvm_pagealloc() do that for us. 1532 */ 1533 pg = uvm_pagealloc(NULL, 0, anon, 1534 (uobjpage == PGO_DONTCARE) ? UVM_PGA_ZERO : 0); 1535 } 1536 1537 /* 1538 * out of memory resources? 1539 */ 1540 if (anon == NULL || pg == NULL) { 1541 1542 /* 1543 * arg! must unbusy our page and fail or sleep. 1544 */ 1545 if (uobjpage != PGO_DONTCARE) { 1546 if (uobjpage->pg_flags & PG_WANTED) 1547 /* still holding object lock */ 1548 wakeup(uobjpage); 1549 1550 uvm_lock_pageq(); 1551 uvm_pageactivate(uobjpage); 1552 uvm_unlock_pageq(); 1553 atomic_clearbits_int(&uobjpage->pg_flags, 1554 PG_BUSY|PG_WANTED); 1555 UVM_PAGE_OWN(uobjpage, NULL); 1556 } 1557 1558 /* unlock and fail ... */ 1559 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1560 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1561 if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) { 1562 uvmexp.fltnoanon++; 1563 return (ENOMEM); 1564 } 1565 1566 uvm_anfree(anon); 1567 uvmexp.fltnoram++; 1568 uvm_wait("flt_noram5"); 1569 goto ReFault; 1570 } 1571 1572 /* 1573 * fill in the data 1574 */ 1575 1576 if (uobjpage != PGO_DONTCARE) { 1577 uvmexp.flt_prcopy++; 1578 /* copy page [pg now dirty] */ 1579 uvm_pagecopy(uobjpage, pg); 1580 1581 /* 1582 * promote to shared amap? make sure all sharing 1583 * procs see it 1584 */ 1585 if ((amap_flags(amap) & AMAP_SHARED) != 0) { 1586 pmap_page_protect(uobjpage, VM_PROT_NONE); 1587 } 1588 1589 /* 1590 * dispose of uobjpage. drop handle to uobj as well. 1591 */ 1592 1593 if (uobjpage->pg_flags & PG_WANTED) 1594 /* still have the obj lock */ 1595 wakeup(uobjpage); 1596 atomic_clearbits_int(&uobjpage->pg_flags, 1597 PG_BUSY|PG_WANTED); 1598 UVM_PAGE_OWN(uobjpage, NULL); 1599 uvm_lock_pageq(); 1600 uvm_pageactivate(uobjpage); 1601 uvm_unlock_pageq(); 1602 simple_unlock(&uobj->vmobjlock); 1603 uobj = NULL; 1604 } else { 1605 uvmexp.flt_przero++; 1606 /* 1607 * Page is zero'd and marked dirty by uvm_pagealloc() 1608 * above. 1609 */ 1610 } 1611 1612 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start, 1613 anon, 0); 1614 } 1615 1616 /* 1617 * locked: 1618 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj) 1619 * 1620 * note: pg is either the uobjpage or the new page in the new anon 1621 */ 1622 1623 /* 1624 * all resources are present. we can now map it in and free our 1625 * resources. 1626 */ 1627 1628 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1629 enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) 1630 != 0) { 1631 1632 /* 1633 * No need to undo what we did; we can simply think of 1634 * this as the pmap throwing away the mapping information. 1635 * 1636 * We do, however, have to go through the ReFault path, 1637 * as the map may change while we're asleep. 1638 */ 1639 1640 if (pg->pg_flags & PG_WANTED) 1641 wakeup(pg); /* lock still held */ 1642 1643 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED); 1644 UVM_PAGE_OWN(pg, NULL); 1645 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1646 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1647 if (uvmexp.swpgonly == uvmexp.swpages) { 1648 /* XXX instrumentation */ 1649 return (ENOMEM); 1650 } 1651 /* XXX instrumentation */ 1652 uvm_wait("flt_pmfail2"); 1653 goto ReFault; 1654 } 1655 1656 uvm_lock_pageq(); 1657 1658 if (fault_type == VM_FAULT_WIRE) { 1659 uvm_pagewire(pg); 1660 if (pg->pg_flags & PQ_AOBJ) { 1661 1662 /* 1663 * since the now-wired page cannot be paged out, 1664 * release its swap resources for others to use. 1665 * since an aobj page with no swap cannot be PG_CLEAN, 1666 * clear its clean flag now. 1667 */ 1668 atomic_clearbits_int(&pg->pg_flags, PG_CLEAN); 1669 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT); 1670 } 1671 } else { 1672 /* activate it */ 1673 uvm_pageactivate(pg); 1674 } 1675 uvm_unlock_pageq(); 1676 1677 if (pg->pg_flags & PG_WANTED) 1678 wakeup(pg); /* lock still held */ 1679 1680 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED); 1681 UVM_PAGE_OWN(pg, NULL); 1682 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1683 pmap_update(ufi.orig_map->pmap); 1684 1685 return (0); 1686 } 1687 1688 1689 /* 1690 * uvm_fault_wire: wire down a range of virtual addresses in a map. 1691 * 1692 * => map may be read-locked by caller, but MUST NOT be write-locked. 1693 * => if map is read-locked, any operations which may cause map to 1694 * be write-locked in uvm_fault() must be taken care of by 1695 * the caller. See uvm_map_pageable(). 1696 */ 1697 1698 int 1699 uvm_fault_wire(vm_map_t map, vaddr_t start, vaddr_t end, vm_prot_t access_type) 1700 { 1701 vaddr_t va; 1702 pmap_t pmap; 1703 int rv; 1704 1705 pmap = vm_map_pmap(map); 1706 1707 /* 1708 * now fault it in a page at a time. if the fault fails then we have 1709 * to undo what we have done. note that in uvm_fault VM_PROT_NONE 1710 * is replaced with the max protection if fault_type is VM_FAULT_WIRE. 1711 */ 1712 1713 for (va = start ; va < end ; va += PAGE_SIZE) { 1714 rv = uvm_fault(map, va, VM_FAULT_WIRE, access_type); 1715 if (rv) { 1716 if (va != start) { 1717 uvm_fault_unwire(map, start, va); 1718 } 1719 return (rv); 1720 } 1721 } 1722 1723 return (0); 1724 } 1725 1726 /* 1727 * uvm_fault_unwire(): unwire range of virtual space. 1728 */ 1729 1730 void 1731 uvm_fault_unwire(vm_map_t map, vaddr_t start, vaddr_t end) 1732 { 1733 1734 vm_map_lock_read(map); 1735 uvm_fault_unwire_locked(map, start, end); 1736 vm_map_unlock_read(map); 1737 } 1738 1739 /* 1740 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire(). 1741 * 1742 * => map must be at least read-locked. 1743 */ 1744 1745 void 1746 uvm_fault_unwire_locked(vm_map_t map, vaddr_t start, vaddr_t end) 1747 { 1748 vm_map_entry_t entry, next; 1749 pmap_t pmap = vm_map_pmap(map); 1750 vaddr_t va; 1751 paddr_t pa; 1752 struct vm_page *pg; 1753 1754 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0); 1755 1756 /* 1757 * we assume that the area we are unwiring has actually been wired 1758 * in the first place. this means that we should be able to extract 1759 * the PAs from the pmap. we also lock out the page daemon so that 1760 * we can call uvm_pageunwire. 1761 */ 1762 1763 uvm_lock_pageq(); 1764 1765 /* 1766 * find the beginning map entry for the region. 1767 */ 1768 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map)); 1769 if (uvm_map_lookup_entry(map, start, &entry) == FALSE) 1770 panic("uvm_fault_unwire_locked: address not in map"); 1771 1772 for (va = start; va < end ; va += PAGE_SIZE) { 1773 if (pmap_extract(pmap, va, &pa) == FALSE) 1774 continue; 1775 1776 /* 1777 * find the map entry for the current address. 1778 */ 1779 KASSERT(va >= entry->start); 1780 while (va >= entry->end) { 1781 next = RB_NEXT(uvm_map_addr, &map->addr, entry); 1782 KASSERT(next != NULL && next->start <= entry->end); 1783 entry = next; 1784 } 1785 1786 /* 1787 * if the entry is no longer wired, tell the pmap. 1788 */ 1789 if (VM_MAPENT_ISWIRED(entry) == 0) 1790 pmap_unwire(pmap, va); 1791 1792 pg = PHYS_TO_VM_PAGE(pa); 1793 if (pg) 1794 uvm_pageunwire(pg); 1795 } 1796 1797 uvm_unlock_pageq(); 1798 } 1799 1800 /* 1801 * uvmfault_unlockmaps: unlock the maps 1802 */ 1803 void 1804 uvmfault_unlockmaps(struct uvm_faultinfo *ufi, boolean_t write_locked) 1805 { 1806 /* 1807 * ufi can be NULL when this isn't really a fault, 1808 * but merely paging in anon data. 1809 */ 1810 1811 if (ufi == NULL) { 1812 return; 1813 } 1814 1815 uvmfault_update_stats(ufi); 1816 if (write_locked) { 1817 vm_map_unlock(ufi->map); 1818 } else { 1819 vm_map_unlock_read(ufi->map); 1820 } 1821 } 1822 1823 /* 1824 * uvmfault_unlockall: unlock everything passed in. 1825 * 1826 * => maps must be read-locked (not write-locked). 1827 */ 1828 void 1829 uvmfault_unlockall(struct uvm_faultinfo *ufi, struct vm_amap *amap, 1830 struct uvm_object *uobj, struct vm_anon *anon) 1831 { 1832 1833 if (anon) 1834 simple_unlock(&anon->an_lock); 1835 if (uobj) 1836 simple_unlock(&uobj->vmobjlock); 1837 uvmfault_unlockmaps(ufi, FALSE); 1838 } 1839 1840 /* 1841 * uvmfault_lookup: lookup a virtual address in a map 1842 * 1843 * => caller must provide a uvm_faultinfo structure with the IN 1844 * params properly filled in 1845 * => we will lookup the map entry (handling submaps) as we go 1846 * => if the lookup is a success we will return with the maps locked 1847 * => if "write_lock" is TRUE, we write_lock the map, otherwise we only 1848 * get a read lock. 1849 * => note that submaps can only appear in the kernel and they are 1850 * required to use the same virtual addresses as the map they 1851 * are referenced by (thus address translation between the main 1852 * map and the submap is unnecessary). 1853 */ 1854 1855 boolean_t 1856 uvmfault_lookup(struct uvm_faultinfo *ufi, boolean_t write_lock) 1857 { 1858 vm_map_t tmpmap; 1859 1860 /* 1861 * init ufi values for lookup. 1862 */ 1863 1864 ufi->map = ufi->orig_map; 1865 ufi->size = ufi->orig_size; 1866 1867 /* 1868 * keep going down levels until we are done. note that there can 1869 * only be two levels so we won't loop very long. 1870 */ 1871 1872 while (1) { 1873 if (ufi->orig_rvaddr < ufi->map->min_offset || 1874 ufi->orig_rvaddr >= ufi->map->max_offset) 1875 return(FALSE); 1876 1877 /* 1878 * lock map 1879 */ 1880 if (write_lock) { 1881 vm_map_lock(ufi->map); 1882 } else { 1883 vm_map_lock_read(ufi->map); 1884 } 1885 1886 /* 1887 * lookup 1888 */ 1889 if (!uvm_map_lookup_entry(ufi->map, ufi->orig_rvaddr, 1890 &ufi->entry)) { 1891 uvmfault_unlockmaps(ufi, write_lock); 1892 return(FALSE); 1893 } 1894 1895 /* 1896 * reduce size if necessary 1897 */ 1898 if (ufi->entry->end - ufi->orig_rvaddr < ufi->size) 1899 ufi->size = ufi->entry->end - ufi->orig_rvaddr; 1900 1901 /* 1902 * submap? replace map with the submap and lookup again. 1903 * note: VAs in submaps must match VAs in main map. 1904 */ 1905 if (UVM_ET_ISSUBMAP(ufi->entry)) { 1906 tmpmap = ufi->entry->object.sub_map; 1907 uvmfault_unlockmaps(ufi, write_lock); 1908 ufi->map = tmpmap; 1909 continue; 1910 } 1911 1912 /* 1913 * got it! 1914 */ 1915 1916 ufi->mapv = ufi->map->timestamp; 1917 return(TRUE); 1918 1919 } /* while loop */ 1920 1921 /*NOTREACHED*/ 1922 } 1923 1924 /* 1925 * uvmfault_relock: attempt to relock the same version of the map 1926 * 1927 * => fault data structures should be unlocked before calling. 1928 * => if a success (TRUE) maps will be locked after call. 1929 */ 1930 boolean_t 1931 uvmfault_relock(struct uvm_faultinfo *ufi) 1932 { 1933 /* 1934 * ufi can be NULL when this isn't really a fault, 1935 * but merely paging in anon data. 1936 */ 1937 1938 if (ufi == NULL) { 1939 return TRUE; 1940 } 1941 1942 uvmexp.fltrelck++; 1943 1944 /* 1945 * relock map. fail if version mismatch (in which case nothing 1946 * gets locked). 1947 */ 1948 1949 vm_map_lock_read(ufi->map); 1950 if (ufi->mapv != ufi->map->timestamp) { 1951 vm_map_unlock_read(ufi->map); 1952 return(FALSE); 1953 } 1954 1955 uvmexp.fltrelckok++; 1956 return(TRUE); /* got it! */ 1957 } 1958