1 /* $OpenBSD: uvm_fault.c,v 1.50 2008/09/16 18:52:52 chl Exp $ */ 2 /* $NetBSD: uvm_fault.c,v 1.51 2000/08/06 00:22:53 thorpej Exp $ */ 3 4 /* 5 * 6 * Copyright (c) 1997 Charles D. Cranor and Washington University. 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed by Charles D. Cranor and 20 * Washington University. 21 * 4. The name of the author may not be used to endorse or promote products 22 * derived from this software without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 25 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 26 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 27 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 28 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 29 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 30 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 31 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 32 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 33 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 34 * 35 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp 36 */ 37 38 /* 39 * uvm_fault.c: fault handler 40 */ 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/kernel.h> 45 #include <sys/proc.h> 46 #include <sys/malloc.h> 47 #include <sys/mman.h> 48 #include <sys/user.h> 49 50 #include <uvm/uvm.h> 51 52 /* 53 * 54 * a word on page faults: 55 * 56 * types of page faults we handle: 57 * 58 * CASE 1: upper layer faults CASE 2: lower layer faults 59 * 60 * CASE 1A CASE 1B CASE 2A CASE 2B 61 * read/write1 write>1 read/write +-cow_write/zero 62 * | | | | 63 * +--|--+ +--|--+ +-----+ + | + | +-----+ 64 * amap | V | | ----------->new| | | | ^ | 65 * +-----+ +-----+ +-----+ + | + | +--|--+ 66 * | | | 67 * +-----+ +-----+ +--|--+ | +--|--+ 68 * uobj | d/c | | d/c | | V | +----| | 69 * +-----+ +-----+ +-----+ +-----+ 70 * 71 * d/c = don't care 72 * 73 * case [0]: layerless fault 74 * no amap or uobj is present. this is an error. 75 * 76 * case [1]: upper layer fault [anon active] 77 * 1A: [read] or [write with anon->an_ref == 1] 78 * I/O takes place in top level anon and uobj is not touched. 79 * 1B: [write with anon->an_ref > 1] 80 * new anon is alloc'd and data is copied off ["COW"] 81 * 82 * case [2]: lower layer fault [uobj] 83 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area] 84 * I/O takes place directly in object. 85 * 2B: [write to copy_on_write] or [read on NULL uobj] 86 * data is "promoted" from uobj to a new anon. 87 * if uobj is null, then we zero fill. 88 * 89 * we follow the standard UVM locking protocol ordering: 90 * 91 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ) 92 * we hold a PG_BUSY page if we unlock for I/O 93 * 94 * 95 * the code is structured as follows: 96 * 97 * - init the "IN" params in the ufi structure 98 * ReFault: 99 * - do lookups [locks maps], check protection, handle needs_copy 100 * - check for case 0 fault (error) 101 * - establish "range" of fault 102 * - if we have an amap lock it and extract the anons 103 * - if sequential advice deactivate pages behind us 104 * - at the same time check pmap for unmapped areas and anon for pages 105 * that we could map in (and do map it if found) 106 * - check object for resident pages that we could map in 107 * - if (case 2) goto Case2 108 * - >>> handle case 1 109 * - ensure source anon is resident in RAM 110 * - if case 1B alloc new anon and copy from source 111 * - map the correct page in 112 * Case2: 113 * - >>> handle case 2 114 * - ensure source page is resident (if uobj) 115 * - if case 2B alloc new anon and copy from source (could be zero 116 * fill if uobj == NULL) 117 * - map the correct page in 118 * - done! 119 * 120 * note on paging: 121 * if we have to do I/O we place a PG_BUSY page in the correct object, 122 * unlock everything, and do the I/O. when I/O is done we must reverify 123 * the state of the world before assuming that our data structures are 124 * valid. [because mappings could change while the map is unlocked] 125 * 126 * alternative 1: unbusy the page in question and restart the page fault 127 * from the top (ReFault). this is easy but does not take advantage 128 * of the information that we already have from our previous lookup, 129 * although it is possible that the "hints" in the vm_map will help here. 130 * 131 * alternative 2: the system already keeps track of a "version" number of 132 * a map. [i.e. every time you write-lock a map (e.g. to change a 133 * mapping) you bump the version number up by one...] so, we can save 134 * the version number of the map before we release the lock and start I/O. 135 * then when I/O is done we can relock and check the version numbers 136 * to see if anything changed. this might save us some over 1 because 137 * we don't have to unbusy the page and may be less compares(?). 138 * 139 * alternative 3: put in backpointers or a way to "hold" part of a map 140 * in place while I/O is in progress. this could be complex to 141 * implement (especially with structures like amap that can be referenced 142 * by multiple map entries, and figuring out what should wait could be 143 * complex as well...). 144 * 145 * given that we are not currently multiprocessor or multithreaded we might 146 * as well choose alternative 2 now. maybe alternative 3 would be useful 147 * in the future. XXX keep in mind for future consideration//rechecking. 148 */ 149 150 /* 151 * local data structures 152 */ 153 154 struct uvm_advice { 155 int advice; 156 int nback; 157 int nforw; 158 }; 159 160 /* 161 * page range array: 162 * note: index in array must match "advice" value 163 * XXX: borrowed numbers from freebsd. do they work well for us? 164 */ 165 166 static struct uvm_advice uvmadvice[] = { 167 { MADV_NORMAL, 3, 4 }, 168 { MADV_RANDOM, 0, 0 }, 169 { MADV_SEQUENTIAL, 8, 7}, 170 }; 171 172 #define UVM_MAXRANGE 16 /* must be max() of nback+nforw+1 */ 173 174 /* 175 * private prototypes 176 */ 177 178 static void uvmfault_amapcopy(struct uvm_faultinfo *); 179 static __inline void uvmfault_anonflush(struct vm_anon **, int); 180 181 /* 182 * inline functions 183 */ 184 185 /* 186 * uvmfault_anonflush: try and deactivate pages in specified anons 187 * 188 * => does not have to deactivate page if it is busy 189 */ 190 191 static __inline void 192 uvmfault_anonflush(anons, n) 193 struct vm_anon **anons; 194 int n; 195 { 196 int lcv; 197 struct vm_page *pg; 198 199 for (lcv = 0 ; lcv < n ; lcv++) { 200 if (anons[lcv] == NULL) 201 continue; 202 simple_lock(&anons[lcv]->an_lock); 203 pg = anons[lcv]->an_page; 204 if (pg && (pg->pg_flags & PG_BUSY) == 0 && pg->loan_count == 0) { 205 uvm_lock_pageq(); 206 if (pg->wire_count == 0) { 207 #ifdef UBC 208 pmap_clear_reference(pg); 209 #else 210 pmap_page_protect(pg, VM_PROT_NONE); 211 #endif 212 uvm_pagedeactivate(pg); 213 } 214 uvm_unlock_pageq(); 215 } 216 simple_unlock(&anons[lcv]->an_lock); 217 } 218 } 219 220 /* 221 * normal functions 222 */ 223 224 /* 225 * uvmfault_amapcopy: clear "needs_copy" in a map. 226 * 227 * => called with VM data structures unlocked (usually, see below) 228 * => we get a write lock on the maps and clear needs_copy for a VA 229 * => if we are out of RAM we sleep (waiting for more) 230 */ 231 232 static void 233 uvmfault_amapcopy(ufi) 234 struct uvm_faultinfo *ufi; 235 { 236 237 /* 238 * while we haven't done the job 239 */ 240 241 while (1) { 242 243 /* 244 * no mapping? give up. 245 */ 246 247 if (uvmfault_lookup(ufi, TRUE) == FALSE) 248 return; 249 250 /* 251 * copy if needed. 252 */ 253 254 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) 255 amap_copy(ufi->map, ufi->entry, M_NOWAIT, TRUE, 256 ufi->orig_rvaddr, ufi->orig_rvaddr + 1); 257 258 /* 259 * didn't work? must be out of RAM. unlock and sleep. 260 */ 261 262 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 263 uvmfault_unlockmaps(ufi, TRUE); 264 uvm_wait("fltamapcopy"); 265 continue; 266 } 267 268 /* 269 * got it! unlock and return. 270 */ 271 272 uvmfault_unlockmaps(ufi, TRUE); 273 return; 274 } 275 /*NOTREACHED*/ 276 } 277 278 /* 279 * uvmfault_anonget: get data in an anon into a non-busy, non-released 280 * page in that anon. 281 * 282 * => maps, amap, and anon locked by caller. 283 * => if we fail (result != VM_PAGER_OK) we unlock everything. 284 * => if we are successful, we return with everything still locked. 285 * => we don't move the page on the queues [gets moved later] 286 * => if we allocate a new page [we_own], it gets put on the queues. 287 * either way, the result is that the page is on the queues at return time 288 * => for pages which are on loan from a uvm_object (and thus are not 289 * owned by the anon): if successful, we return with the owning object 290 * locked. the caller must unlock this object when it unlocks everything 291 * else. 292 */ 293 294 int 295 uvmfault_anonget(ufi, amap, anon) 296 struct uvm_faultinfo *ufi; 297 struct vm_amap *amap; 298 struct vm_anon *anon; 299 { 300 boolean_t we_own; /* we own anon's page? */ 301 boolean_t locked; /* did we relock? */ 302 struct vm_page *pg; 303 int result; 304 UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist); 305 306 result = 0; /* XXX shut up gcc */ 307 uvmexp.fltanget++; 308 /* bump rusage counters */ 309 if (anon->an_page) 310 curproc->p_addr->u_stats.p_ru.ru_minflt++; 311 else 312 curproc->p_addr->u_stats.p_ru.ru_majflt++; 313 314 /* 315 * loop until we get it, or fail. 316 */ 317 318 while (1) { 319 320 we_own = FALSE; /* TRUE if we set PG_BUSY on a page */ 321 pg = anon->an_page; 322 323 /* 324 * if there is a resident page and it is loaned, then anon 325 * may not own it. call out to uvm_anon_lockpage() to ensure 326 * the real owner of the page has been identified and locked. 327 */ 328 329 if (pg && pg->loan_count) 330 pg = uvm_anon_lockloanpg(anon); 331 332 /* 333 * page there? make sure it is not busy/released. 334 */ 335 336 if (pg) { 337 338 /* 339 * at this point, if the page has a uobject [meaning 340 * we have it on loan], then that uobject is locked 341 * by us! if the page is busy, we drop all the 342 * locks (including uobject) and try again. 343 */ 344 345 if ((pg->pg_flags & (PG_BUSY|PG_RELEASED)) == 0) { 346 UVMHIST_LOG(maphist, "<- OK",0,0,0,0); 347 return (VM_PAGER_OK); 348 } 349 atomic_setbits_int(&pg->pg_flags, PG_WANTED); 350 uvmexp.fltpgwait++; 351 352 /* 353 * the last unlock must be an atomic unlock+wait on 354 * the owner of page 355 */ 356 if (pg->uobject) { /* owner is uobject ? */ 357 uvmfault_unlockall(ufi, amap, NULL, anon); 358 UVMHIST_LOG(maphist, " unlock+wait on uobj",0, 359 0,0,0); 360 UVM_UNLOCK_AND_WAIT(pg, 361 &pg->uobject->vmobjlock, 362 FALSE, "anonget1",0); 363 } else { 364 /* anon owns page */ 365 uvmfault_unlockall(ufi, amap, NULL, NULL); 366 UVMHIST_LOG(maphist, " unlock+wait on anon",0, 367 0,0,0); 368 UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0, 369 "anonget2",0); 370 } 371 /* ready to relock and try again */ 372 373 } else { 374 375 /* 376 * no page, we must try and bring it in. 377 */ 378 pg = uvm_pagealloc(NULL, 0, anon, 0); 379 380 if (pg == NULL) { /* out of RAM. */ 381 382 uvmfault_unlockall(ufi, amap, NULL, anon); 383 uvmexp.fltnoram++; 384 UVMHIST_LOG(maphist, " noram -- UVM_WAIT",0, 385 0,0,0); 386 uvm_wait("flt_noram1"); 387 /* ready to relock and try again */ 388 389 } else { 390 391 /* we set the PG_BUSY bit */ 392 we_own = TRUE; 393 uvmfault_unlockall(ufi, amap, NULL, anon); 394 395 /* 396 * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN 397 * page into the uvm_swap_get function with 398 * all data structures unlocked. note that 399 * it is ok to read an_swslot here because 400 * we hold PG_BUSY on the page. 401 */ 402 uvmexp.pageins++; 403 result = uvm_swap_get(pg, anon->an_swslot, 404 PGO_SYNCIO); 405 406 /* 407 * we clean up after the i/o below in the 408 * "we_own" case 409 */ 410 /* ready to relock and try again */ 411 } 412 } 413 414 /* 415 * now relock and try again 416 */ 417 418 locked = uvmfault_relock(ufi); 419 if (locked || we_own) 420 simple_lock(&anon->an_lock); 421 422 /* 423 * if we own the page (i.e. we set PG_BUSY), then we need 424 * to clean up after the I/O. there are three cases to 425 * consider: 426 * [1] page released during I/O: free anon and ReFault. 427 * [2] I/O not OK. free the page and cause the fault 428 * to fail. 429 * [3] I/O OK! activate the page and sync with the 430 * non-we_own case (i.e. drop anon lock if not locked). 431 */ 432 433 if (we_own) { 434 435 if (pg->pg_flags & PG_WANTED) { 436 /* still holding object lock */ 437 wakeup(pg); 438 } 439 /* un-busy! */ 440 atomic_clearbits_int(&pg->pg_flags, 441 PG_WANTED|PG_BUSY|PG_FAKE); 442 UVM_PAGE_OWN(pg, NULL); 443 444 /* 445 * if we were RELEASED during I/O, then our anon is 446 * no longer part of an amap. we need to free the 447 * anon and try again. 448 */ 449 if (pg->pg_flags & PG_RELEASED) { 450 pmap_page_protect(pg, VM_PROT_NONE); 451 simple_unlock(&anon->an_lock); 452 uvm_anfree(anon); /* frees page for us */ 453 if (locked) 454 uvmfault_unlockall(ufi, amap, NULL, 455 NULL); 456 uvmexp.fltpgrele++; 457 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 458 return (VM_PAGER_REFAULT); /* refault! */ 459 } 460 461 if (result != VM_PAGER_OK) { 462 KASSERT(result != VM_PAGER_PEND); 463 464 /* remove page from anon */ 465 anon->an_page = NULL; 466 467 /* 468 * remove the swap slot from the anon 469 * and mark the anon as having no real slot. 470 * don't free the swap slot, thus preventing 471 * it from being used again. 472 */ 473 uvm_swap_markbad(anon->an_swslot, 1); 474 anon->an_swslot = SWSLOT_BAD; 475 476 /* 477 * note: page was never !PG_BUSY, so it 478 * can't be mapped and thus no need to 479 * pmap_page_protect it... 480 */ 481 uvm_lock_pageq(); 482 uvm_pagefree(pg); 483 uvm_unlock_pageq(); 484 485 if (locked) 486 uvmfault_unlockall(ufi, amap, NULL, 487 anon); 488 else 489 simple_unlock(&anon->an_lock); 490 UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0); 491 return (VM_PAGER_ERROR); 492 } 493 494 /* 495 * must be OK, clear modify (already PG_CLEAN) 496 * and activate 497 */ 498 pmap_clear_modify(pg); 499 uvm_lock_pageq(); 500 uvm_pageactivate(pg); 501 uvm_unlock_pageq(); 502 if (!locked) 503 simple_unlock(&anon->an_lock); 504 } 505 506 /* 507 * we were not able to relock. restart fault. 508 */ 509 510 if (!locked) { 511 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 512 return (VM_PAGER_REFAULT); 513 } 514 515 /* 516 * verify no one has touched the amap and moved the anon on us. 517 */ 518 519 if (ufi != NULL && 520 amap_lookup(&ufi->entry->aref, 521 ufi->orig_rvaddr - ufi->entry->start) != anon) { 522 523 uvmfault_unlockall(ufi, amap, NULL, anon); 524 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 525 return (VM_PAGER_REFAULT); 526 } 527 528 /* 529 * try it again! 530 */ 531 532 uvmexp.fltanretry++; 533 continue; 534 535 } /* while (1) */ 536 537 /*NOTREACHED*/ 538 } 539 540 /* 541 * F A U L T - m a i n e n t r y p o i n t 542 */ 543 544 /* 545 * uvm_fault: page fault handler 546 * 547 * => called from MD code to resolve a page fault 548 * => VM data structures usually should be unlocked. however, it is 549 * possible to call here with the main map locked if the caller 550 * gets a write lock, sets it recursive, and then calls us (c.f. 551 * uvm_map_pageable). this should be avoided because it keeps 552 * the map locked off during I/O. 553 */ 554 555 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 556 ~VM_PROT_WRITE : VM_PROT_ALL) 557 558 int 559 uvm_fault(orig_map, vaddr, fault_type, access_type) 560 vm_map_t orig_map; 561 vaddr_t vaddr; 562 vm_fault_t fault_type; 563 vm_prot_t access_type; 564 { 565 struct uvm_faultinfo ufi; 566 vm_prot_t enter_prot; 567 boolean_t wired, narrow, promote, locked, shadowed; 568 int npages, nback, nforw, centeridx, result, lcv, gotpages; 569 vaddr_t startva, currva; 570 voff_t uoff; 571 paddr_t pa; 572 struct vm_amap *amap; 573 struct uvm_object *uobj; 574 struct vm_anon *anons_store[UVM_MAXRANGE], **anons, *anon, *oanon; 575 struct vm_page *pages[UVM_MAXRANGE], *pg, *uobjpage; 576 UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist); 577 578 UVMHIST_LOG(maphist, "(map=%p, vaddr=0x%lx, ft=%ld, at=%ld)", 579 orig_map, vaddr, fault_type, access_type); 580 581 anon = NULL; 582 pg = NULL; 583 584 uvmexp.faults++; /* XXX: locking? */ 585 586 /* 587 * init the IN parameters in the ufi 588 */ 589 590 ufi.orig_map = orig_map; 591 ufi.orig_rvaddr = trunc_page(vaddr); 592 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ 593 if (fault_type == VM_FAULT_WIRE) 594 narrow = TRUE; /* don't look for neighborhood 595 * pages on wire */ 596 else 597 narrow = FALSE; /* normal fault */ 598 599 /* 600 * "goto ReFault" means restart the page fault from ground zero. 601 */ 602 ReFault: 603 604 /* 605 * lookup and lock the maps 606 */ 607 608 if (uvmfault_lookup(&ufi, FALSE) == FALSE) { 609 UVMHIST_LOG(maphist, "<- no mapping @ 0x%lx", vaddr, 0,0,0); 610 return (EFAULT); 611 } 612 /* locked: maps(read) */ 613 614 #ifdef DIAGNOSTIC 615 if ((ufi.map->flags & VM_MAP_PAGEABLE) == 0) 616 panic("uvm_fault: fault on non-pageable map (%p, 0x%lx)", 617 ufi.map, vaddr); 618 #endif 619 620 /* 621 * check protection 622 */ 623 624 if ((ufi.entry->protection & access_type) != access_type) { 625 UVMHIST_LOG(maphist, 626 "<- protection failure (prot=0x%lx, access=0x%lx)", 627 ufi.entry->protection, access_type, 0, 0); 628 uvmfault_unlockmaps(&ufi, FALSE); 629 return (EACCES); 630 } 631 632 /* 633 * "enter_prot" is the protection we want to enter the page in at. 634 * for certain pages (e.g. copy-on-write pages) this protection can 635 * be more strict than ufi.entry->protection. "wired" means either 636 * the entry is wired or we are fault-wiring the pg. 637 */ 638 639 enter_prot = ufi.entry->protection; 640 wired = VM_MAPENT_ISWIRED(ufi.entry) || (fault_type == VM_FAULT_WIRE); 641 if (wired) 642 access_type = enter_prot; /* full access for wired */ 643 644 /* 645 * handle "needs_copy" case. if we need to copy the amap we will 646 * have to drop our readlock and relock it with a write lock. (we 647 * need a write lock to change anything in a map entry [e.g. 648 * needs_copy]). 649 */ 650 651 if (UVM_ET_ISNEEDSCOPY(ufi.entry)) { 652 if ((access_type & VM_PROT_WRITE) || 653 (ufi.entry->object.uvm_obj == NULL)) { 654 /* need to clear */ 655 UVMHIST_LOG(maphist, 656 " need to clear needs_copy and refault",0,0,0,0); 657 uvmfault_unlockmaps(&ufi, FALSE); 658 uvmfault_amapcopy(&ufi); 659 uvmexp.fltamcopy++; 660 goto ReFault; 661 662 } else { 663 664 /* 665 * ensure that we pmap_enter page R/O since 666 * needs_copy is still true 667 */ 668 enter_prot &= ~VM_PROT_WRITE; 669 670 } 671 } 672 673 /* 674 * identify the players 675 */ 676 677 amap = ufi.entry->aref.ar_amap; /* top layer */ 678 uobj = ufi.entry->object.uvm_obj; /* bottom layer */ 679 680 /* 681 * check for a case 0 fault. if nothing backing the entry then 682 * error now. 683 */ 684 685 if (amap == NULL && uobj == NULL) { 686 uvmfault_unlockmaps(&ufi, FALSE); 687 UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0); 688 return (EFAULT); 689 } 690 691 /* 692 * establish range of interest based on advice from mapper 693 * and then clip to fit map entry. note that we only want 694 * to do this the first time through the fault. if we 695 * ReFault we will disable this by setting "narrow" to true. 696 */ 697 698 if (narrow == FALSE) { 699 700 /* wide fault (!narrow) */ 701 KASSERT(uvmadvice[ufi.entry->advice].advice == 702 ufi.entry->advice); 703 nback = min(uvmadvice[ufi.entry->advice].nback, 704 (ufi.orig_rvaddr - ufi.entry->start) >> PAGE_SHIFT); 705 startva = ufi.orig_rvaddr - (nback << PAGE_SHIFT); 706 nforw = min(uvmadvice[ufi.entry->advice].nforw, 707 ((ufi.entry->end - ufi.orig_rvaddr) >> 708 PAGE_SHIFT) - 1); 709 /* 710 * note: "-1" because we don't want to count the 711 * faulting page as forw 712 */ 713 npages = nback + nforw + 1; 714 centeridx = nback; 715 716 narrow = TRUE; /* ensure only once per-fault */ 717 718 } else { 719 720 /* narrow fault! */ 721 nback = nforw = 0; 722 startva = ufi.orig_rvaddr; 723 npages = 1; 724 centeridx = 0; 725 726 } 727 728 /* locked: maps(read) */ 729 UVMHIST_LOG(maphist, " narrow=%ld, back=%ld, forw=%ld, startva=0x%lx", 730 narrow, nback, nforw, startva); 731 UVMHIST_LOG(maphist, " entry=%p, amap=%p, obj=%p", ufi.entry, 732 amap, uobj, 0); 733 734 /* 735 * if we've got an amap, lock it and extract current anons. 736 */ 737 738 if (amap) { 739 anons = anons_store; 740 amap_lookups(&ufi.entry->aref, startva - ufi.entry->start, 741 anons, npages); 742 } else { 743 anons = NULL; /* to be safe */ 744 } 745 746 /* locked: maps(read), amap(if there) */ 747 748 /* 749 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages 750 * now and then forget about them (for the rest of the fault). 751 */ 752 753 if (ufi.entry->advice == MADV_SEQUENTIAL && nback != 0) { 754 755 UVMHIST_LOG(maphist, " MADV_SEQUENTIAL: flushing backpages", 756 0,0,0,0); 757 /* flush back-page anons? */ 758 if (amap) 759 uvmfault_anonflush(anons, nback); 760 761 /* flush object? */ 762 if (uobj) { 763 uoff = (startva - ufi.entry->start) + ufi.entry->offset; 764 simple_lock(&uobj->vmobjlock); 765 (void) uobj->pgops->pgo_flush(uobj, uoff, uoff + 766 (nback << PAGE_SHIFT), PGO_DEACTIVATE); 767 simple_unlock(&uobj->vmobjlock); 768 } 769 770 /* now forget about the backpages */ 771 if (amap) 772 anons += nback; 773 startva += (nback << PAGE_SHIFT); 774 npages -= nback; 775 centeridx = 0; 776 } 777 778 /* locked: maps(read), amap(if there) */ 779 780 /* 781 * map in the backpages and frontpages we found in the amap in hopes 782 * of preventing future faults. we also init the pages[] array as 783 * we go. 784 */ 785 786 currva = startva; 787 shadowed = FALSE; 788 for (lcv = 0 ; lcv < npages ; lcv++, currva += PAGE_SIZE) { 789 790 /* 791 * dont play with VAs that are already mapped 792 * except for center) 793 */ 794 if (lcv != centeridx && 795 pmap_extract(ufi.orig_map->pmap, currva, &pa)) { 796 pages[lcv] = PGO_DONTCARE; 797 continue; 798 } 799 800 /* 801 * unmapped or center page. check if any anon at this level. 802 */ 803 if (amap == NULL || anons[lcv] == NULL) { 804 pages[lcv] = NULL; 805 continue; 806 } 807 808 /* 809 * check for present page and map if possible. re-activate it. 810 */ 811 812 pages[lcv] = PGO_DONTCARE; 813 if (lcv == centeridx) { /* save center for later! */ 814 shadowed = TRUE; 815 continue; 816 } 817 anon = anons[lcv]; 818 simple_lock(&anon->an_lock); 819 /* ignore loaned pages */ 820 if (anon->an_page && anon->an_page->loan_count == 0 && 821 (anon->an_page->pg_flags & (PG_RELEASED|PG_BUSY)) == 0) { 822 uvm_lock_pageq(); 823 uvm_pageactivate(anon->an_page); /* reactivate */ 824 uvm_unlock_pageq(); 825 UVMHIST_LOG(maphist, 826 " MAPPING: n anon: pm=%p, va=0x%lx, pg=%p", 827 ufi.orig_map->pmap, currva, anon->an_page, 0); 828 uvmexp.fltnamap++; 829 830 /* 831 * Since this isn't the page that's actually faulting, 832 * ignore pmap_enter() failures; it's not critical 833 * that we enter these right now. 834 */ 835 836 (void) pmap_enter(ufi.orig_map->pmap, currva, 837 VM_PAGE_TO_PHYS(anon->an_page), 838 (anon->an_ref > 1) ? (enter_prot & ~VM_PROT_WRITE) : 839 enter_prot, 840 PMAP_CANFAIL | 841 (VM_MAPENT_ISWIRED(ufi.entry) ? PMAP_WIRED : 0)); 842 } 843 simple_unlock(&anon->an_lock); 844 pmap_update(ufi.orig_map->pmap); 845 } 846 847 /* locked: maps(read), amap(if there) */ 848 /* (shadowed == TRUE) if there is an anon at the faulting address */ 849 UVMHIST_LOG(maphist, " shadowed=%ld, will_get=%ld", shadowed, 850 (uobj && shadowed == FALSE),0,0); 851 852 /* 853 * note that if we are really short of RAM we could sleep in the above 854 * call to pmap_enter with everything locked. bad? 855 * 856 * XXX Actually, that is bad; pmap_enter() should just fail in that 857 * XXX case. --thorpej 858 */ 859 860 /* 861 * if the desired page is not shadowed by the amap and we have a 862 * backing object, then we check to see if the backing object would 863 * prefer to handle the fault itself (rather than letting us do it 864 * with the usual pgo_get hook). the backing object signals this by 865 * providing a pgo_fault routine. 866 */ 867 868 if (uobj && shadowed == FALSE && uobj->pgops->pgo_fault != NULL) { 869 simple_lock(&uobj->vmobjlock); 870 871 /* locked: maps(read), amap (if there), uobj */ 872 result = uobj->pgops->pgo_fault(&ufi, startva, pages, npages, 873 centeridx, fault_type, access_type, 874 PGO_LOCKED); 875 876 /* locked: nothing, pgo_fault has unlocked everything */ 877 878 if (result == VM_PAGER_OK) 879 return (0); /* pgo_fault did pmap enter */ 880 else if (result == VM_PAGER_REFAULT) 881 goto ReFault; /* try again! */ 882 else 883 return (EACCES); 884 } 885 886 /* 887 * now, if the desired page is not shadowed by the amap and we have 888 * a backing object that does not have a special fault routine, then 889 * we ask (with pgo_get) the object for resident pages that we care 890 * about and attempt to map them in. we do not let pgo_get block 891 * (PGO_LOCKED). 892 * 893 * ("get" has the option of doing a pmap_enter for us) 894 */ 895 896 if (uobj && shadowed == FALSE) { 897 simple_lock(&uobj->vmobjlock); 898 899 /* locked (!shadowed): maps(read), amap (if there), uobj */ 900 /* 901 * the following call to pgo_get does _not_ change locking state 902 */ 903 904 uvmexp.fltlget++; 905 gotpages = npages; 906 (void) uobj->pgops->pgo_get(uobj, ufi.entry->offset + 907 (startva - ufi.entry->start), 908 pages, &gotpages, centeridx, 909 access_type & MASK(ufi.entry), 910 ufi.entry->advice, PGO_LOCKED); 911 912 /* 913 * check for pages to map, if we got any 914 */ 915 916 uobjpage = NULL; 917 918 if (gotpages) { 919 currva = startva; 920 for (lcv = 0 ; lcv < npages ; 921 lcv++, currva += PAGE_SIZE) { 922 923 if (pages[lcv] == NULL || 924 pages[lcv] == PGO_DONTCARE) 925 continue; 926 927 KASSERT((pages[lcv]->pg_flags & PG_RELEASED) == 0); 928 929 /* 930 * if center page is resident and not 931 * PG_BUSY|PG_RELEASED then pgo_get 932 * made it PG_BUSY for us and gave 933 * us a handle to it. remember this 934 * page as "uobjpage." (for later use). 935 */ 936 937 if (lcv == centeridx) { 938 uobjpage = pages[lcv]; 939 UVMHIST_LOG(maphist, " got uobjpage " 940 "(%p) with locked get", 941 uobjpage, 0,0,0); 942 continue; 943 } 944 945 /* 946 * note: calling pgo_get with locked data 947 * structures returns us pages which are 948 * neither busy nor released, so we don't 949 * need to check for this. we can just 950 * directly enter the page (after moving it 951 * to the head of the active queue [useful?]). 952 */ 953 954 uvm_lock_pageq(); 955 uvm_pageactivate(pages[lcv]); /* reactivate */ 956 uvm_unlock_pageq(); 957 UVMHIST_LOG(maphist, 958 " MAPPING: n obj: pm=%p, va=0x%lx, pg=%p", 959 ufi.orig_map->pmap, currva, pages[lcv], 0); 960 uvmexp.fltnomap++; 961 962 /* 963 * Since this page isn't the page that's 964 * actually fauling, ignore pmap_enter() 965 * failures; it's not critical that we 966 * enter these right now. 967 */ 968 969 (void) pmap_enter(ufi.orig_map->pmap, currva, 970 VM_PAGE_TO_PHYS(pages[lcv]), 971 enter_prot & MASK(ufi.entry), 972 PMAP_CANFAIL | 973 (wired ? PMAP_WIRED : 0)); 974 975 /* 976 * NOTE: page can't be PG_WANTED or PG_RELEASED 977 * because we've held the lock the whole time 978 * we've had the handle. 979 */ 980 981 atomic_clearbits_int(&pages[lcv]->pg_flags, 982 PG_BUSY); 983 UVM_PAGE_OWN(pages[lcv], NULL); 984 } /* for "lcv" loop */ 985 pmap_update(ufi.orig_map->pmap); 986 } /* "gotpages" != 0 */ 987 /* note: object still _locked_ */ 988 } else { 989 uobjpage = NULL; 990 } 991 992 /* locked (shadowed): maps(read), amap */ 993 /* locked (!shadowed): maps(read), amap(if there), 994 uobj(if !null), uobjpage(if !null) */ 995 996 /* 997 * note that at this point we are done with any front or back pages. 998 * we are now going to focus on the center page (i.e. the one we've 999 * faulted on). if we have faulted on the top (anon) layer 1000 * [i.e. case 1], then the anon we want is anons[centeridx] (we have 1001 * not touched it yet). if we have faulted on the bottom (uobj) 1002 * layer [i.e. case 2] and the page was both present and available, 1003 * then we've got a pointer to it as "uobjpage" and we've already 1004 * made it BUSY. 1005 */ 1006 1007 /* 1008 * there are four possible cases we must address: 1A, 1B, 2A, and 2B 1009 */ 1010 1011 /* 1012 * redirect case 2: if we are not shadowed, go to case 2. 1013 */ 1014 1015 if (shadowed == FALSE) 1016 goto Case2; 1017 1018 /* locked: maps(read), amap */ 1019 1020 /* 1021 * handle case 1: fault on an anon in our amap 1022 */ 1023 1024 anon = anons[centeridx]; 1025 UVMHIST_LOG(maphist, " case 1 fault: anon=%p", anon, 0,0,0); 1026 simple_lock(&anon->an_lock); 1027 1028 /* locked: maps(read), amap, anon */ 1029 1030 /* 1031 * no matter if we have case 1A or case 1B we are going to need to 1032 * have the anon's memory resident. ensure that now. 1033 */ 1034 1035 /* 1036 * let uvmfault_anonget do the dirty work. 1037 * if it fails (!OK) it will unlock everything for us. 1038 * if it succeeds, locks are still valid and locked. 1039 * also, if it is OK, then the anon's page is on the queues. 1040 * if the page is on loan from a uvm_object, then anonget will 1041 * lock that object for us if it does not fail. 1042 */ 1043 1044 result = uvmfault_anonget(&ufi, amap, anon); 1045 switch (result) { 1046 case VM_PAGER_OK: 1047 break; 1048 1049 case VM_PAGER_REFAULT: 1050 goto ReFault; 1051 1052 case VM_PAGER_ERROR: 1053 /* 1054 * An error occured while trying to bring in the 1055 * page -- this is the only error we return right 1056 * now. 1057 */ 1058 return (EACCES); /* XXX */ 1059 1060 default: 1061 #ifdef DIAGNOSTIC 1062 panic("uvm_fault: uvmfault_anonget -> %d", result); 1063 #else 1064 return (EACCES); 1065 #endif 1066 } 1067 1068 /* 1069 * uobj is non null if the page is on loan from an object (i.e. uobj) 1070 */ 1071 1072 uobj = anon->an_page->uobject; /* locked by anonget if !NULL */ 1073 1074 /* locked: maps(read), amap, anon, uobj(if one) */ 1075 1076 /* 1077 * special handling for loaned pages 1078 */ 1079 1080 if (anon->an_page->loan_count) { 1081 1082 if ((access_type & VM_PROT_WRITE) == 0) { 1083 1084 /* 1085 * for read faults on loaned pages we just cap the 1086 * protection at read-only. 1087 */ 1088 1089 enter_prot = enter_prot & ~VM_PROT_WRITE; 1090 1091 } else { 1092 /* 1093 * note that we can't allow writes into a loaned page! 1094 * 1095 * if we have a write fault on a loaned page in an 1096 * anon then we need to look at the anon's ref count. 1097 * if it is greater than one then we are going to do 1098 * a normal copy-on-write fault into a new anon (this 1099 * is not a problem). however, if the reference count 1100 * is one (a case where we would normally allow a 1101 * write directly to the page) then we need to kill 1102 * the loan before we continue. 1103 */ 1104 1105 /* >1 case is already ok */ 1106 if (anon->an_ref == 1) { 1107 1108 /* get new un-owned replacement page */ 1109 pg = uvm_pagealloc(NULL, 0, NULL, 0); 1110 if (pg == NULL) { 1111 uvmfault_unlockall(&ufi, amap, uobj, 1112 anon); 1113 uvm_wait("flt_noram2"); 1114 goto ReFault; 1115 } 1116 1117 /* 1118 * copy data, kill loan, and drop uobj lock 1119 * (if any) 1120 */ 1121 /* copy old -> new */ 1122 uvm_pagecopy(anon->an_page, pg); 1123 1124 /* force reload */ 1125 pmap_page_protect(anon->an_page, 1126 VM_PROT_NONE); 1127 uvm_lock_pageq(); /* KILL loan */ 1128 if (uobj) 1129 /* if we were loaning */ 1130 anon->an_page->loan_count--; 1131 anon->an_page->uanon = NULL; 1132 /* in case we owned */ 1133 atomic_clearbits_int( 1134 &anon->an_page->pg_flags, PQ_ANON); 1135 uvm_pageactivate(pg); 1136 uvm_unlock_pageq(); 1137 if (uobj) { 1138 simple_unlock(&uobj->vmobjlock); 1139 uobj = NULL; 1140 } 1141 1142 /* install new page in anon */ 1143 anon->an_page = pg; 1144 pg->uanon = anon; 1145 atomic_setbits_int(&pg->pg_flags, PQ_ANON); 1146 atomic_clearbits_int(&pg->pg_flags, 1147 PG_BUSY|PG_FAKE); 1148 UVM_PAGE_OWN(pg, NULL); 1149 1150 /* done! */ 1151 } /* ref == 1 */ 1152 } /* write fault */ 1153 } /* loan count */ 1154 1155 /* 1156 * if we are case 1B then we will need to allocate a new blank 1157 * anon to transfer the data into. note that we have a lock 1158 * on anon, so no one can busy or release the page until we are done. 1159 * also note that the ref count can't drop to zero here because 1160 * it is > 1 and we are only dropping one ref. 1161 * 1162 * in the (hopefully very rare) case that we are out of RAM we 1163 * will unlock, wait for more RAM, and refault. 1164 * 1165 * if we are out of anon VM we kill the process (XXX: could wait?). 1166 */ 1167 1168 if ((access_type & VM_PROT_WRITE) != 0 && anon->an_ref > 1) { 1169 1170 UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0); 1171 uvmexp.flt_acow++; 1172 oanon = anon; /* oanon = old, locked anon */ 1173 anon = uvm_analloc(); 1174 if (anon) { 1175 pg = uvm_pagealloc(NULL, 0, anon, 0); 1176 } 1177 1178 /* check for out of RAM */ 1179 if (anon == NULL || pg == NULL) { 1180 if (anon) 1181 uvm_anfree(anon); 1182 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1183 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1184 if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) { 1185 UVMHIST_LOG(maphist, 1186 "<- failed. out of VM",0,0,0,0); 1187 uvmexp.fltnoanon++; 1188 return (ENOMEM); 1189 } 1190 1191 uvmexp.fltnoram++; 1192 uvm_wait("flt_noram3"); /* out of RAM, wait for more */ 1193 goto ReFault; 1194 } 1195 1196 /* got all resources, replace anon with nanon */ 1197 1198 uvm_pagecopy(oanon->an_page, pg); /* pg now !PG_CLEAN */ 1199 /* un-busy! new page */ 1200 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE); 1201 UVM_PAGE_OWN(pg, NULL); 1202 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start, 1203 anon, 1); 1204 1205 /* deref: can not drop to zero here by defn! */ 1206 oanon->an_ref--; 1207 1208 /* 1209 * note: oanon still locked. anon is _not_ locked, but we 1210 * have the sole references to in from amap which _is_ locked. 1211 * thus, no one can get at it until we are done with it. 1212 */ 1213 1214 } else { 1215 1216 uvmexp.flt_anon++; 1217 oanon = anon; /* old, locked anon is same as anon */ 1218 pg = anon->an_page; 1219 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */ 1220 enter_prot = enter_prot & ~VM_PROT_WRITE; 1221 1222 } 1223 1224 /* locked: maps(read), amap, oanon */ 1225 1226 /* 1227 * now map the page in ... 1228 * XXX: old fault unlocks object before pmap_enter. this seems 1229 * suspect since some other thread could blast the page out from 1230 * under us between the unlock and the pmap_enter. 1231 */ 1232 1233 UVMHIST_LOG(maphist, " MAPPING: anon: pm=%p, va=0x%lx, pg=%p", 1234 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, 0); 1235 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1236 enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) 1237 != 0) { 1238 /* 1239 * No need to undo what we did; we can simply think of 1240 * this as the pmap throwing away the mapping information. 1241 * 1242 * We do, however, have to go through the ReFault path, 1243 * as the map may change while we're asleep. 1244 */ 1245 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1246 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1247 if (uvmexp.swpgonly == uvmexp.swpages) { 1248 UVMHIST_LOG(maphist, 1249 "<- failed. out of VM",0,0,0,0); 1250 /* XXX instrumentation */ 1251 return (ENOMEM); 1252 } 1253 /* XXX instrumentation */ 1254 uvm_wait("flt_pmfail1"); 1255 goto ReFault; 1256 } 1257 1258 /* 1259 * ... update the page queues. 1260 */ 1261 1262 uvm_lock_pageq(); 1263 1264 if (fault_type == VM_FAULT_WIRE) { 1265 uvm_pagewire(pg); 1266 1267 /* 1268 * since the now-wired page cannot be paged out, 1269 * release its swap resources for others to use. 1270 * since an anon with no swap cannot be PG_CLEAN, 1271 * clear its clean flag now. 1272 */ 1273 atomic_clearbits_int(&pg->pg_flags, PG_CLEAN); 1274 uvm_anon_dropswap(anon); 1275 } else { 1276 /* activate it */ 1277 uvm_pageactivate(pg); 1278 } 1279 1280 uvm_unlock_pageq(); 1281 1282 /* 1283 * done case 1! finish up by unlocking everything and returning success 1284 */ 1285 1286 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1287 pmap_update(ufi.orig_map->pmap); 1288 return (0); 1289 1290 1291 Case2: 1292 /* 1293 * handle case 2: faulting on backing object or zero fill 1294 */ 1295 1296 /* 1297 * locked: 1298 * maps(read), amap(if there), uobj(if !null), uobjpage(if !null) 1299 */ 1300 1301 /* 1302 * note that uobjpage can not be PGO_DONTCARE at this point. we now 1303 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we 1304 * have a backing object, check and see if we are going to promote 1305 * the data up to an anon during the fault. 1306 */ 1307 1308 if (uobj == NULL) { 1309 uobjpage = PGO_DONTCARE; 1310 promote = TRUE; /* always need anon here */ 1311 } else { 1312 KASSERT(uobjpage != PGO_DONTCARE); 1313 promote = (access_type & VM_PROT_WRITE) && 1314 UVM_ET_ISCOPYONWRITE(ufi.entry); 1315 } 1316 UVMHIST_LOG(maphist, " case 2 fault: promote=%ld, zfill=%ld", 1317 promote, (uobj == NULL), 0,0); 1318 1319 /* 1320 * if uobjpage is not null then we do not need to do I/O to get the 1321 * uobjpage. 1322 * 1323 * if uobjpage is null, then we need to unlock and ask the pager to 1324 * get the data for us. once we have the data, we need to reverify 1325 * the state the world. we are currently not holding any resources. 1326 */ 1327 1328 if (uobjpage) { 1329 /* update rusage counters */ 1330 curproc->p_addr->u_stats.p_ru.ru_minflt++; 1331 } else { 1332 /* update rusage counters */ 1333 curproc->p_addr->u_stats.p_ru.ru_majflt++; 1334 1335 /* locked: maps(read), amap(if there), uobj */ 1336 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1337 /* locked: uobj */ 1338 1339 uvmexp.fltget++; 1340 gotpages = 1; 1341 uoff = (ufi.orig_rvaddr - ufi.entry->start) + ufi.entry->offset; 1342 result = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages, 1343 0, access_type & MASK(ufi.entry), ufi.entry->advice, 1344 PGO_SYNCIO); 1345 1346 /* locked: uobjpage(if result OK) */ 1347 1348 /* 1349 * recover from I/O 1350 */ 1351 1352 if (result != VM_PAGER_OK) { 1353 KASSERT(result != VM_PAGER_PEND); 1354 1355 if (result == VM_PAGER_AGAIN) { 1356 UVMHIST_LOG(maphist, 1357 " pgo_get says TRY AGAIN!",0,0,0,0); 1358 tsleep((caddr_t)&lbolt, PVM, "fltagain2", 0); 1359 goto ReFault; 1360 } 1361 1362 UVMHIST_LOG(maphist, "<- pgo_get failed (code %ld)", 1363 result, 0,0,0); 1364 return (EACCES); /* XXX i/o error */ 1365 } 1366 1367 /* locked: uobjpage */ 1368 1369 /* 1370 * re-verify the state of the world by first trying to relock 1371 * the maps. always relock the object. 1372 */ 1373 1374 locked = uvmfault_relock(&ufi); 1375 simple_lock(&uobj->vmobjlock); 1376 1377 /* locked(locked): maps(read), amap(if !null), uobj, uobjpage */ 1378 /* locked(!locked): uobj, uobjpage */ 1379 1380 /* 1381 * verify that the page has not be released and re-verify 1382 * that amap slot is still free. if there is a problem, 1383 * we unlock and clean up. 1384 */ 1385 1386 if ((uobjpage->pg_flags & PG_RELEASED) != 0 || 1387 (locked && amap && 1388 amap_lookup(&ufi.entry->aref, 1389 ufi.orig_rvaddr - ufi.entry->start))) { 1390 if (locked) 1391 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1392 locked = FALSE; 1393 } 1394 1395 /* 1396 * didn't get the lock? release the page and retry. 1397 */ 1398 1399 if (locked == FALSE) { 1400 1401 UVMHIST_LOG(maphist, 1402 " wasn't able to relock after fault: retry", 1403 0,0,0,0); 1404 if (uobjpage->pg_flags & PG_WANTED) 1405 /* still holding object lock */ 1406 wakeup(uobjpage); 1407 1408 if (uobjpage->pg_flags & PG_RELEASED) { 1409 uvmexp.fltpgrele++; 1410 KASSERT(uobj->pgops->pgo_releasepg != NULL); 1411 1412 /* frees page */ 1413 if (uobj->pgops->pgo_releasepg(uobjpage,NULL)) 1414 /* unlock if still alive */ 1415 simple_unlock(&uobj->vmobjlock); 1416 goto ReFault; 1417 } 1418 1419 uvm_lock_pageq(); 1420 /* make sure it is in queues */ 1421 uvm_pageactivate(uobjpage); 1422 1423 uvm_unlock_pageq(); 1424 atomic_clearbits_int(&uobjpage->pg_flags, 1425 PG_BUSY|PG_WANTED); 1426 UVM_PAGE_OWN(uobjpage, NULL); 1427 simple_unlock(&uobj->vmobjlock); 1428 goto ReFault; 1429 1430 } 1431 1432 /* 1433 * we have the data in uobjpage which is PG_BUSY and 1434 * !PG_RELEASED. we are holding object lock (so the page 1435 * can't be released on us). 1436 */ 1437 1438 /* locked: maps(read), amap(if !null), uobj, uobjpage */ 1439 } 1440 1441 /* 1442 * locked: 1443 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj) 1444 */ 1445 1446 /* 1447 * notes: 1448 * - at this point uobjpage can not be NULL 1449 * - at this point uobjpage can not be PG_RELEASED (since we checked 1450 * for it above) 1451 * - at this point uobjpage could be PG_WANTED (handle later) 1452 */ 1453 1454 if (promote == FALSE) { 1455 1456 /* 1457 * we are not promoting. if the mapping is COW ensure that we 1458 * don't give more access than we should (e.g. when doing a read 1459 * fault on a COPYONWRITE mapping we want to map the COW page in 1460 * R/O even though the entry protection could be R/W). 1461 * 1462 * set "pg" to the page we want to map in (uobjpage, usually) 1463 */ 1464 1465 uvmexp.flt_obj++; 1466 if (UVM_ET_ISCOPYONWRITE(ufi.entry)) 1467 enter_prot &= ~VM_PROT_WRITE; 1468 pg = uobjpage; /* map in the actual object */ 1469 1470 /* assert(uobjpage != PGO_DONTCARE) */ 1471 1472 /* 1473 * we are faulting directly on the page. be careful 1474 * about writing to loaned pages... 1475 */ 1476 if (uobjpage->loan_count) { 1477 1478 if ((access_type & VM_PROT_WRITE) == 0) { 1479 /* read fault: cap the protection at readonly */ 1480 /* cap! */ 1481 enter_prot = enter_prot & ~VM_PROT_WRITE; 1482 } else { 1483 /* write fault: must break the loan here */ 1484 1485 /* alloc new un-owned page */ 1486 pg = uvm_pagealloc(NULL, 0, NULL, 0); 1487 1488 if (pg == NULL) { 1489 /* 1490 * drop ownership of page, it can't 1491 * be released 1492 */ 1493 if (uobjpage->pg_flags & PG_WANTED) 1494 wakeup(uobjpage); 1495 atomic_clearbits_int( 1496 &uobjpage->pg_flags, 1497 PG_BUSY|PG_WANTED); 1498 UVM_PAGE_OWN(uobjpage, NULL); 1499 1500 uvm_lock_pageq(); 1501 /* activate: we will need it later */ 1502 uvm_pageactivate(uobjpage); 1503 1504 uvm_unlock_pageq(); 1505 uvmfault_unlockall(&ufi, amap, uobj, 1506 NULL); 1507 UVMHIST_LOG(maphist, 1508 " out of RAM breaking loan, waiting", 1509 0,0,0,0); 1510 uvmexp.fltnoram++; 1511 uvm_wait("flt_noram4"); 1512 goto ReFault; 1513 } 1514 1515 /* 1516 * copy the data from the old page to the new 1517 * one and clear the fake/clean flags on the 1518 * new page (keep it busy). force a reload 1519 * of the old page by clearing it from all 1520 * pmaps. then lock the page queues to 1521 * rename the pages. 1522 */ 1523 uvm_pagecopy(uobjpage, pg); /* old -> new */ 1524 atomic_clearbits_int(&pg->pg_flags, 1525 PG_FAKE|PG_CLEAN); 1526 pmap_page_protect(uobjpage, VM_PROT_NONE); 1527 if (uobjpage->pg_flags & PG_WANTED) 1528 wakeup(uobjpage); 1529 /* uobj still locked */ 1530 atomic_clearbits_int(&uobjpage->pg_flags, 1531 PG_BUSY|PG_WANTED); 1532 UVM_PAGE_OWN(uobjpage, NULL); 1533 1534 uvm_lock_pageq(); 1535 uoff = uobjpage->offset; 1536 /* remove old page */ 1537 uvm_pagerealloc(uobjpage, NULL, 0); 1538 1539 /* 1540 * at this point we have absolutely no 1541 * control over uobjpage 1542 */ 1543 /* install new page */ 1544 uvm_pagerealloc(pg, uobj, uoff); 1545 uvm_unlock_pageq(); 1546 1547 /* 1548 * done! loan is broken and "pg" is 1549 * PG_BUSY. it can now replace uobjpage. 1550 */ 1551 1552 uobjpage = pg; 1553 1554 } /* write fault case */ 1555 } /* if loan_count */ 1556 1557 } else { 1558 1559 /* 1560 * if we are going to promote the data to an anon we 1561 * allocate a blank anon here and plug it into our amap. 1562 */ 1563 #ifdef DIAGNOSTIC 1564 if (amap == NULL) 1565 panic("uvm_fault: want to promote data, but no anon"); 1566 #endif 1567 1568 anon = uvm_analloc(); 1569 if (anon) { 1570 /* 1571 * In `Fill in data...' below, if 1572 * uobjpage == PGO_DONTCARE, we want 1573 * a zero'd, dirty page, so have 1574 * uvm_pagealloc() do that for us. 1575 */ 1576 pg = uvm_pagealloc(NULL, 0, anon, 1577 (uobjpage == PGO_DONTCARE) ? UVM_PGA_ZERO : 0); 1578 } 1579 1580 /* 1581 * out of memory resources? 1582 */ 1583 if (anon == NULL || pg == NULL) { 1584 1585 /* 1586 * arg! must unbusy our page and fail or sleep. 1587 */ 1588 if (uobjpage != PGO_DONTCARE) { 1589 if (uobjpage->pg_flags & PG_WANTED) 1590 /* still holding object lock */ 1591 wakeup(uobjpage); 1592 1593 uvm_lock_pageq(); 1594 uvm_pageactivate(uobjpage); 1595 uvm_unlock_pageq(); 1596 atomic_clearbits_int(&uobjpage->pg_flags, 1597 PG_BUSY|PG_WANTED); 1598 UVM_PAGE_OWN(uobjpage, NULL); 1599 } 1600 1601 /* unlock and fail ... */ 1602 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1603 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1604 if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) { 1605 UVMHIST_LOG(maphist, " promote: out of VM", 1606 0,0,0,0); 1607 uvmexp.fltnoanon++; 1608 return (ENOMEM); 1609 } 1610 1611 UVMHIST_LOG(maphist, " out of RAM, waiting for more", 1612 0,0,0,0); 1613 uvm_anfree(anon); 1614 uvmexp.fltnoram++; 1615 uvm_wait("flt_noram5"); 1616 goto ReFault; 1617 } 1618 1619 /* 1620 * fill in the data 1621 */ 1622 1623 if (uobjpage != PGO_DONTCARE) { 1624 uvmexp.flt_prcopy++; 1625 /* copy page [pg now dirty] */ 1626 uvm_pagecopy(uobjpage, pg); 1627 1628 /* 1629 * promote to shared amap? make sure all sharing 1630 * procs see it 1631 */ 1632 if ((amap_flags(amap) & AMAP_SHARED) != 0) { 1633 pmap_page_protect(uobjpage, VM_PROT_NONE); 1634 } 1635 1636 /* 1637 * dispose of uobjpage. it can't be PG_RELEASED 1638 * since we still hold the object lock. 1639 * drop handle to uobj as well. 1640 */ 1641 1642 if (uobjpage->pg_flags & PG_WANTED) 1643 /* still have the obj lock */ 1644 wakeup(uobjpage); 1645 atomic_clearbits_int(&uobjpage->pg_flags, 1646 PG_BUSY|PG_WANTED); 1647 UVM_PAGE_OWN(uobjpage, NULL); 1648 uvm_lock_pageq(); 1649 uvm_pageactivate(uobjpage); 1650 uvm_unlock_pageq(); 1651 simple_unlock(&uobj->vmobjlock); 1652 uobj = NULL; 1653 1654 UVMHIST_LOG(maphist, 1655 " promote uobjpage %p to anon/page %p/%p", 1656 uobjpage, anon, pg, 0); 1657 1658 } else { 1659 uvmexp.flt_przero++; 1660 /* 1661 * Page is zero'd and marked dirty by uvm_pagealloc() 1662 * above. 1663 */ 1664 UVMHIST_LOG(maphist," zero fill anon/page %p/%p", 1665 anon, pg, 0, 0); 1666 } 1667 1668 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start, 1669 anon, 0); 1670 } 1671 1672 /* 1673 * locked: 1674 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj) 1675 * 1676 * note: pg is either the uobjpage or the new page in the new anon 1677 */ 1678 1679 /* 1680 * all resources are present. we can now map it in and free our 1681 * resources. 1682 */ 1683 1684 UVMHIST_LOG(maphist, 1685 " MAPPING: case2: pm=%p, va=0x%lx, pg=%p, promote=%ld", 1686 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, promote); 1687 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1688 enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) 1689 != 0) { 1690 1691 /* 1692 * No need to undo what we did; we can simply think of 1693 * this as the pmap throwing away the mapping information. 1694 * 1695 * We do, however, have to go through the ReFault path, 1696 * as the map may change while we're asleep. 1697 */ 1698 1699 if (pg->pg_flags & PG_WANTED) 1700 wakeup(pg); /* lock still held */ 1701 1702 /* 1703 * note that pg can't be PG_RELEASED since we did not drop 1704 * the object lock since the last time we checked. 1705 */ 1706 1707 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED); 1708 UVM_PAGE_OWN(pg, NULL); 1709 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1710 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1711 if (uvmexp.swpgonly == uvmexp.swpages) { 1712 UVMHIST_LOG(maphist, 1713 "<- failed. out of VM",0,0,0,0); 1714 /* XXX instrumentation */ 1715 return (ENOMEM); 1716 } 1717 /* XXX instrumentation */ 1718 uvm_wait("flt_pmfail2"); 1719 goto ReFault; 1720 } 1721 1722 uvm_lock_pageq(); 1723 1724 if (fault_type == VM_FAULT_WIRE) { 1725 uvm_pagewire(pg); 1726 if (pg->pg_flags & PQ_AOBJ) { 1727 1728 /* 1729 * since the now-wired page cannot be paged out, 1730 * release its swap resources for others to use. 1731 * since an aobj page with no swap cannot be PG_CLEAN, 1732 * clear its clean flag now. 1733 */ 1734 atomic_clearbits_int(&pg->pg_flags, PG_CLEAN); 1735 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT); 1736 } 1737 } else { 1738 /* activate it */ 1739 uvm_pageactivate(pg); 1740 } 1741 uvm_unlock_pageq(); 1742 1743 if (pg->pg_flags & PG_WANTED) 1744 wakeup(pg); /* lock still held */ 1745 1746 /* 1747 * note that pg can't be PG_RELEASED since we did not drop the object 1748 * lock since the last time we checked. 1749 */ 1750 1751 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED); 1752 UVM_PAGE_OWN(pg, NULL); 1753 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1754 pmap_update(ufi.orig_map->pmap); 1755 1756 UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0); 1757 return (0); 1758 } 1759 1760 1761 /* 1762 * uvm_fault_wire: wire down a range of virtual addresses in a map. 1763 * 1764 * => map may be read-locked by caller, but MUST NOT be write-locked. 1765 * => if map is read-locked, any operations which may cause map to 1766 * be write-locked in uvm_fault() must be taken care of by 1767 * the caller. See uvm_map_pageable(). 1768 */ 1769 1770 int 1771 uvm_fault_wire(map, start, end, access_type) 1772 vm_map_t map; 1773 vaddr_t start, end; 1774 vm_prot_t access_type; 1775 { 1776 vaddr_t va; 1777 pmap_t pmap; 1778 int rv; 1779 1780 pmap = vm_map_pmap(map); 1781 1782 /* 1783 * now fault it in a page at a time. if the fault fails then we have 1784 * to undo what we have done. note that in uvm_fault VM_PROT_NONE 1785 * is replaced with the max protection if fault_type is VM_FAULT_WIRE. 1786 */ 1787 1788 for (va = start ; va < end ; va += PAGE_SIZE) { 1789 rv = uvm_fault(map, va, VM_FAULT_WIRE, access_type); 1790 if (rv) { 1791 if (va != start) { 1792 uvm_fault_unwire(map, start, va); 1793 } 1794 return (rv); 1795 } 1796 } 1797 1798 return (0); 1799 } 1800 1801 /* 1802 * uvm_fault_unwire(): unwire range of virtual space. 1803 */ 1804 1805 void 1806 uvm_fault_unwire(map, start, end) 1807 vm_map_t map; 1808 vaddr_t start, end; 1809 { 1810 1811 vm_map_lock_read(map); 1812 uvm_fault_unwire_locked(map, start, end); 1813 vm_map_unlock_read(map); 1814 } 1815 1816 /* 1817 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire(). 1818 * 1819 * => map must be at least read-locked. 1820 */ 1821 1822 void 1823 uvm_fault_unwire_locked(map, start, end) 1824 vm_map_t map; 1825 vaddr_t start, end; 1826 { 1827 vm_map_entry_t entry; 1828 pmap_t pmap = vm_map_pmap(map); 1829 vaddr_t va; 1830 paddr_t pa; 1831 struct vm_page *pg; 1832 1833 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0); 1834 1835 /* 1836 * we assume that the area we are unwiring has actually been wired 1837 * in the first place. this means that we should be able to extract 1838 * the PAs from the pmap. we also lock out the page daemon so that 1839 * we can call uvm_pageunwire. 1840 */ 1841 1842 uvm_lock_pageq(); 1843 1844 /* 1845 * find the beginning map entry for the region. 1846 */ 1847 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map)); 1848 if (uvm_map_lookup_entry(map, start, &entry) == FALSE) 1849 panic("uvm_fault_unwire_locked: address not in map"); 1850 1851 for (va = start; va < end ; va += PAGE_SIZE) { 1852 if (pmap_extract(pmap, va, &pa) == FALSE) 1853 continue; 1854 1855 /* 1856 * find the map entry for the current address. 1857 */ 1858 KASSERT(va >= entry->start); 1859 while (va >= entry->end) { 1860 KASSERT(entry->next != &map->header && 1861 entry->next->start <= entry->end); 1862 entry = entry->next; 1863 } 1864 1865 /* 1866 * if the entry is no longer wired, tell the pmap. 1867 */ 1868 if (VM_MAPENT_ISWIRED(entry) == 0) 1869 pmap_unwire(pmap, va); 1870 1871 pg = PHYS_TO_VM_PAGE(pa); 1872 if (pg) 1873 uvm_pageunwire(pg); 1874 } 1875 1876 uvm_unlock_pageq(); 1877 } 1878