1 /* $NetBSD: uvm_fault.c,v 1.48 2000/04/10 01:17:41 thorpej Exp $ */ 2 3 /* 4 * 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by Charles D. Cranor and 19 * Washington University. 20 * 4. The name of the author may not be used to endorse or promote products 21 * derived from this software without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 24 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 25 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 26 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 27 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 28 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 32 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 * 34 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp 35 */ 36 37 #include "opt_uvmhist.h" 38 39 /* 40 * uvm_fault.c: fault handler 41 */ 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/kernel.h> 46 #include <sys/proc.h> 47 #include <sys/malloc.h> 48 #include <sys/mman.h> 49 #include <sys/user.h> 50 51 #include <vm/vm.h> 52 #include <vm/vm_page.h> 53 #include <vm/vm_kern.h> 54 55 #include <uvm/uvm.h> 56 57 /* 58 * 59 * a word on page faults: 60 * 61 * types of page faults we handle: 62 * 63 * CASE 1: upper layer faults CASE 2: lower layer faults 64 * 65 * CASE 1A CASE 1B CASE 2A CASE 2B 66 * read/write1 write>1 read/write +-cow_write/zero 67 * | | | | 68 * +--|--+ +--|--+ +-----+ + | + | +-----+ 69 * amap | V | | ----------->new| | | | ^ | 70 * +-----+ +-----+ +-----+ + | + | +--|--+ 71 * | | | 72 * +-----+ +-----+ +--|--+ | +--|--+ 73 * uobj | d/c | | d/c | | V | +----| | 74 * +-----+ +-----+ +-----+ +-----+ 75 * 76 * d/c = don't care 77 * 78 * case [0]: layerless fault 79 * no amap or uobj is present. this is an error. 80 * 81 * case [1]: upper layer fault [anon active] 82 * 1A: [read] or [write with anon->an_ref == 1] 83 * I/O takes place in top level anon and uobj is not touched. 84 * 1B: [write with anon->an_ref > 1] 85 * new anon is alloc'd and data is copied off ["COW"] 86 * 87 * case [2]: lower layer fault [uobj] 88 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area] 89 * I/O takes place directly in object. 90 * 2B: [write to copy_on_write] or [read on NULL uobj] 91 * data is "promoted" from uobj to a new anon. 92 * if uobj is null, then we zero fill. 93 * 94 * we follow the standard UVM locking protocol ordering: 95 * 96 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ) 97 * we hold a PG_BUSY page if we unlock for I/O 98 * 99 * 100 * the code is structured as follows: 101 * 102 * - init the "IN" params in the ufi structure 103 * ReFault: 104 * - do lookups [locks maps], check protection, handle needs_copy 105 * - check for case 0 fault (error) 106 * - establish "range" of fault 107 * - if we have an amap lock it and extract the anons 108 * - if sequential advice deactivate pages behind us 109 * - at the same time check pmap for unmapped areas and anon for pages 110 * that we could map in (and do map it if found) 111 * - check object for resident pages that we could map in 112 * - if (case 2) goto Case2 113 * - >>> handle case 1 114 * - ensure source anon is resident in RAM 115 * - if case 1B alloc new anon and copy from source 116 * - map the correct page in 117 * Case2: 118 * - >>> handle case 2 119 * - ensure source page is resident (if uobj) 120 * - if case 2B alloc new anon and copy from source (could be zero 121 * fill if uobj == NULL) 122 * - map the correct page in 123 * - done! 124 * 125 * note on paging: 126 * if we have to do I/O we place a PG_BUSY page in the correct object, 127 * unlock everything, and do the I/O. when I/O is done we must reverify 128 * the state of the world before assuming that our data structures are 129 * valid. [because mappings could change while the map is unlocked] 130 * 131 * alternative 1: unbusy the page in question and restart the page fault 132 * from the top (ReFault). this is easy but does not take advantage 133 * of the information that we already have from our previous lookup, 134 * although it is possible that the "hints" in the vm_map will help here. 135 * 136 * alternative 2: the system already keeps track of a "version" number of 137 * a map. [i.e. every time you write-lock a map (e.g. to change a 138 * mapping) you bump the version number up by one...] so, we can save 139 * the version number of the map before we release the lock and start I/O. 140 * then when I/O is done we can relock and check the version numbers 141 * to see if anything changed. this might save us some over 1 because 142 * we don't have to unbusy the page and may be less compares(?). 143 * 144 * alternative 3: put in backpointers or a way to "hold" part of a map 145 * in place while I/O is in progress. this could be complex to 146 * implement (especially with structures like amap that can be referenced 147 * by multiple map entries, and figuring out what should wait could be 148 * complex as well...). 149 * 150 * given that we are not currently multiprocessor or multithreaded we might 151 * as well choose alternative 2 now. maybe alternative 3 would be useful 152 * in the future. XXX keep in mind for future consideration//rechecking. 153 */ 154 155 /* 156 * local data structures 157 */ 158 159 struct uvm_advice { 160 int advice; 161 int nback; 162 int nforw; 163 }; 164 165 /* 166 * page range array: 167 * note: index in array must match "advice" value 168 * XXX: borrowed numbers from freebsd. do they work well for us? 169 */ 170 171 static struct uvm_advice uvmadvice[] = { 172 { MADV_NORMAL, 3, 4 }, 173 { MADV_RANDOM, 0, 0 }, 174 { MADV_SEQUENTIAL, 8, 7}, 175 }; 176 177 #define UVM_MAXRANGE 16 /* must be max() of nback+nforw+1 */ 178 179 /* 180 * private prototypes 181 */ 182 183 static void uvmfault_amapcopy __P((struct uvm_faultinfo *)); 184 static __inline void uvmfault_anonflush __P((struct vm_anon **, int)); 185 186 /* 187 * inline functions 188 */ 189 190 /* 191 * uvmfault_anonflush: try and deactivate pages in specified anons 192 * 193 * => does not have to deactivate page if it is busy 194 */ 195 196 static __inline void 197 uvmfault_anonflush(anons, n) 198 struct vm_anon **anons; 199 int n; 200 { 201 int lcv; 202 struct vm_page *pg; 203 204 for (lcv = 0 ; lcv < n ; lcv++) { 205 if (anons[lcv] == NULL) 206 continue; 207 simple_lock(&anons[lcv]->an_lock); 208 pg = anons[lcv]->u.an_page; 209 if (pg && (pg->flags & PG_BUSY) == 0 && pg->loan_count == 0) { 210 uvm_lock_pageq(); 211 if (pg->wire_count == 0) { 212 pmap_page_protect(pg, VM_PROT_NONE); 213 uvm_pagedeactivate(pg); 214 } 215 uvm_unlock_pageq(); 216 } 217 simple_unlock(&anons[lcv]->an_lock); 218 } 219 } 220 221 /* 222 * normal functions 223 */ 224 225 /* 226 * uvmfault_amapcopy: clear "needs_copy" in a map. 227 * 228 * => called with VM data structures unlocked (usually, see below) 229 * => we get a write lock on the maps and clear needs_copy for a VA 230 * => if we are out of RAM we sleep (waiting for more) 231 */ 232 233 static void 234 uvmfault_amapcopy(ufi) 235 struct uvm_faultinfo *ufi; 236 { 237 238 /* 239 * while we haven't done the job 240 */ 241 242 while (1) { 243 244 /* 245 * no mapping? give up. 246 */ 247 248 if (uvmfault_lookup(ufi, TRUE) == FALSE) 249 return; 250 251 /* 252 * copy if needed. 253 */ 254 255 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) 256 amap_copy(ufi->map, ufi->entry, M_NOWAIT, TRUE, 257 ufi->orig_rvaddr, ufi->orig_rvaddr + 1); 258 259 /* 260 * didn't work? must be out of RAM. unlock and sleep. 261 */ 262 263 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 264 uvmfault_unlockmaps(ufi, TRUE); 265 uvm_wait("fltamapcopy"); 266 continue; 267 } 268 269 /* 270 * got it! unlock and return. 271 */ 272 273 uvmfault_unlockmaps(ufi, TRUE); 274 return; 275 } 276 /*NOTREACHED*/ 277 } 278 279 /* 280 * uvmfault_anonget: get data in an anon into a non-busy, non-released 281 * page in that anon. 282 * 283 * => maps, amap, and anon locked by caller. 284 * => if we fail (result != VM_PAGER_OK) we unlock everything except anon. 285 * => if we are successful, we return with everything still locked. 286 * => we don't move the page on the queues [gets moved later] 287 * => if we allocate a new page [we_own], it gets put on the queues. 288 * either way, the result is that the page is on the queues at return time 289 * => for pages which are on loan from a uvm_object (and thus are not 290 * owned by the anon): if successful, we return with the owning object 291 * locked. the caller must unlock this object when it unlocks everything 292 * else. 293 */ 294 295 int 296 uvmfault_anonget(ufi, amap, anon) 297 struct uvm_faultinfo *ufi; 298 struct vm_amap *amap; 299 struct vm_anon *anon; 300 { 301 boolean_t we_own; /* we own anon's page? */ 302 boolean_t locked; /* did we relock? */ 303 struct vm_page *pg; 304 int result; 305 UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist); 306 307 result = 0; /* XXX shut up gcc */ 308 uvmexp.fltanget++; 309 /* bump rusage counters */ 310 if (anon->u.an_page) 311 curproc->p_addr->u_stats.p_ru.ru_minflt++; 312 else 313 curproc->p_addr->u_stats.p_ru.ru_majflt++; 314 315 /* 316 * loop until we get it, or fail. 317 */ 318 319 while (1) { 320 321 we_own = FALSE; /* TRUE if we set PG_BUSY on a page */ 322 pg = anon->u.an_page; 323 324 /* 325 * if there is a resident page and it is loaned, then anon 326 * may not own it. call out to uvm_anon_lockpage() to ensure 327 * the real owner of the page has been identified and locked. 328 */ 329 330 if (pg && pg->loan_count) 331 pg = uvm_anon_lockloanpg(anon); 332 333 /* 334 * page there? make sure it is not busy/released. 335 */ 336 337 if (pg) { 338 339 /* 340 * at this point, if the page has a uobject [meaning 341 * we have it on loan], then that uobject is locked 342 * by us! if the page is busy, we drop all the 343 * locks (including uobject) and try again. 344 */ 345 346 if ((pg->flags & (PG_BUSY|PG_RELEASED)) == 0) { 347 UVMHIST_LOG(maphist, "<- OK",0,0,0,0); 348 return (VM_PAGER_OK); 349 } 350 pg->flags |= PG_WANTED; 351 uvmexp.fltpgwait++; 352 353 /* 354 * the last unlock must be an atomic unlock+wait on 355 * the owner of page 356 */ 357 if (pg->uobject) { /* owner is uobject ? */ 358 uvmfault_unlockall(ufi, amap, NULL, anon); 359 UVMHIST_LOG(maphist, " unlock+wait on uobj",0, 360 0,0,0); 361 UVM_UNLOCK_AND_WAIT(pg, 362 &pg->uobject->vmobjlock, 363 FALSE, "anonget1",0); 364 } else { 365 /* anon owns page */ 366 uvmfault_unlockall(ufi, amap, NULL, NULL); 367 UVMHIST_LOG(maphist, " unlock+wait on anon",0, 368 0,0,0); 369 UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0, 370 "anonget2",0); 371 } 372 /* ready to relock and try again */ 373 374 } else { 375 376 /* 377 * no page, we must try and bring it in. 378 */ 379 pg = uvm_pagealloc(NULL, 0, anon, 0); 380 381 if (pg == NULL) { /* out of RAM. */ 382 383 uvmfault_unlockall(ufi, amap, NULL, anon); 384 uvmexp.fltnoram++; 385 UVMHIST_LOG(maphist, " noram -- UVM_WAIT",0, 386 0,0,0); 387 uvm_wait("flt_noram1"); 388 /* ready to relock and try again */ 389 390 } else { 391 392 /* we set the PG_BUSY bit */ 393 we_own = TRUE; 394 uvmfault_unlockall(ufi, amap, NULL, anon); 395 396 /* 397 * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN 398 * page into the uvm_swap_get function with 399 * all data structures unlocked. note that 400 * it is ok to read an_swslot here because 401 * we hold PG_BUSY on the page. 402 */ 403 uvmexp.pageins++; 404 result = uvm_swap_get(pg, anon->an_swslot, 405 PGO_SYNCIO); 406 407 /* 408 * we clean up after the i/o below in the 409 * "we_own" case 410 */ 411 /* ready to relock and try again */ 412 } 413 } 414 415 /* 416 * now relock and try again 417 */ 418 419 locked = uvmfault_relock(ufi); 420 if (locked && amap != NULL) { 421 amap_lock(amap); 422 } 423 if (locked || we_own) 424 simple_lock(&anon->an_lock); 425 426 /* 427 * if we own the page (i.e. we set PG_BUSY), then we need 428 * to clean up after the I/O. there are three cases to 429 * consider: 430 * [1] page released during I/O: free anon and ReFault. 431 * [2] I/O not OK. free the page and cause the fault 432 * to fail. 433 * [3] I/O OK! activate the page and sync with the 434 * non-we_own case (i.e. drop anon lock if not locked). 435 */ 436 437 if (we_own) { 438 439 if (pg->flags & PG_WANTED) { 440 /* still holding object lock */ 441 wakeup(pg); 442 } 443 /* un-busy! */ 444 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE); 445 UVM_PAGE_OWN(pg, NULL); 446 447 /* 448 * if we were RELEASED during I/O, then our anon is 449 * no longer part of an amap. we need to free the 450 * anon and try again. 451 */ 452 if (pg->flags & PG_RELEASED) { 453 pmap_page_protect(pg, VM_PROT_NONE); 454 simple_unlock(&anon->an_lock); 455 uvm_anfree(anon); /* frees page for us */ 456 if (locked) 457 uvmfault_unlockall(ufi, amap, NULL, 458 NULL); 459 uvmexp.fltpgrele++; 460 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 461 return (VM_PAGER_REFAULT); /* refault! */ 462 } 463 464 if (result != VM_PAGER_OK) { 465 #ifdef DIAGNOSTIC 466 if (result == VM_PAGER_PEND) { 467 panic("uvmfault_anonget: " 468 "got PENDING for non-async I/O"); 469 } 470 #endif 471 /* remove page from anon */ 472 anon->u.an_page = NULL; 473 474 /* 475 * remove the swap slot from the anon 476 * and mark the anon as having no real slot. 477 * don't free the swap slot, thus preventing 478 * it from being used again. 479 */ 480 uvm_swap_markbad(anon->an_swslot, 1); 481 anon->an_swslot = SWSLOT_BAD; 482 483 /* 484 * note: page was never !PG_BUSY, so it 485 * can't be mapped and thus no need to 486 * pmap_page_protect it... 487 */ 488 uvm_lock_pageq(); 489 uvm_pagefree(pg); 490 uvm_unlock_pageq(); 491 492 if (locked) 493 uvmfault_unlockall(ufi, amap, NULL, 494 anon); 495 else 496 simple_unlock(&anon->an_lock); 497 UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0); 498 return (VM_PAGER_ERROR); 499 } 500 501 /* 502 * must be OK, clear modify (already PG_CLEAN) 503 * and activate 504 */ 505 pmap_clear_modify(pg); 506 uvm_lock_pageq(); 507 uvm_pageactivate(pg); 508 uvm_unlock_pageq(); 509 if (!locked) 510 simple_unlock(&anon->an_lock); 511 } 512 513 /* 514 * we were not able to relock. restart fault. 515 */ 516 517 if (!locked) { 518 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 519 return (VM_PAGER_REFAULT); 520 } 521 522 /* 523 * verify no one has touched the amap and moved the anon on us. 524 */ 525 526 if (ufi != NULL && 527 amap_lookup(&ufi->entry->aref, 528 ufi->orig_rvaddr - ufi->entry->start) != anon) { 529 530 uvmfault_unlockall(ufi, amap, NULL, anon); 531 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 532 return (VM_PAGER_REFAULT); 533 } 534 535 /* 536 * try it again! 537 */ 538 539 uvmexp.fltanretry++; 540 continue; 541 542 } /* while (1) */ 543 544 /*NOTREACHED*/ 545 } 546 547 /* 548 * F A U L T - m a i n e n t r y p o i n t 549 */ 550 551 /* 552 * uvm_fault: page fault handler 553 * 554 * => called from MD code to resolve a page fault 555 * => VM data structures usually should be unlocked. however, it is 556 * possible to call here with the main map locked if the caller 557 * gets a write lock, sets it recusive, and then calls us (c.f. 558 * uvm_map_pageable). this should be avoided because it keeps 559 * the map locked off during I/O. 560 */ 561 562 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 563 ~VM_PROT_WRITE : VM_PROT_ALL) 564 565 int 566 uvm_fault(orig_map, vaddr, fault_type, access_type) 567 vm_map_t orig_map; 568 vaddr_t vaddr; 569 vm_fault_t fault_type; 570 vm_prot_t access_type; 571 { 572 struct uvm_faultinfo ufi; 573 vm_prot_t enter_prot; 574 boolean_t wired, narrow, promote, locked, shadowed; 575 int npages, nback, nforw, centeridx, result, lcv, gotpages; 576 vaddr_t startva, objaddr, currva, offset; 577 paddr_t pa; 578 struct vm_amap *amap; 579 struct uvm_object *uobj; 580 struct vm_anon *anons_store[UVM_MAXRANGE], **anons, *anon, *oanon; 581 struct vm_page *pages[UVM_MAXRANGE], *pg, *uobjpage; 582 UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist); 583 584 UVMHIST_LOG(maphist, "(map=0x%x, vaddr=0x%x, ft=%d, at=%d)", 585 orig_map, vaddr, fault_type, access_type); 586 587 anon = NULL; /* XXX: shut up gcc */ 588 589 uvmexp.faults++; /* XXX: locking? */ 590 591 /* 592 * init the IN parameters in the ufi 593 */ 594 595 ufi.orig_map = orig_map; 596 ufi.orig_rvaddr = trunc_page(vaddr); 597 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ 598 if (fault_type == VM_FAULT_WIRE) 599 narrow = TRUE; /* don't look for neighborhood 600 * pages on wire */ 601 else 602 narrow = FALSE; /* normal fault */ 603 604 /* 605 * before we do anything else, if this is a fault on a kernel 606 * address, check to see if the address is managed by an 607 * interrupt-safe map. If it is, we fail immediately. Intrsafe 608 * maps are never pageable, and this approach avoids an evil 609 * locking mess. 610 */ 611 if (orig_map == kernel_map && uvmfault_check_intrsafe(&ufi)) { 612 UVMHIST_LOG(maphist, "<- VA 0x%lx in intrsafe map %p", 613 ufi.orig_rvaddr, ufi.map, 0, 0); 614 return (KERN_FAILURE); 615 } 616 617 /* 618 * "goto ReFault" means restart the page fault from ground zero. 619 */ 620 ReFault: 621 622 /* 623 * lookup and lock the maps 624 */ 625 626 if (uvmfault_lookup(&ufi, FALSE) == FALSE) { 627 UVMHIST_LOG(maphist, "<- no mapping @ 0x%x", vaddr, 0,0,0); 628 return (KERN_INVALID_ADDRESS); 629 } 630 /* locked: maps(read) */ 631 632 /* 633 * check protection 634 */ 635 636 if ((ufi.entry->protection & access_type) != access_type) { 637 UVMHIST_LOG(maphist, 638 "<- protection failure (prot=0x%x, access=0x%x)", 639 ufi.entry->protection, access_type, 0, 0); 640 uvmfault_unlockmaps(&ufi, FALSE); 641 return (KERN_PROTECTION_FAILURE); 642 } 643 644 /* 645 * if the map is not a pageable map, a page fault always fails. 646 */ 647 648 if ((ufi.map->flags & VM_MAP_PAGEABLE) == 0) { 649 UVMHIST_LOG(maphist, 650 "<- map %p not pageable", ufi.map, 0, 0, 0); 651 uvmfault_unlockmaps(&ufi, FALSE); 652 return (KERN_FAILURE); 653 } 654 655 /* 656 * "enter_prot" is the protection we want to enter the page in at. 657 * for certain pages (e.g. copy-on-write pages) this protection can 658 * be more strict than ufi.entry->protection. "wired" means either 659 * the entry is wired or we are fault-wiring the pg. 660 */ 661 662 enter_prot = ufi.entry->protection; 663 wired = VM_MAPENT_ISWIRED(ufi.entry) || (fault_type == VM_FAULT_WIRE); 664 if (wired) 665 access_type = enter_prot; /* full access for wired */ 666 667 /* 668 * handle "needs_copy" case. if we need to copy the amap we will 669 * have to drop our readlock and relock it with a write lock. (we 670 * need a write lock to change anything in a map entry [e.g. 671 * needs_copy]). 672 */ 673 674 if (UVM_ET_ISNEEDSCOPY(ufi.entry)) { 675 if ((access_type & VM_PROT_WRITE) || 676 (ufi.entry->object.uvm_obj == NULL)) { 677 /* need to clear */ 678 UVMHIST_LOG(maphist, 679 " need to clear needs_copy and refault",0,0,0,0); 680 uvmfault_unlockmaps(&ufi, FALSE); 681 uvmfault_amapcopy(&ufi); 682 uvmexp.fltamcopy++; 683 goto ReFault; 684 685 } else { 686 687 /* 688 * ensure that we pmap_enter page R/O since 689 * needs_copy is still true 690 */ 691 enter_prot &= ~VM_PROT_WRITE; 692 693 } 694 } 695 696 /* 697 * identify the players 698 */ 699 700 amap = ufi.entry->aref.ar_amap; /* top layer */ 701 uobj = ufi.entry->object.uvm_obj; /* bottom layer */ 702 703 /* 704 * check for a case 0 fault. if nothing backing the entry then 705 * error now. 706 */ 707 708 if (amap == NULL && uobj == NULL) { 709 uvmfault_unlockmaps(&ufi, FALSE); 710 UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0); 711 return (KERN_INVALID_ADDRESS); 712 } 713 714 /* 715 * establish range of interest based on advice from mapper 716 * and then clip to fit map entry. note that we only want 717 * to do this the first time through the fault. if we 718 * ReFault we will disable this by setting "narrow" to true. 719 */ 720 721 if (narrow == FALSE) { 722 723 /* wide fault (!narrow) */ 724 #ifdef DIAGNOSTIC 725 if (uvmadvice[ufi.entry->advice].advice != ufi.entry->advice) 726 panic("fault: advice mismatch!"); 727 #endif 728 nback = min(uvmadvice[ufi.entry->advice].nback, 729 (ufi.orig_rvaddr - ufi.entry->start) >> PAGE_SHIFT); 730 startva = ufi.orig_rvaddr - (nback << PAGE_SHIFT); 731 nforw = min(uvmadvice[ufi.entry->advice].nforw, 732 ((ufi.entry->end - ufi.orig_rvaddr) >> 733 PAGE_SHIFT) - 1); 734 /* 735 * note: "-1" because we don't want to count the 736 * faulting page as forw 737 */ 738 npages = nback + nforw + 1; 739 centeridx = nback; 740 741 narrow = TRUE; /* ensure only once per-fault */ 742 743 } else { 744 745 /* narrow fault! */ 746 nback = nforw = 0; 747 startva = ufi.orig_rvaddr; 748 npages = 1; 749 centeridx = 0; 750 751 } 752 753 /* locked: maps(read) */ 754 UVMHIST_LOG(maphist, " narrow=%d, back=%d, forw=%d, startva=0x%x", 755 narrow, nback, nforw, startva); 756 UVMHIST_LOG(maphist, " entry=0x%x, amap=0x%x, obj=0x%x", ufi.entry, 757 amap, uobj, 0); 758 759 /* 760 * if we've got an amap, lock it and extract current anons. 761 */ 762 763 if (amap) { 764 amap_lock(amap); 765 anons = anons_store; 766 amap_lookups(&ufi.entry->aref, startva - ufi.entry->start, 767 anons, npages); 768 } else { 769 anons = NULL; /* to be safe */ 770 } 771 772 /* locked: maps(read), amap(if there) */ 773 774 /* 775 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages 776 * now and then forget about them (for the rest of the fault). 777 */ 778 779 if (ufi.entry->advice == MADV_SEQUENTIAL) { 780 781 UVMHIST_LOG(maphist, " MADV_SEQUENTIAL: flushing backpages", 782 0,0,0,0); 783 /* flush back-page anons? */ 784 if (amap) 785 uvmfault_anonflush(anons, nback); 786 787 /* flush object? */ 788 if (uobj) { 789 objaddr = 790 (startva - ufi.entry->start) + ufi.entry->offset; 791 simple_lock(&uobj->vmobjlock); 792 (void) uobj->pgops->pgo_flush(uobj, objaddr, objaddr + 793 (nback << PAGE_SHIFT), PGO_DEACTIVATE); 794 simple_unlock(&uobj->vmobjlock); 795 } 796 797 /* now forget about the backpages */ 798 if (amap) 799 anons += nback; 800 startva = startva + (nback << PAGE_SHIFT); 801 npages -= nback; 802 nback = centeridx = 0; 803 } 804 805 /* locked: maps(read), amap(if there) */ 806 807 /* 808 * map in the backpages and frontpages we found in the amap in hopes 809 * of preventing future faults. we also init the pages[] array as 810 * we go. 811 */ 812 813 currva = startva; 814 shadowed = FALSE; 815 for (lcv = 0 ; lcv < npages ; lcv++, currva += PAGE_SIZE) { 816 817 /* 818 * dont play with VAs that are already mapped 819 * except for center) 820 */ 821 if (lcv != centeridx) { 822 if (pmap_extract(ufi.orig_map->pmap, currva, &pa) == 823 TRUE) { 824 pages[lcv] = PGO_DONTCARE; 825 continue; 826 } 827 } 828 829 /* 830 * unmapped or center page. check if any anon at this level. 831 */ 832 if (amap == NULL || anons[lcv] == NULL) { 833 pages[lcv] = NULL; 834 continue; 835 } 836 837 /* 838 * check for present page and map if possible. re-activate it. 839 */ 840 841 pages[lcv] = PGO_DONTCARE; 842 if (lcv == centeridx) { /* save center for later! */ 843 shadowed = TRUE; 844 continue; 845 } 846 anon = anons[lcv]; 847 simple_lock(&anon->an_lock); 848 /* ignore loaned pages */ 849 if (anon->u.an_page && anon->u.an_page->loan_count == 0 && 850 (anon->u.an_page->flags & (PG_RELEASED|PG_BUSY)) == 0) { 851 uvm_lock_pageq(); 852 uvm_pageactivate(anon->u.an_page); /* reactivate */ 853 uvm_unlock_pageq(); 854 UVMHIST_LOG(maphist, 855 " MAPPING: n anon: pm=0x%x, va=0x%x, pg=0x%x", 856 ufi.orig_map->pmap, currva, anon->u.an_page, 0); 857 uvmexp.fltnamap++; 858 /* 859 * Since this isn't the page that's actually faulting, 860 * ignore pmap_enter() failures; it's not critical 861 * that we enter these right now. 862 */ 863 (void) pmap_enter(ufi.orig_map->pmap, currva, 864 VM_PAGE_TO_PHYS(anon->u.an_page), 865 (anon->an_ref > 1) ? (enter_prot & ~VM_PROT_WRITE) : 866 enter_prot, 867 PMAP_CANFAIL | 868 (VM_MAPENT_ISWIRED(ufi.entry) ? PMAP_WIRED : 0)); 869 } 870 simple_unlock(&anon->an_lock); 871 } 872 873 /* locked: maps(read), amap(if there) */ 874 /* (shadowed == TRUE) if there is an anon at the faulting address */ 875 UVMHIST_LOG(maphist, " shadowed=%d, will_get=%d", shadowed, 876 (uobj && shadowed == FALSE),0,0); 877 878 /* 879 * note that if we are really short of RAM we could sleep in the above 880 * call to pmap_enter with everything locked. bad? 881 * 882 * XXX Actually, that is bad; pmap_enter() should just fail in that 883 * XXX case. --thorpej 884 */ 885 886 /* 887 * if the desired page is not shadowed by the amap and we have a 888 * backing object, then we check to see if the backing object would 889 * prefer to handle the fault itself (rather than letting us do it 890 * with the usual pgo_get hook). the backing object signals this by 891 * providing a pgo_fault routine. 892 */ 893 894 if (uobj && shadowed == FALSE && uobj->pgops->pgo_fault != NULL) { 895 896 simple_lock(&uobj->vmobjlock); 897 898 /* locked: maps(read), amap (if there), uobj */ 899 result = uobj->pgops->pgo_fault(&ufi, startva, pages, npages, 900 centeridx, fault_type, access_type, 901 PGO_LOCKED); 902 /* locked: nothing, pgo_fault has unlocked everything */ 903 904 if (result == VM_PAGER_OK) 905 return (KERN_SUCCESS); /* pgo_fault did pmap enter */ 906 else if (result == VM_PAGER_REFAULT) 907 goto ReFault; /* try again! */ 908 else 909 return (KERN_PROTECTION_FAILURE); 910 } 911 912 /* 913 * now, if the desired page is not shadowed by the amap and we have 914 * a backing object that does not have a special fault routine, then 915 * we ask (with pgo_get) the object for resident pages that we care 916 * about and attempt to map them in. we do not let pgo_get block 917 * (PGO_LOCKED). 918 * 919 * ("get" has the option of doing a pmap_enter for us) 920 */ 921 922 if (uobj && shadowed == FALSE) { 923 simple_lock(&uobj->vmobjlock); 924 925 /* locked (!shadowed): maps(read), amap (if there), uobj */ 926 /* 927 * the following call to pgo_get does _not_ change locking state 928 */ 929 930 uvmexp.fltlget++; 931 gotpages = npages; 932 result = uobj->pgops->pgo_get(uobj, ufi.entry->offset + 933 (startva - ufi.entry->start), 934 pages, &gotpages, centeridx, 935 access_type & MASK(ufi.entry), 936 ufi.entry->advice, PGO_LOCKED); 937 938 /* 939 * check for pages to map, if we got any 940 */ 941 942 uobjpage = NULL; 943 944 if (gotpages) { 945 currva = startva; 946 for (lcv = 0 ; lcv < npages ; 947 lcv++, currva += PAGE_SIZE) { 948 949 if (pages[lcv] == NULL || 950 pages[lcv] == PGO_DONTCARE) 951 continue; 952 953 #ifdef DIAGNOSTIC 954 /* 955 * pager sanity check: pgo_get with 956 * PGO_LOCKED should never return a 957 * released page to us. 958 */ 959 if (pages[lcv]->flags & PG_RELEASED) 960 panic("uvm_fault: pgo_get PGO_LOCKED gave us a RELEASED page"); 961 #endif 962 963 /* 964 * if center page is resident and not 965 * PG_BUSY|PG_RELEASED then pgo_get 966 * made it PG_BUSY for us and gave 967 * us a handle to it. remember this 968 * page as "uobjpage." (for later use). 969 */ 970 971 if (lcv == centeridx) { 972 uobjpage = pages[lcv]; 973 UVMHIST_LOG(maphist, " got uobjpage (0x%x) with locked get", 974 uobjpage, 0,0,0); 975 continue; 976 } 977 978 /* 979 * note: calling pgo_get with locked data 980 * structures returns us pages which are 981 * neither busy nor released, so we don't 982 * need to check for this. we can just 983 * directly enter the page (after moving it 984 * to the head of the active queue [useful?]). 985 */ 986 987 uvm_lock_pageq(); 988 uvm_pageactivate(pages[lcv]); /* reactivate */ 989 uvm_unlock_pageq(); 990 UVMHIST_LOG(maphist, 991 " MAPPING: n obj: pm=0x%x, va=0x%x, pg=0x%x", 992 ufi.orig_map->pmap, currva, pages[lcv], 0); 993 uvmexp.fltnomap++; 994 /* 995 * Since this page isn't the page that's 996 * actually fauling, ignore pmap_enter() 997 * failures; it's not critical that we 998 * enter these right now. 999 */ 1000 (void) pmap_enter(ufi.orig_map->pmap, currva, 1001 VM_PAGE_TO_PHYS(pages[lcv]), 1002 enter_prot & MASK(ufi.entry), 1003 PMAP_CANFAIL | 1004 (wired ? PMAP_WIRED : 0)); 1005 1006 /* 1007 * NOTE: page can't be PG_WANTED or PG_RELEASED 1008 * because we've held the lock the whole time 1009 * we've had the handle. 1010 */ 1011 pages[lcv]->flags &= ~(PG_BUSY); /* un-busy! */ 1012 UVM_PAGE_OWN(pages[lcv], NULL); 1013 1014 /* done! */ 1015 } /* for "lcv" loop */ 1016 } /* "gotpages" != 0 */ 1017 1018 /* note: object still _locked_ */ 1019 } else { 1020 1021 uobjpage = NULL; 1022 1023 } 1024 1025 /* locked (shadowed): maps(read), amap */ 1026 /* locked (!shadowed): maps(read), amap(if there), 1027 uobj(if !null), uobjpage(if !null) */ 1028 1029 /* 1030 * note that at this point we are done with any front or back pages. 1031 * we are now going to focus on the center page (i.e. the one we've 1032 * faulted on). if we have faulted on the top (anon) layer 1033 * [i.e. case 1], then the anon we want is anons[centeridx] (we have 1034 * not touched it yet). if we have faulted on the bottom (uobj) 1035 * layer [i.e. case 2] and the page was both present and available, 1036 * then we've got a pointer to it as "uobjpage" and we've already 1037 * made it BUSY. 1038 */ 1039 1040 /* 1041 * there are four possible cases we must address: 1A, 1B, 2A, and 2B 1042 */ 1043 1044 /* 1045 * redirect case 2: if we are not shadowed, go to case 2. 1046 */ 1047 1048 if (shadowed == FALSE) 1049 goto Case2; 1050 1051 /* locked: maps(read), amap */ 1052 1053 /* 1054 * handle case 1: fault on an anon in our amap 1055 */ 1056 1057 anon = anons[centeridx]; 1058 UVMHIST_LOG(maphist, " case 1 fault: anon=0x%x", anon, 0,0,0); 1059 simple_lock(&anon->an_lock); 1060 1061 /* locked: maps(read), amap, anon */ 1062 1063 /* 1064 * no matter if we have case 1A or case 1B we are going to need to 1065 * have the anon's memory resident. ensure that now. 1066 */ 1067 1068 /* 1069 * let uvmfault_anonget do the dirty work. 1070 * if it fails (!OK) it will unlock all but the anon for us. 1071 * if it succeeds, locks are still valid and locked. 1072 * also, if it is OK, then the anon's page is on the queues. 1073 * if the page is on loan from a uvm_object, then anonget will 1074 * lock that object for us if it does not fail. 1075 */ 1076 1077 result = uvmfault_anonget(&ufi, amap, anon); 1078 if (result != VM_PAGER_OK) { 1079 simple_unlock(&anon->an_lock); 1080 } 1081 1082 if (result == VM_PAGER_REFAULT) 1083 goto ReFault; 1084 1085 if (result == VM_PAGER_AGAIN) { 1086 tsleep((caddr_t)&lbolt, PVM, "fltagain1", 0); 1087 goto ReFault; 1088 } 1089 1090 if (result != VM_PAGER_OK) 1091 return (KERN_PROTECTION_FAILURE); /* XXX??? */ 1092 1093 /* 1094 * uobj is non null if the page is on loan from an object (i.e. uobj) 1095 */ 1096 1097 uobj = anon->u.an_page->uobject; /* locked by anonget if !NULL */ 1098 1099 /* locked: maps(read), amap, anon, uobj(if one) */ 1100 1101 /* 1102 * special handling for loaned pages 1103 */ 1104 if (anon->u.an_page->loan_count) { 1105 1106 if ((access_type & VM_PROT_WRITE) == 0) { 1107 1108 /* 1109 * for read faults on loaned pages we just cap the 1110 * protection at read-only. 1111 */ 1112 1113 enter_prot = enter_prot & ~VM_PROT_WRITE; 1114 1115 } else { 1116 /* 1117 * note that we can't allow writes into a loaned page! 1118 * 1119 * if we have a write fault on a loaned page in an 1120 * anon then we need to look at the anon's ref count. 1121 * if it is greater than one then we are going to do 1122 * a normal copy-on-write fault into a new anon (this 1123 * is not a problem). however, if the reference count 1124 * is one (a case where we would normally allow a 1125 * write directly to the page) then we need to kill 1126 * the loan before we continue. 1127 */ 1128 1129 /* >1 case is already ok */ 1130 if (anon->an_ref == 1) { 1131 1132 /* get new un-owned replacement page */ 1133 pg = uvm_pagealloc(NULL, 0, NULL, 0); 1134 if (pg == NULL) { 1135 uvmfault_unlockall(&ufi, amap, uobj, 1136 anon); 1137 uvm_wait("flt_noram2"); 1138 goto ReFault; 1139 } 1140 1141 /* 1142 * copy data, kill loan, and drop uobj lock 1143 * (if any) 1144 */ 1145 /* copy old -> new */ 1146 uvm_pagecopy(anon->u.an_page, pg); 1147 1148 /* force reload */ 1149 pmap_page_protect(anon->u.an_page, 1150 VM_PROT_NONE); 1151 uvm_lock_pageq(); /* KILL loan */ 1152 if (uobj) 1153 /* if we were loaning */ 1154 anon->u.an_page->loan_count--; 1155 anon->u.an_page->uanon = NULL; 1156 /* in case we owned */ 1157 anon->u.an_page->pqflags &= ~PQ_ANON; 1158 uvm_unlock_pageq(); 1159 if (uobj) { 1160 simple_unlock(&uobj->vmobjlock); 1161 uobj = NULL; 1162 } 1163 1164 /* install new page in anon */ 1165 anon->u.an_page = pg; 1166 pg->uanon = anon; 1167 pg->pqflags |= PQ_ANON; 1168 pg->flags &= ~(PG_BUSY|PG_FAKE); 1169 UVM_PAGE_OWN(pg, NULL); 1170 1171 /* done! */ 1172 } /* ref == 1 */ 1173 } /* write fault */ 1174 } /* loan count */ 1175 1176 /* 1177 * if we are case 1B then we will need to allocate a new blank 1178 * anon to transfer the data into. note that we have a lock 1179 * on anon, so no one can busy or release the page until we are done. 1180 * also note that the ref count can't drop to zero here because 1181 * it is > 1 and we are only dropping one ref. 1182 * 1183 * in the (hopefully very rare) case that we are out of RAM we 1184 * will unlock, wait for more RAM, and refault. 1185 * 1186 * if we are out of anon VM we kill the process (XXX: could wait?). 1187 */ 1188 1189 if ((access_type & VM_PROT_WRITE) != 0 && anon->an_ref > 1) { 1190 1191 UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0); 1192 uvmexp.flt_acow++; 1193 oanon = anon; /* oanon = old, locked anon */ 1194 anon = uvm_analloc(); 1195 if (anon) 1196 pg = uvm_pagealloc(NULL, 0, anon, 0); 1197 #ifdef __GNUC__ 1198 else 1199 pg = NULL; /* XXX: gcc */ 1200 #endif 1201 1202 /* check for out of RAM */ 1203 if (anon == NULL || pg == NULL) { 1204 if (anon) 1205 uvm_anfree(anon); 1206 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1207 #ifdef DIAGNOSTIC 1208 if (uvmexp.swpgonly > uvmexp.swpages) { 1209 panic("uvmexp.swpgonly botch"); 1210 } 1211 #endif 1212 if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) { 1213 UVMHIST_LOG(maphist, 1214 "<- failed. out of VM",0,0,0,0); 1215 uvmexp.fltnoanon++; 1216 return (KERN_RESOURCE_SHORTAGE); 1217 } 1218 1219 uvmexp.fltnoram++; 1220 uvm_wait("flt_noram3"); /* out of RAM, wait for more */ 1221 goto ReFault; 1222 } 1223 1224 /* got all resources, replace anon with nanon */ 1225 1226 uvm_pagecopy(oanon->u.an_page, pg); /* pg now !PG_CLEAN */ 1227 pg->flags &= ~(PG_BUSY|PG_FAKE); /* un-busy! new page */ 1228 UVM_PAGE_OWN(pg, NULL); 1229 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start, 1230 anon, 1); 1231 1232 /* deref: can not drop to zero here by defn! */ 1233 oanon->an_ref--; 1234 1235 /* 1236 * note: oanon still locked. anon is _not_ locked, but we 1237 * have the sole references to in from amap which _is_ locked. 1238 * thus, no one can get at it until we are done with it. 1239 */ 1240 1241 } else { 1242 1243 uvmexp.flt_anon++; 1244 oanon = anon; /* old, locked anon is same as anon */ 1245 pg = anon->u.an_page; 1246 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */ 1247 enter_prot = enter_prot & ~VM_PROT_WRITE; 1248 1249 } 1250 1251 /* locked: maps(read), amap, anon */ 1252 1253 /* 1254 * now map the page in ... 1255 * XXX: old fault unlocks object before pmap_enter. this seems 1256 * suspect since some other thread could blast the page out from 1257 * under us between the unlock and the pmap_enter. 1258 */ 1259 1260 UVMHIST_LOG(maphist, " MAPPING: anon: pm=0x%x, va=0x%x, pg=0x%x", 1261 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, 0); 1262 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1263 enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) 1264 != KERN_SUCCESS) { 1265 /* 1266 * No need to undo what we did; we can simply think of 1267 * this as the pmap throwing away the mapping information. 1268 * 1269 * We do, however, have to go through the ReFault path, 1270 * as the map may change while we're asleep. 1271 */ 1272 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1273 #ifdef DIAGNOSTIC 1274 if (uvmexp.swpgonly > uvmexp.swpages) 1275 panic("uvmexp.swpgonly botch"); 1276 #endif 1277 if (uvmexp.swpgonly == uvmexp.swpages) { 1278 UVMHIST_LOG(maphist, 1279 "<- failed. out of VM",0,0,0,0); 1280 /* XXX instrumentation */ 1281 return (KERN_RESOURCE_SHORTAGE); 1282 } 1283 /* XXX instrumentation */ 1284 uvm_wait("flt_pmfail1"); 1285 goto ReFault; 1286 } 1287 1288 /* 1289 * ... update the page queues. 1290 */ 1291 1292 uvm_lock_pageq(); 1293 1294 if (fault_type == VM_FAULT_WIRE) { 1295 uvm_pagewire(pg); 1296 1297 /* 1298 * since the now-wired page cannot be paged out, 1299 * release its swap resources for others to use. 1300 * since an anon with no swap cannot be PG_CLEAN, 1301 * clear its clean flag now. 1302 */ 1303 1304 pg->flags &= ~(PG_CLEAN); 1305 uvm_anon_dropswap(anon); 1306 } else { 1307 /* activate it */ 1308 uvm_pageactivate(pg); 1309 } 1310 1311 uvm_unlock_pageq(); 1312 1313 /* 1314 * done case 1! finish up by unlocking everything and returning success 1315 */ 1316 1317 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1318 return (KERN_SUCCESS); 1319 1320 1321 Case2: 1322 /* 1323 * handle case 2: faulting on backing object or zero fill 1324 */ 1325 1326 /* 1327 * locked: 1328 * maps(read), amap(if there), uobj(if !null), uobjpage(if !null) 1329 */ 1330 1331 /* 1332 * note that uobjpage can not be PGO_DONTCARE at this point. we now 1333 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we 1334 * have a backing object, check and see if we are going to promote 1335 * the data up to an anon during the fault. 1336 */ 1337 1338 if (uobj == NULL) { 1339 uobjpage = PGO_DONTCARE; 1340 promote = TRUE; /* always need anon here */ 1341 } else { 1342 /* assert(uobjpage != PGO_DONTCARE) */ 1343 promote = (access_type & VM_PROT_WRITE) && 1344 UVM_ET_ISCOPYONWRITE(ufi.entry); 1345 } 1346 UVMHIST_LOG(maphist, " case 2 fault: promote=%d, zfill=%d", 1347 promote, (uobj == NULL), 0,0); 1348 1349 /* 1350 * if uobjpage is not null then we do not need to do I/O to get the 1351 * uobjpage. 1352 * 1353 * if uobjpage is null, then we need to unlock and ask the pager to 1354 * get the data for us. once we have the data, we need to reverify 1355 * the state the world. we are currently not holding any resources. 1356 */ 1357 1358 if (uobjpage) { 1359 /* update rusage counters */ 1360 curproc->p_addr->u_stats.p_ru.ru_minflt++; 1361 } else { 1362 /* update rusage counters */ 1363 curproc->p_addr->u_stats.p_ru.ru_majflt++; 1364 1365 /* locked: maps(read), amap(if there), uobj */ 1366 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1367 /* locked: uobj */ 1368 1369 uvmexp.fltget++; 1370 gotpages = 1; 1371 result = uobj->pgops->pgo_get(uobj, 1372 (ufi.orig_rvaddr - ufi.entry->start) + ufi.entry->offset, 1373 &uobjpage, &gotpages, 0, 1374 access_type & MASK(ufi.entry), 1375 ufi.entry->advice, 0); 1376 1377 /* locked: uobjpage(if result OK) */ 1378 1379 /* 1380 * recover from I/O 1381 */ 1382 1383 if (result != VM_PAGER_OK) { 1384 #ifdef DIAGNOSTIC 1385 if (result == VM_PAGER_PEND) 1386 panic("uvm_fault: pgo_get got PENDing " 1387 "on non-async I/O"); 1388 #endif 1389 1390 if (result == VM_PAGER_AGAIN) { 1391 UVMHIST_LOG(maphist, 1392 " pgo_get says TRY AGAIN!",0,0,0,0); 1393 tsleep((caddr_t)&lbolt, PVM, "fltagain2", 0); 1394 goto ReFault; 1395 } 1396 1397 UVMHIST_LOG(maphist, "<- pgo_get failed (code %d)", 1398 result, 0,0,0); 1399 return (KERN_PROTECTION_FAILURE); /* XXX i/o error */ 1400 } 1401 1402 /* locked: uobjpage */ 1403 1404 /* 1405 * re-verify the state of the world by first trying to relock 1406 * the maps. always relock the object. 1407 */ 1408 1409 locked = uvmfault_relock(&ufi); 1410 if (locked && amap) 1411 amap_lock(amap); 1412 simple_lock(&uobj->vmobjlock); 1413 1414 /* locked(locked): maps(read), amap(if !null), uobj, uobjpage */ 1415 /* locked(!locked): uobj, uobjpage */ 1416 1417 /* 1418 * verify that the page has not be released and re-verify 1419 * that amap slot is still free. if there is a problem, 1420 * we unlock and clean up. 1421 */ 1422 1423 if ((uobjpage->flags & PG_RELEASED) != 0 || 1424 (locked && amap && 1425 amap_lookup(&ufi.entry->aref, 1426 ufi.orig_rvaddr - ufi.entry->start))) { 1427 if (locked) 1428 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1429 locked = FALSE; 1430 } 1431 1432 /* 1433 * didn't get the lock? release the page and retry. 1434 */ 1435 1436 if (locked == FALSE) { 1437 1438 UVMHIST_LOG(maphist, 1439 " wasn't able to relock after fault: retry", 1440 0,0,0,0); 1441 if (uobjpage->flags & PG_WANTED) 1442 /* still holding object lock */ 1443 wakeup(uobjpage); 1444 1445 if (uobjpage->flags & PG_RELEASED) { 1446 uvmexp.fltpgrele++; 1447 #ifdef DIAGNOSTIC 1448 if (uobj->pgops->pgo_releasepg == NULL) 1449 panic("uvm_fault: object has no " 1450 "releasepg function"); 1451 #endif 1452 /* frees page */ 1453 if (uobj->pgops->pgo_releasepg(uobjpage,NULL)) 1454 /* unlock if still alive */ 1455 simple_unlock(&uobj->vmobjlock); 1456 goto ReFault; 1457 } 1458 1459 uvm_lock_pageq(); 1460 /* make sure it is in queues */ 1461 uvm_pageactivate(uobjpage); 1462 1463 uvm_unlock_pageq(); 1464 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1465 UVM_PAGE_OWN(uobjpage, NULL); 1466 simple_unlock(&uobj->vmobjlock); 1467 goto ReFault; 1468 1469 } 1470 1471 /* 1472 * we have the data in uobjpage which is PG_BUSY and 1473 * !PG_RELEASED. we are holding object lock (so the page 1474 * can't be released on us). 1475 */ 1476 1477 /* locked: maps(read), amap(if !null), uobj, uobjpage */ 1478 1479 } 1480 1481 /* 1482 * locked: 1483 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj) 1484 */ 1485 1486 /* 1487 * notes: 1488 * - at this point uobjpage can not be NULL 1489 * - at this point uobjpage can not be PG_RELEASED (since we checked 1490 * for it above) 1491 * - at this point uobjpage could be PG_WANTED (handle later) 1492 */ 1493 1494 if (promote == FALSE) { 1495 1496 /* 1497 * we are not promoting. if the mapping is COW ensure that we 1498 * don't give more access than we should (e.g. when doing a read 1499 * fault on a COPYONWRITE mapping we want to map the COW page in 1500 * R/O even though the entry protection could be R/W). 1501 * 1502 * set "pg" to the page we want to map in (uobjpage, usually) 1503 */ 1504 1505 uvmexp.flt_obj++; 1506 if (UVM_ET_ISCOPYONWRITE(ufi.entry)) 1507 enter_prot &= ~VM_PROT_WRITE; 1508 pg = uobjpage; /* map in the actual object */ 1509 1510 /* assert(uobjpage != PGO_DONTCARE) */ 1511 1512 /* 1513 * we are faulting directly on the page. be careful 1514 * about writing to loaned pages... 1515 */ 1516 if (uobjpage->loan_count) { 1517 1518 if ((access_type & VM_PROT_WRITE) == 0) { 1519 /* read fault: cap the protection at readonly */ 1520 /* cap! */ 1521 enter_prot = enter_prot & ~VM_PROT_WRITE; 1522 } else { 1523 /* write fault: must break the loan here */ 1524 1525 /* alloc new un-owned page */ 1526 pg = uvm_pagealloc(NULL, 0, NULL, 0); 1527 1528 if (pg == NULL) { 1529 /* 1530 * drop ownership of page, it can't 1531 * be released 1532 */ 1533 if (uobjpage->flags & PG_WANTED) 1534 wakeup(uobjpage); 1535 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1536 UVM_PAGE_OWN(uobjpage, NULL); 1537 1538 uvm_lock_pageq(); 1539 /* activate: we will need it later */ 1540 uvm_pageactivate(uobjpage); 1541 1542 uvm_unlock_pageq(); 1543 uvmfault_unlockall(&ufi, amap, uobj, 1544 NULL); 1545 UVMHIST_LOG(maphist, 1546 " out of RAM breaking loan, waiting", 1547 0,0,0,0); 1548 uvmexp.fltnoram++; 1549 uvm_wait("flt_noram4"); 1550 goto ReFault; 1551 } 1552 1553 /* 1554 * copy the data from the old page to the new 1555 * one and clear the fake/clean flags on the 1556 * new page (keep it busy). force a reload 1557 * of the old page by clearing it from all 1558 * pmaps. then lock the page queues to 1559 * rename the pages. 1560 */ 1561 uvm_pagecopy(uobjpage, pg); /* old -> new */ 1562 pg->flags &= ~(PG_FAKE|PG_CLEAN); 1563 pmap_page_protect(uobjpage, VM_PROT_NONE); 1564 if (uobjpage->flags & PG_WANTED) 1565 wakeup(uobjpage); 1566 /* uobj still locked */ 1567 uobjpage->flags &= ~(PG_WANTED|PG_BUSY); 1568 UVM_PAGE_OWN(uobjpage, NULL); 1569 1570 uvm_lock_pageq(); 1571 offset = uobjpage->offset; 1572 /* remove old page */ 1573 uvm_pagerealloc(uobjpage, NULL, 0); 1574 1575 /* 1576 * at this point we have absolutely no 1577 * control over uobjpage 1578 */ 1579 /* install new page */ 1580 uvm_pagerealloc(pg, uobj, offset); 1581 uvm_unlock_pageq(); 1582 1583 /* 1584 * done! loan is broken and "pg" is 1585 * PG_BUSY. it can now replace uobjpage. 1586 */ 1587 1588 uobjpage = pg; 1589 1590 } /* write fault case */ 1591 } /* if loan_count */ 1592 1593 } else { 1594 1595 /* 1596 * if we are going to promote the data to an anon we 1597 * allocate a blank anon here and plug it into our amap. 1598 */ 1599 #if DIAGNOSTIC 1600 if (amap == NULL) 1601 panic("uvm_fault: want to promote data, but no anon"); 1602 #endif 1603 1604 anon = uvm_analloc(); 1605 if (anon) { 1606 /* 1607 * In `Fill in data...' below, if 1608 * uobjpage == PGO_DONTCARE, we want 1609 * a zero'd, dirty page, so have 1610 * uvm_pagealloc() do that for us. 1611 */ 1612 pg = uvm_pagealloc(NULL, 0, anon, 1613 (uobjpage == PGO_DONTCARE) ? UVM_PGA_ZERO : 0); 1614 } 1615 #ifdef __GNUC__ 1616 else 1617 pg = NULL; /* XXX: gcc */ 1618 #endif 1619 1620 /* 1621 * out of memory resources? 1622 */ 1623 if (anon == NULL || pg == NULL) { 1624 1625 /* 1626 * arg! must unbusy our page and fail or sleep. 1627 */ 1628 if (uobjpage != PGO_DONTCARE) { 1629 if (uobjpage->flags & PG_WANTED) 1630 /* still holding object lock */ 1631 wakeup(uobjpage); 1632 1633 uvm_lock_pageq(); 1634 /* make sure it is in queues */ 1635 uvm_pageactivate(uobjpage); 1636 uvm_unlock_pageq(); 1637 /* un-busy! (still locked) */ 1638 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1639 UVM_PAGE_OWN(uobjpage, NULL); 1640 } 1641 1642 /* unlock and fail ... */ 1643 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1644 #ifdef DIAGNOSTIC 1645 if (uvmexp.swpgonly > uvmexp.swpages) { 1646 panic("uvmexp.swpgonly botch"); 1647 } 1648 #endif 1649 if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) { 1650 UVMHIST_LOG(maphist, " promote: out of VM", 1651 0,0,0,0); 1652 uvmexp.fltnoanon++; 1653 return (KERN_RESOURCE_SHORTAGE); 1654 } 1655 1656 UVMHIST_LOG(maphist, " out of RAM, waiting for more", 1657 0,0,0,0); 1658 uvm_anfree(anon); 1659 uvmexp.fltnoram++; 1660 uvm_wait("flt_noram5"); 1661 goto ReFault; 1662 } 1663 1664 /* 1665 * fill in the data 1666 */ 1667 1668 if (uobjpage != PGO_DONTCARE) { 1669 uvmexp.flt_prcopy++; 1670 /* copy page [pg now dirty] */ 1671 uvm_pagecopy(uobjpage, pg); 1672 1673 /* 1674 * promote to shared amap? make sure all sharing 1675 * procs see it 1676 */ 1677 if ((amap_flags(amap) & AMAP_SHARED) != 0) { 1678 pmap_page_protect(uobjpage, VM_PROT_NONE); 1679 } 1680 1681 /* 1682 * dispose of uobjpage. it can't be PG_RELEASED 1683 * since we still hold the object lock. drop 1684 * handle to uobj as well. 1685 */ 1686 1687 if (uobjpage->flags & PG_WANTED) 1688 /* still have the obj lock */ 1689 wakeup(uobjpage); 1690 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1691 UVM_PAGE_OWN(uobjpage, NULL); 1692 uvm_lock_pageq(); 1693 uvm_pageactivate(uobjpage); /* put it back */ 1694 uvm_unlock_pageq(); 1695 simple_unlock(&uobj->vmobjlock); 1696 uobj = NULL; 1697 UVMHIST_LOG(maphist, 1698 " promote uobjpage 0x%x to anon/page 0x%x/0x%x", 1699 uobjpage, anon, pg, 0); 1700 1701 } else { 1702 uvmexp.flt_przero++; 1703 /* 1704 * Page is zero'd and marked dirty by uvm_pagealloc() 1705 * above. 1706 */ 1707 UVMHIST_LOG(maphist," zero fill anon/page 0x%x/0%x", 1708 anon, pg, 0, 0); 1709 } 1710 1711 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start, 1712 anon, 0); 1713 1714 } 1715 1716 /* 1717 * locked: 1718 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj) 1719 * 1720 * note: pg is either the uobjpage or the new page in the new anon 1721 */ 1722 1723 /* 1724 * all resources are present. we can now map it in and free our 1725 * resources. 1726 */ 1727 1728 UVMHIST_LOG(maphist, 1729 " MAPPING: case2: pm=0x%x, va=0x%x, pg=0x%x, promote=%d", 1730 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, promote); 1731 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1732 enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) 1733 != KERN_SUCCESS) { 1734 /* 1735 * No need to undo what we did; we can simply think of 1736 * this as the pmap throwing away the mapping information. 1737 * 1738 * We do, however, have to go through the ReFault path, 1739 * as the map may change while we're asleep. 1740 */ 1741 if (pg->flags & PG_WANTED) 1742 wakeup(pg); /* lock still held */ 1743 1744 /* 1745 * note that pg can't be PG_RELEASED since we did not drop 1746 * the object lock since the last time we checked. 1747 */ 1748 1749 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); 1750 UVM_PAGE_OWN(pg, NULL); 1751 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1752 #ifdef DIAGNOSTIC 1753 if (uvmexp.swpgonly > uvmexp.swpages) 1754 panic("uvmexp.swpgonly botch"); 1755 #endif 1756 if (uvmexp.swpgonly == uvmexp.swpages) { 1757 UVMHIST_LOG(maphist, 1758 "<- failed. out of VM",0,0,0,0); 1759 /* XXX instrumentation */ 1760 return (KERN_RESOURCE_SHORTAGE); 1761 } 1762 /* XXX instrumentation */ 1763 uvm_wait("flt_pmfail2"); 1764 goto ReFault; 1765 } 1766 1767 uvm_lock_pageq(); 1768 1769 if (fault_type == VM_FAULT_WIRE) { 1770 uvm_pagewire(pg); 1771 if (pg->pqflags & PQ_AOBJ) { 1772 1773 /* 1774 * since the now-wired page cannot be paged out, 1775 * release its swap resources for others to use. 1776 * since an aobj page with no swap cannot be PG_CLEAN, 1777 * clear its clean flag now. 1778 */ 1779 1780 pg->flags &= ~(PG_CLEAN); 1781 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT); 1782 } 1783 } else { 1784 /* activate it */ 1785 uvm_pageactivate(pg); 1786 } 1787 1788 uvm_unlock_pageq(); 1789 1790 if (pg->flags & PG_WANTED) 1791 wakeup(pg); /* lock still held */ 1792 1793 /* 1794 * note that pg can't be PG_RELEASED since we did not drop the object 1795 * lock since the last time we checked. 1796 */ 1797 1798 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); 1799 UVM_PAGE_OWN(pg, NULL); 1800 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1801 1802 UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0); 1803 return (KERN_SUCCESS); 1804 } 1805 1806 1807 /* 1808 * uvm_fault_wire: wire down a range of virtual addresses in a map. 1809 * 1810 * => map may be read-locked by caller, but MUST NOT be write-locked. 1811 * => if map is read-locked, any operations which may cause map to 1812 * be write-locked in uvm_fault() must be taken care of by 1813 * the caller. See uvm_map_pageable(). 1814 */ 1815 1816 int 1817 uvm_fault_wire(map, start, end, access_type) 1818 vm_map_t map; 1819 vaddr_t start, end; 1820 vm_prot_t access_type; 1821 { 1822 vaddr_t va; 1823 pmap_t pmap; 1824 int rv; 1825 1826 pmap = vm_map_pmap(map); 1827 1828 /* 1829 * now fault it in a page at a time. if the fault fails then we have 1830 * to undo what we have done. note that in uvm_fault VM_PROT_NONE 1831 * is replaced with the max protection if fault_type is VM_FAULT_WIRE. 1832 */ 1833 1834 for (va = start ; va < end ; va += PAGE_SIZE) { 1835 rv = uvm_fault(map, va, VM_FAULT_WIRE, access_type); 1836 if (rv) { 1837 if (va != start) { 1838 uvm_fault_unwire(map, start, va); 1839 } 1840 return (rv); 1841 } 1842 } 1843 1844 return (KERN_SUCCESS); 1845 } 1846 1847 /* 1848 * uvm_fault_unwire(): unwire range of virtual space. 1849 */ 1850 1851 void 1852 uvm_fault_unwire(map, start, end) 1853 vm_map_t map; 1854 vaddr_t start, end; 1855 { 1856 1857 vm_map_lock_read(map); 1858 uvm_fault_unwire_locked(map, start, end); 1859 vm_map_unlock_read(map); 1860 } 1861 1862 /* 1863 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire(). 1864 * 1865 * => map must be at least read-locked. 1866 */ 1867 1868 void 1869 uvm_fault_unwire_locked(map, start, end) 1870 vm_map_t map; 1871 vaddr_t start, end; 1872 { 1873 vm_map_entry_t entry; 1874 pmap_t pmap = vm_map_pmap(map); 1875 vaddr_t va; 1876 paddr_t pa; 1877 struct vm_page *pg; 1878 1879 #ifdef DIAGNOSTIC 1880 if (map->flags & VM_MAP_INTRSAFE) 1881 panic("uvm_fault_unwire_locked: intrsafe map"); 1882 #endif 1883 1884 /* 1885 * we assume that the area we are unwiring has actually been wired 1886 * in the first place. this means that we should be able to extract 1887 * the PAs from the pmap. we also lock out the page daemon so that 1888 * we can call uvm_pageunwire. 1889 */ 1890 1891 uvm_lock_pageq(); 1892 1893 /* 1894 * find the beginning map entry for the region. 1895 */ 1896 #ifdef DIAGNOSTIC 1897 if (start < vm_map_min(map) || end > vm_map_max(map)) 1898 panic("uvm_fault_unwire_locked: address out of range"); 1899 #endif 1900 if (uvm_map_lookup_entry(map, start, &entry) == FALSE) 1901 panic("uvm_fault_unwire_locked: address not in map"); 1902 1903 for (va = start; va < end ; va += PAGE_SIZE) { 1904 if (pmap_extract(pmap, va, &pa) == FALSE) 1905 panic("uvm_fault_unwire_locked: unwiring " 1906 "non-wired memory"); 1907 1908 /* 1909 * make sure the current entry is for the address we're 1910 * dealing with. if not, grab the next entry. 1911 */ 1912 #ifdef DIAGNOSTIC 1913 if (va < entry->start) 1914 panic("uvm_fault_unwire_locked: hole 1"); 1915 #endif 1916 if (va >= entry->end) { 1917 #ifdef DIAGNOSTIC 1918 if (entry->next == &map->header || 1919 entry->next->start > entry->end) 1920 panic("uvm_fault_unwire_locked: hole 2"); 1921 #endif 1922 entry = entry->next; 1923 } 1924 1925 /* 1926 * if the entry is no longer wired, tell the pmap. 1927 */ 1928 if (VM_MAPENT_ISWIRED(entry) == 0) 1929 pmap_unwire(pmap, va); 1930 1931 pg = PHYS_TO_VM_PAGE(pa); 1932 if (pg) 1933 uvm_pageunwire(pg); 1934 } 1935 1936 uvm_unlock_pageq(); 1937 } 1938