1 /* $NetBSD: uvm_fault.c,v 1.57 2001/03/10 22:46:49 chs Exp $ */ 2 3 /* 4 * 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by Charles D. Cranor and 19 * Washington University. 20 * 4. The name of the author may not be used to endorse or promote products 21 * derived from this software without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 24 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 25 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 26 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 27 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 28 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 32 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 * 34 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp 35 */ 36 37 #include "opt_uvmhist.h" 38 39 /* 40 * uvm_fault.c: fault handler 41 */ 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/kernel.h> 46 #include <sys/proc.h> 47 #include <sys/malloc.h> 48 #include <sys/mman.h> 49 #include <sys/user.h> 50 51 #include <uvm/uvm.h> 52 53 /* 54 * 55 * a word on page faults: 56 * 57 * types of page faults we handle: 58 * 59 * CASE 1: upper layer faults CASE 2: lower layer faults 60 * 61 * CASE 1A CASE 1B CASE 2A CASE 2B 62 * read/write1 write>1 read/write +-cow_write/zero 63 * | | | | 64 * +--|--+ +--|--+ +-----+ + | + | +-----+ 65 * amap | V | | ----------->new| | | | ^ | 66 * +-----+ +-----+ +-----+ + | + | +--|--+ 67 * | | | 68 * +-----+ +-----+ +--|--+ | +--|--+ 69 * uobj | d/c | | d/c | | V | +----| | 70 * +-----+ +-----+ +-----+ +-----+ 71 * 72 * d/c = don't care 73 * 74 * case [0]: layerless fault 75 * no amap or uobj is present. this is an error. 76 * 77 * case [1]: upper layer fault [anon active] 78 * 1A: [read] or [write with anon->an_ref == 1] 79 * I/O takes place in top level anon and uobj is not touched. 80 * 1B: [write with anon->an_ref > 1] 81 * new anon is alloc'd and data is copied off ["COW"] 82 * 83 * case [2]: lower layer fault [uobj] 84 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area] 85 * I/O takes place directly in object. 86 * 2B: [write to copy_on_write] or [read on NULL uobj] 87 * data is "promoted" from uobj to a new anon. 88 * if uobj is null, then we zero fill. 89 * 90 * we follow the standard UVM locking protocol ordering: 91 * 92 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ) 93 * we hold a PG_BUSY page if we unlock for I/O 94 * 95 * 96 * the code is structured as follows: 97 * 98 * - init the "IN" params in the ufi structure 99 * ReFault: 100 * - do lookups [locks maps], check protection, handle needs_copy 101 * - check for case 0 fault (error) 102 * - establish "range" of fault 103 * - if we have an amap lock it and extract the anons 104 * - if sequential advice deactivate pages behind us 105 * - at the same time check pmap for unmapped areas and anon for pages 106 * that we could map in (and do map it if found) 107 * - check object for resident pages that we could map in 108 * - if (case 2) goto Case2 109 * - >>> handle case 1 110 * - ensure source anon is resident in RAM 111 * - if case 1B alloc new anon and copy from source 112 * - map the correct page in 113 * Case2: 114 * - >>> handle case 2 115 * - ensure source page is resident (if uobj) 116 * - if case 2B alloc new anon and copy from source (could be zero 117 * fill if uobj == NULL) 118 * - map the correct page in 119 * - done! 120 * 121 * note on paging: 122 * if we have to do I/O we place a PG_BUSY page in the correct object, 123 * unlock everything, and do the I/O. when I/O is done we must reverify 124 * the state of the world before assuming that our data structures are 125 * valid. [because mappings could change while the map is unlocked] 126 * 127 * alternative 1: unbusy the page in question and restart the page fault 128 * from the top (ReFault). this is easy but does not take advantage 129 * of the information that we already have from our previous lookup, 130 * although it is possible that the "hints" in the vm_map will help here. 131 * 132 * alternative 2: the system already keeps track of a "version" number of 133 * a map. [i.e. every time you write-lock a map (e.g. to change a 134 * mapping) you bump the version number up by one...] so, we can save 135 * the version number of the map before we release the lock and start I/O. 136 * then when I/O is done we can relock and check the version numbers 137 * to see if anything changed. this might save us some over 1 because 138 * we don't have to unbusy the page and may be less compares(?). 139 * 140 * alternative 3: put in backpointers or a way to "hold" part of a map 141 * in place while I/O is in progress. this could be complex to 142 * implement (especially with structures like amap that can be referenced 143 * by multiple map entries, and figuring out what should wait could be 144 * complex as well...). 145 * 146 * given that we are not currently multiprocessor or multithreaded we might 147 * as well choose alternative 2 now. maybe alternative 3 would be useful 148 * in the future. XXX keep in mind for future consideration//rechecking. 149 */ 150 151 /* 152 * local data structures 153 */ 154 155 struct uvm_advice { 156 int advice; 157 int nback; 158 int nforw; 159 }; 160 161 /* 162 * page range array: 163 * note: index in array must match "advice" value 164 * XXX: borrowed numbers from freebsd. do they work well for us? 165 */ 166 167 static struct uvm_advice uvmadvice[] = { 168 { MADV_NORMAL, 3, 4 }, 169 { MADV_RANDOM, 0, 0 }, 170 { MADV_SEQUENTIAL, 8, 7}, 171 }; 172 173 #define UVM_MAXRANGE 16 /* must be max() of nback+nforw+1 */ 174 175 /* 176 * private prototypes 177 */ 178 179 static void uvmfault_amapcopy __P((struct uvm_faultinfo *)); 180 static __inline void uvmfault_anonflush __P((struct vm_anon **, int)); 181 182 /* 183 * inline functions 184 */ 185 186 /* 187 * uvmfault_anonflush: try and deactivate pages in specified anons 188 * 189 * => does not have to deactivate page if it is busy 190 */ 191 192 static __inline void 193 uvmfault_anonflush(anons, n) 194 struct vm_anon **anons; 195 int n; 196 { 197 int lcv; 198 struct vm_page *pg; 199 200 for (lcv = 0 ; lcv < n ; lcv++) { 201 if (anons[lcv] == NULL) 202 continue; 203 simple_lock(&anons[lcv]->an_lock); 204 pg = anons[lcv]->u.an_page; 205 if (pg && (pg->flags & PG_BUSY) == 0 && pg->loan_count == 0) { 206 uvm_lock_pageq(); 207 if (pg->wire_count == 0) { 208 pmap_clear_reference(pg); 209 uvm_pagedeactivate(pg); 210 } 211 uvm_unlock_pageq(); 212 } 213 simple_unlock(&anons[lcv]->an_lock); 214 } 215 } 216 217 /* 218 * normal functions 219 */ 220 221 /* 222 * uvmfault_amapcopy: clear "needs_copy" in a map. 223 * 224 * => called with VM data structures unlocked (usually, see below) 225 * => we get a write lock on the maps and clear needs_copy for a VA 226 * => if we are out of RAM we sleep (waiting for more) 227 */ 228 229 static void 230 uvmfault_amapcopy(ufi) 231 struct uvm_faultinfo *ufi; 232 { 233 234 /* 235 * while we haven't done the job 236 */ 237 238 while (1) { 239 240 /* 241 * no mapping? give up. 242 */ 243 244 if (uvmfault_lookup(ufi, TRUE) == FALSE) 245 return; 246 247 /* 248 * copy if needed. 249 */ 250 251 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) 252 amap_copy(ufi->map, ufi->entry, M_NOWAIT, TRUE, 253 ufi->orig_rvaddr, ufi->orig_rvaddr + 1); 254 255 /* 256 * didn't work? must be out of RAM. unlock and sleep. 257 */ 258 259 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 260 uvmfault_unlockmaps(ufi, TRUE); 261 uvm_wait("fltamapcopy"); 262 continue; 263 } 264 265 /* 266 * got it! unlock and return. 267 */ 268 269 uvmfault_unlockmaps(ufi, TRUE); 270 return; 271 } 272 /*NOTREACHED*/ 273 } 274 275 /* 276 * uvmfault_anonget: get data in an anon into a non-busy, non-released 277 * page in that anon. 278 * 279 * => maps, amap, and anon locked by caller. 280 * => if we fail (result != 0) we unlock everything. 281 * => if we are successful, we return with everything still locked. 282 * => we don't move the page on the queues [gets moved later] 283 * => if we allocate a new page [we_own], it gets put on the queues. 284 * either way, the result is that the page is on the queues at return time 285 * => for pages which are on loan from a uvm_object (and thus are not 286 * owned by the anon): if successful, we return with the owning object 287 * locked. the caller must unlock this object when it unlocks everything 288 * else. 289 */ 290 291 int 292 uvmfault_anonget(ufi, amap, anon) 293 struct uvm_faultinfo *ufi; 294 struct vm_amap *amap; 295 struct vm_anon *anon; 296 { 297 boolean_t we_own; /* we own anon's page? */ 298 boolean_t locked; /* did we relock? */ 299 struct vm_page *pg; 300 int result; 301 UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist); 302 303 LOCK_ASSERT(simple_lock_held(&anon->an_lock)); 304 305 result = 0; /* XXX shut up gcc */ 306 uvmexp.fltanget++; 307 /* bump rusage counters */ 308 if (anon->u.an_page) 309 curproc->p_addr->u_stats.p_ru.ru_minflt++; 310 else 311 curproc->p_addr->u_stats.p_ru.ru_majflt++; 312 313 /* 314 * loop until we get it, or fail. 315 */ 316 317 while (1) { 318 319 we_own = FALSE; /* TRUE if we set PG_BUSY on a page */ 320 pg = anon->u.an_page; 321 322 /* 323 * if there is a resident page and it is loaned, then anon 324 * may not own it. call out to uvm_anon_lockpage() to ensure 325 * the real owner of the page has been identified and locked. 326 */ 327 328 if (pg && pg->loan_count) 329 pg = uvm_anon_lockloanpg(anon); 330 331 /* 332 * page there? make sure it is not busy/released. 333 */ 334 335 if (pg) { 336 337 /* 338 * at this point, if the page has a uobject [meaning 339 * we have it on loan], then that uobject is locked 340 * by us! if the page is busy, we drop all the 341 * locks (including uobject) and try again. 342 */ 343 344 if ((pg->flags & (PG_BUSY|PG_RELEASED)) == 0) { 345 UVMHIST_LOG(maphist, "<- OK",0,0,0,0); 346 return (0); 347 } 348 pg->flags |= PG_WANTED; 349 uvmexp.fltpgwait++; 350 351 /* 352 * the last unlock must be an atomic unlock+wait on 353 * the owner of page 354 */ 355 if (pg->uobject) { /* owner is uobject ? */ 356 uvmfault_unlockall(ufi, amap, NULL, anon); 357 UVMHIST_LOG(maphist, " unlock+wait on uobj",0, 358 0,0,0); 359 UVM_UNLOCK_AND_WAIT(pg, 360 &pg->uobject->vmobjlock, 361 FALSE, "anonget1",0); 362 } else { 363 /* anon owns page */ 364 uvmfault_unlockall(ufi, amap, NULL, NULL); 365 UVMHIST_LOG(maphist, " unlock+wait on anon",0, 366 0,0,0); 367 UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0, 368 "anonget2",0); 369 } 370 /* ready to relock and try again */ 371 372 } else { 373 374 /* 375 * no page, we must try and bring it in. 376 */ 377 pg = uvm_pagealloc(NULL, 0, anon, 0); 378 379 if (pg == NULL) { /* out of RAM. */ 380 381 uvmfault_unlockall(ufi, amap, NULL, anon); 382 uvmexp.fltnoram++; 383 UVMHIST_LOG(maphist, " noram -- UVM_WAIT",0, 384 0,0,0); 385 uvm_wait("flt_noram1"); 386 /* ready to relock and try again */ 387 388 } else { 389 390 /* we set the PG_BUSY bit */ 391 we_own = TRUE; 392 uvmfault_unlockall(ufi, amap, NULL, anon); 393 394 /* 395 * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN 396 * page into the uvm_swap_get function with 397 * all data structures unlocked. note that 398 * it is ok to read an_swslot here because 399 * we hold PG_BUSY on the page. 400 */ 401 uvmexp.pageins++; 402 result = uvm_swap_get(pg, anon->an_swslot, 403 PGO_SYNCIO); 404 405 /* 406 * we clean up after the i/o below in the 407 * "we_own" case 408 */ 409 /* ready to relock and try again */ 410 } 411 } 412 413 /* 414 * now relock and try again 415 */ 416 417 locked = uvmfault_relock(ufi); 418 if (locked && amap != NULL) { 419 amap_lock(amap); 420 } 421 if (locked || we_own) 422 simple_lock(&anon->an_lock); 423 424 /* 425 * if we own the page (i.e. we set PG_BUSY), then we need 426 * to clean up after the I/O. there are three cases to 427 * consider: 428 * [1] page released during I/O: free anon and ReFault. 429 * [2] I/O not OK. free the page and cause the fault 430 * to fail. 431 * [3] I/O OK! activate the page and sync with the 432 * non-we_own case (i.e. drop anon lock if not locked). 433 */ 434 435 if (we_own) { 436 437 if (pg->flags & PG_WANTED) { 438 /* still holding object lock */ 439 wakeup(pg); 440 } 441 /* un-busy! */ 442 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE); 443 UVM_PAGE_OWN(pg, NULL); 444 445 /* 446 * if we were RELEASED during I/O, then our anon is 447 * no longer part of an amap. we need to free the 448 * anon and try again. 449 */ 450 if (pg->flags & PG_RELEASED) { 451 pmap_page_protect(pg, VM_PROT_NONE); 452 simple_unlock(&anon->an_lock); 453 uvm_anfree(anon); /* frees page for us */ 454 if (locked) 455 uvmfault_unlockall(ufi, amap, NULL, 456 NULL); 457 uvmexp.fltpgrele++; 458 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 459 return (ERESTART); /* refault! */ 460 } 461 462 if (result != 0) { 463 /* remove page from anon */ 464 anon->u.an_page = NULL; 465 466 /* 467 * remove the swap slot from the anon 468 * and mark the anon as having no real slot. 469 * don't free the swap slot, thus preventing 470 * it from being used again. 471 */ 472 uvm_swap_markbad(anon->an_swslot, 1); 473 anon->an_swslot = SWSLOT_BAD; 474 475 /* 476 * note: page was never !PG_BUSY, so it 477 * can't be mapped and thus no need to 478 * pmap_page_protect it... 479 */ 480 uvm_lock_pageq(); 481 uvm_pagefree(pg); 482 uvm_unlock_pageq(); 483 484 if (locked) 485 uvmfault_unlockall(ufi, amap, NULL, 486 anon); 487 else 488 simple_unlock(&anon->an_lock); 489 UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0); 490 return (EIO); 491 } 492 493 /* 494 * must be OK, clear modify (already PG_CLEAN) 495 * and activate 496 */ 497 pmap_clear_modify(pg); 498 uvm_lock_pageq(); 499 uvm_pageactivate(pg); 500 uvm_unlock_pageq(); 501 if (!locked) 502 simple_unlock(&anon->an_lock); 503 } 504 505 /* 506 * we were not able to relock. restart fault. 507 */ 508 509 if (!locked) { 510 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 511 return (ERESTART); 512 } 513 514 /* 515 * verify no one has touched the amap and moved the anon on us. 516 */ 517 518 if (ufi != NULL && 519 amap_lookup(&ufi->entry->aref, 520 ufi->orig_rvaddr - ufi->entry->start) != anon) { 521 522 uvmfault_unlockall(ufi, amap, NULL, anon); 523 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 524 return (ERESTART); 525 } 526 527 /* 528 * try it again! 529 */ 530 531 uvmexp.fltanretry++; 532 continue; 533 534 } /* while (1) */ 535 536 /*NOTREACHED*/ 537 } 538 539 /* 540 * F A U L T - m a i n e n t r y p o i n t 541 */ 542 543 /* 544 * uvm_fault: page fault handler 545 * 546 * => called from MD code to resolve a page fault 547 * => VM data structures usually should be unlocked. however, it is 548 * possible to call here with the main map locked if the caller 549 * gets a write lock, sets it recusive, and then calls us (c.f. 550 * uvm_map_pageable). this should be avoided because it keeps 551 * the map locked off during I/O. 552 */ 553 554 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 555 ~VM_PROT_WRITE : VM_PROT_ALL) 556 557 int 558 uvm_fault(orig_map, vaddr, fault_type, access_type) 559 vm_map_t orig_map; 560 vaddr_t vaddr; 561 vm_fault_t fault_type; 562 vm_prot_t access_type; 563 { 564 struct uvm_faultinfo ufi; 565 vm_prot_t enter_prot; 566 boolean_t wired, narrow, promote, locked, shadowed; 567 int npages, nback, nforw, centeridx, result, lcv, gotpages; 568 vaddr_t startva, objaddr, currva, offset, uoff; 569 paddr_t pa; 570 struct vm_amap *amap; 571 struct uvm_object *uobj; 572 struct vm_anon *anons_store[UVM_MAXRANGE], **anons, *anon, *oanon; 573 struct vm_page *pages[UVM_MAXRANGE], *pg, *uobjpage; 574 UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist); 575 576 UVMHIST_LOG(maphist, "(map=0x%x, vaddr=0x%x, ft=%d, at=%d)", 577 orig_map, vaddr, fault_type, access_type); 578 579 anon = NULL; 580 pg = NULL; 581 582 uvmexp.faults++; /* XXX: locking? */ 583 584 /* 585 * init the IN parameters in the ufi 586 */ 587 588 ufi.orig_map = orig_map; 589 ufi.orig_rvaddr = trunc_page(vaddr); 590 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ 591 if (fault_type == VM_FAULT_WIRE) 592 narrow = TRUE; /* don't look for neighborhood 593 * pages on wire */ 594 else 595 narrow = FALSE; /* normal fault */ 596 597 /* 598 * before we do anything else, if this is a fault on a kernel 599 * address, check to see if the address is managed by an 600 * interrupt-safe map. If it is, we fail immediately. Intrsafe 601 * maps are never pageable, and this approach avoids an evil 602 * locking mess. 603 */ 604 if (orig_map == kernel_map && uvmfault_check_intrsafe(&ufi)) { 605 UVMHIST_LOG(maphist, "<- VA 0x%lx in intrsafe map %p", 606 ufi.orig_rvaddr, ufi.map, 0, 0); 607 return (KERN_FAILURE); 608 } 609 610 /* 611 * "goto ReFault" means restart the page fault from ground zero. 612 */ 613 ReFault: 614 615 /* 616 * lookup and lock the maps 617 */ 618 619 if (uvmfault_lookup(&ufi, FALSE) == FALSE) { 620 UVMHIST_LOG(maphist, "<- no mapping @ 0x%x", vaddr, 0,0,0); 621 return (KERN_INVALID_ADDRESS); 622 } 623 /* locked: maps(read) */ 624 625 /* 626 * check protection 627 */ 628 629 if ((ufi.entry->protection & access_type) != access_type) { 630 UVMHIST_LOG(maphist, 631 "<- protection failure (prot=0x%x, access=0x%x)", 632 ufi.entry->protection, access_type, 0, 0); 633 uvmfault_unlockmaps(&ufi, FALSE); 634 return (KERN_PROTECTION_FAILURE); 635 } 636 637 /* 638 * if the map is not a pageable map, a page fault always fails. 639 */ 640 641 if ((ufi.map->flags & VM_MAP_PAGEABLE) == 0) { 642 UVMHIST_LOG(maphist, 643 "<- map %p not pageable", ufi.map, 0, 0, 0); 644 uvmfault_unlockmaps(&ufi, FALSE); 645 return (KERN_FAILURE); 646 } 647 648 /* 649 * "enter_prot" is the protection we want to enter the page in at. 650 * for certain pages (e.g. copy-on-write pages) this protection can 651 * be more strict than ufi.entry->protection. "wired" means either 652 * the entry is wired or we are fault-wiring the pg. 653 */ 654 655 enter_prot = ufi.entry->protection; 656 wired = VM_MAPENT_ISWIRED(ufi.entry) || (fault_type == VM_FAULT_WIRE); 657 if (wired) 658 access_type = enter_prot; /* full access for wired */ 659 660 /* 661 * handle "needs_copy" case. if we need to copy the amap we will 662 * have to drop our readlock and relock it with a write lock. (we 663 * need a write lock to change anything in a map entry [e.g. 664 * needs_copy]). 665 */ 666 667 if (UVM_ET_ISNEEDSCOPY(ufi.entry)) { 668 if ((access_type & VM_PROT_WRITE) || 669 (ufi.entry->object.uvm_obj == NULL)) { 670 /* need to clear */ 671 UVMHIST_LOG(maphist, 672 " need to clear needs_copy and refault",0,0,0,0); 673 uvmfault_unlockmaps(&ufi, FALSE); 674 uvmfault_amapcopy(&ufi); 675 uvmexp.fltamcopy++; 676 goto ReFault; 677 678 } else { 679 680 /* 681 * ensure that we pmap_enter page R/O since 682 * needs_copy is still true 683 */ 684 enter_prot &= ~VM_PROT_WRITE; 685 686 } 687 } 688 689 /* 690 * identify the players 691 */ 692 693 amap = ufi.entry->aref.ar_amap; /* top layer */ 694 uobj = ufi.entry->object.uvm_obj; /* bottom layer */ 695 696 /* 697 * check for a case 0 fault. if nothing backing the entry then 698 * error now. 699 */ 700 701 if (amap == NULL && uobj == NULL) { 702 uvmfault_unlockmaps(&ufi, FALSE); 703 UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0); 704 return (KERN_INVALID_ADDRESS); 705 } 706 707 /* 708 * establish range of interest based on advice from mapper 709 * and then clip to fit map entry. note that we only want 710 * to do this the first time through the fault. if we 711 * ReFault we will disable this by setting "narrow" to true. 712 */ 713 714 if (narrow == FALSE) { 715 716 /* wide fault (!narrow) */ 717 KASSERT(uvmadvice[ufi.entry->advice].advice == 718 ufi.entry->advice); 719 nback = min(uvmadvice[ufi.entry->advice].nback, 720 (ufi.orig_rvaddr - ufi.entry->start) >> PAGE_SHIFT); 721 startva = ufi.orig_rvaddr - (nback << PAGE_SHIFT); 722 nforw = min(uvmadvice[ufi.entry->advice].nforw, 723 ((ufi.entry->end - ufi.orig_rvaddr) >> 724 PAGE_SHIFT) - 1); 725 /* 726 * note: "-1" because we don't want to count the 727 * faulting page as forw 728 */ 729 npages = nback + nforw + 1; 730 centeridx = nback; 731 732 narrow = TRUE; /* ensure only once per-fault */ 733 734 } else { 735 736 /* narrow fault! */ 737 nback = nforw = 0; 738 startva = ufi.orig_rvaddr; 739 npages = 1; 740 centeridx = 0; 741 742 } 743 744 /* locked: maps(read) */ 745 UVMHIST_LOG(maphist, " narrow=%d, back=%d, forw=%d, startva=0x%x", 746 narrow, nback, nforw, startva); 747 UVMHIST_LOG(maphist, " entry=0x%x, amap=0x%x, obj=0x%x", ufi.entry, 748 amap, uobj, 0); 749 750 /* 751 * if we've got an amap, lock it and extract current anons. 752 */ 753 754 if (amap) { 755 amap_lock(amap); 756 anons = anons_store; 757 amap_lookups(&ufi.entry->aref, startva - ufi.entry->start, 758 anons, npages); 759 } else { 760 anons = NULL; /* to be safe */ 761 } 762 763 /* locked: maps(read), amap(if there) */ 764 765 /* 766 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages 767 * now and then forget about them (for the rest of the fault). 768 */ 769 770 if (ufi.entry->advice == MADV_SEQUENTIAL) { 771 772 UVMHIST_LOG(maphist, " MADV_SEQUENTIAL: flushing backpages", 773 0,0,0,0); 774 /* flush back-page anons? */ 775 if (amap) 776 uvmfault_anonflush(anons, nback); 777 778 /* flush object? */ 779 if (uobj) { 780 objaddr = 781 (startva - ufi.entry->start) + ufi.entry->offset; 782 simple_lock(&uobj->vmobjlock); 783 (void) uobj->pgops->pgo_flush(uobj, objaddr, objaddr + 784 (nback << PAGE_SHIFT), PGO_DEACTIVATE); 785 simple_unlock(&uobj->vmobjlock); 786 } 787 788 /* now forget about the backpages */ 789 if (amap) 790 anons += nback; 791 startva += (nback << PAGE_SHIFT); 792 npages -= nback; 793 nback = centeridx = 0; 794 } 795 796 /* locked: maps(read), amap(if there) */ 797 798 /* 799 * map in the backpages and frontpages we found in the amap in hopes 800 * of preventing future faults. we also init the pages[] array as 801 * we go. 802 */ 803 804 currva = startva; 805 shadowed = FALSE; 806 for (lcv = 0 ; lcv < npages ; lcv++, currva += PAGE_SIZE) { 807 808 /* 809 * dont play with VAs that are already mapped 810 * except for center) 811 */ 812 if (lcv != centeridx && 813 pmap_extract(ufi.orig_map->pmap, currva, &pa)) { 814 pages[lcv] = PGO_DONTCARE; 815 continue; 816 } 817 818 /* 819 * unmapped or center page. check if any anon at this level. 820 */ 821 if (amap == NULL || anons[lcv] == NULL) { 822 pages[lcv] = NULL; 823 continue; 824 } 825 826 /* 827 * check for present page and map if possible. re-activate it. 828 */ 829 830 pages[lcv] = PGO_DONTCARE; 831 if (lcv == centeridx) { /* save center for later! */ 832 shadowed = TRUE; 833 continue; 834 } 835 anon = anons[lcv]; 836 simple_lock(&anon->an_lock); 837 /* ignore loaned pages */ 838 if (anon->u.an_page && anon->u.an_page->loan_count == 0 && 839 (anon->u.an_page->flags & (PG_RELEASED|PG_BUSY)) == 0) { 840 uvm_lock_pageq(); 841 uvm_pageactivate(anon->u.an_page); /* reactivate */ 842 uvm_unlock_pageq(); 843 UVMHIST_LOG(maphist, 844 " MAPPING: n anon: pm=0x%x, va=0x%x, pg=0x%x", 845 ufi.orig_map->pmap, currva, anon->u.an_page, 0); 846 uvmexp.fltnamap++; 847 848 /* 849 * Since this isn't the page that's actually faulting, 850 * ignore pmap_enter() failures; it's not critical 851 * that we enter these right now. 852 */ 853 854 (void) pmap_enter(ufi.orig_map->pmap, currva, 855 VM_PAGE_TO_PHYS(anon->u.an_page), 856 (anon->an_ref > 1) ? (enter_prot & ~VM_PROT_WRITE) : 857 enter_prot, 858 PMAP_CANFAIL | 859 (VM_MAPENT_ISWIRED(ufi.entry) ? PMAP_WIRED : 0)); 860 } 861 simple_unlock(&anon->an_lock); 862 } 863 864 /* locked: maps(read), amap(if there) */ 865 /* (shadowed == TRUE) if there is an anon at the faulting address */ 866 UVMHIST_LOG(maphist, " shadowed=%d, will_get=%d", shadowed, 867 (uobj && shadowed == FALSE),0,0); 868 869 /* 870 * note that if we are really short of RAM we could sleep in the above 871 * call to pmap_enter with everything locked. bad? 872 * 873 * XXX Actually, that is bad; pmap_enter() should just fail in that 874 * XXX case. --thorpej 875 */ 876 877 /* 878 * if the desired page is not shadowed by the amap and we have a 879 * backing object, then we check to see if the backing object would 880 * prefer to handle the fault itself (rather than letting us do it 881 * with the usual pgo_get hook). the backing object signals this by 882 * providing a pgo_fault routine. 883 */ 884 885 if (uobj && shadowed == FALSE && uobj->pgops->pgo_fault != NULL) { 886 simple_lock(&uobj->vmobjlock); 887 888 /* locked: maps(read), amap (if there), uobj */ 889 result = uobj->pgops->pgo_fault(&ufi, startva, pages, npages, 890 centeridx, fault_type, access_type, 891 PGO_LOCKED|PGO_SYNCIO); 892 893 /* locked: nothing, pgo_fault has unlocked everything */ 894 895 if (result == 0) 896 return (KERN_SUCCESS); /* pgo_fault did pmap enter */ 897 else if (result == ERESTART) 898 goto ReFault; /* try again! */ 899 else 900 return (KERN_PROTECTION_FAILURE); 901 } 902 903 /* 904 * now, if the desired page is not shadowed by the amap and we have 905 * a backing object that does not have a special fault routine, then 906 * we ask (with pgo_get) the object for resident pages that we care 907 * about and attempt to map them in. we do not let pgo_get block 908 * (PGO_LOCKED). 909 * 910 * ("get" has the option of doing a pmap_enter for us) 911 */ 912 913 if (uobj && shadowed == FALSE) { 914 simple_lock(&uobj->vmobjlock); 915 916 /* locked (!shadowed): maps(read), amap (if there), uobj */ 917 /* 918 * the following call to pgo_get does _not_ change locking state 919 */ 920 921 uvmexp.fltlget++; 922 gotpages = npages; 923 (void) uobj->pgops->pgo_get(uobj, ufi.entry->offset + 924 (startva - ufi.entry->start), 925 pages, &gotpages, centeridx, 926 access_type & MASK(ufi.entry), 927 ufi.entry->advice, PGO_LOCKED); 928 929 /* 930 * check for pages to map, if we got any 931 */ 932 933 uobjpage = NULL; 934 935 if (gotpages) { 936 currva = startva; 937 for (lcv = 0 ; lcv < npages ; 938 lcv++, currva += PAGE_SIZE) { 939 940 if (pages[lcv] == NULL || 941 pages[lcv] == PGO_DONTCARE) 942 continue; 943 944 KASSERT((pages[lcv]->flags & PG_RELEASED) == 0); 945 946 /* 947 * if center page is resident and not 948 * PG_BUSY|PG_RELEASED then pgo_get 949 * made it PG_BUSY for us and gave 950 * us a handle to it. remember this 951 * page as "uobjpage." (for later use). 952 */ 953 954 if (lcv == centeridx) { 955 uobjpage = pages[lcv]; 956 UVMHIST_LOG(maphist, " got uobjpage " 957 "(0x%x) with locked get", 958 uobjpage, 0,0,0); 959 continue; 960 } 961 962 /* 963 * note: calling pgo_get with locked data 964 * structures returns us pages which are 965 * neither busy nor released, so we don't 966 * need to check for this. we can just 967 * directly enter the page (after moving it 968 * to the head of the active queue [useful?]). 969 */ 970 971 uvm_lock_pageq(); 972 uvm_pageactivate(pages[lcv]); /* reactivate */ 973 uvm_unlock_pageq(); 974 UVMHIST_LOG(maphist, 975 " MAPPING: n obj: pm=0x%x, va=0x%x, pg=0x%x", 976 ufi.orig_map->pmap, currva, pages[lcv], 0); 977 uvmexp.fltnomap++; 978 979 /* 980 * Since this page isn't the page that's 981 * actually fauling, ignore pmap_enter() 982 * failures; it's not critical that we 983 * enter these right now. 984 */ 985 986 (void) pmap_enter(ufi.orig_map->pmap, currva, 987 VM_PAGE_TO_PHYS(pages[lcv]), 988 pages[lcv]->flags & PG_RDONLY ? 989 VM_PROT_READ : enter_prot & MASK(ufi.entry), 990 PMAP_CANFAIL | 991 (wired ? PMAP_WIRED : 0)); 992 993 /* 994 * NOTE: page can't be PG_WANTED or PG_RELEASED 995 * because we've held the lock the whole time 996 * we've had the handle. 997 */ 998 999 pages[lcv]->flags &= ~(PG_BUSY); /* un-busy! */ 1000 UVM_PAGE_OWN(pages[lcv], NULL); 1001 } /* for "lcv" loop */ 1002 } /* "gotpages" != 0 */ 1003 /* note: object still _locked_ */ 1004 } else { 1005 uobjpage = NULL; 1006 } 1007 1008 /* locked (shadowed): maps(read), amap */ 1009 /* locked (!shadowed): maps(read), amap(if there), 1010 uobj(if !null), uobjpage(if !null) */ 1011 1012 /* 1013 * note that at this point we are done with any front or back pages. 1014 * we are now going to focus on the center page (i.e. the one we've 1015 * faulted on). if we have faulted on the top (anon) layer 1016 * [i.e. case 1], then the anon we want is anons[centeridx] (we have 1017 * not touched it yet). if we have faulted on the bottom (uobj) 1018 * layer [i.e. case 2] and the page was both present and available, 1019 * then we've got a pointer to it as "uobjpage" and we've already 1020 * made it BUSY. 1021 */ 1022 1023 /* 1024 * there are four possible cases we must address: 1A, 1B, 2A, and 2B 1025 */ 1026 1027 /* 1028 * redirect case 2: if we are not shadowed, go to case 2. 1029 */ 1030 1031 if (shadowed == FALSE) 1032 goto Case2; 1033 1034 /* locked: maps(read), amap */ 1035 1036 /* 1037 * handle case 1: fault on an anon in our amap 1038 */ 1039 1040 anon = anons[centeridx]; 1041 UVMHIST_LOG(maphist, " case 1 fault: anon=0x%x", anon, 0,0,0); 1042 simple_lock(&anon->an_lock); 1043 1044 /* locked: maps(read), amap, anon */ 1045 1046 /* 1047 * no matter if we have case 1A or case 1B we are going to need to 1048 * have the anon's memory resident. ensure that now. 1049 */ 1050 1051 /* 1052 * let uvmfault_anonget do the dirty work. 1053 * if it fails (!OK) it will unlock everything for us. 1054 * if it succeeds, locks are still valid and locked. 1055 * also, if it is OK, then the anon's page is on the queues. 1056 * if the page is on loan from a uvm_object, then anonget will 1057 * lock that object for us if it does not fail. 1058 */ 1059 1060 result = uvmfault_anonget(&ufi, amap, anon); 1061 switch (result) { 1062 case 0: 1063 break; 1064 1065 case ERESTART: 1066 goto ReFault; 1067 1068 case EAGAIN: 1069 tsleep(&lbolt, PVM, "fltagain1", 0); 1070 goto ReFault; 1071 1072 default: 1073 #ifdef DIAGNOSTIC 1074 panic("uvm_fault: uvmfault_anonget -> %d", result); 1075 #else 1076 return (KERN_PROTECTION_FAILURE); 1077 #endif 1078 } 1079 1080 /* 1081 * uobj is non null if the page is on loan from an object (i.e. uobj) 1082 */ 1083 1084 uobj = anon->u.an_page->uobject; /* locked by anonget if !NULL */ 1085 1086 /* locked: maps(read), amap, anon, uobj(if one) */ 1087 1088 /* 1089 * special handling for loaned pages 1090 */ 1091 1092 if (anon->u.an_page->loan_count) { 1093 1094 if ((access_type & VM_PROT_WRITE) == 0) { 1095 1096 /* 1097 * for read faults on loaned pages we just cap the 1098 * protection at read-only. 1099 */ 1100 1101 enter_prot = enter_prot & ~VM_PROT_WRITE; 1102 1103 } else { 1104 /* 1105 * note that we can't allow writes into a loaned page! 1106 * 1107 * if we have a write fault on a loaned page in an 1108 * anon then we need to look at the anon's ref count. 1109 * if it is greater than one then we are going to do 1110 * a normal copy-on-write fault into a new anon (this 1111 * is not a problem). however, if the reference count 1112 * is one (a case where we would normally allow a 1113 * write directly to the page) then we need to kill 1114 * the loan before we continue. 1115 */ 1116 1117 /* >1 case is already ok */ 1118 if (anon->an_ref == 1) { 1119 1120 /* get new un-owned replacement page */ 1121 pg = uvm_pagealloc(NULL, 0, NULL, 0); 1122 if (pg == NULL) { 1123 uvmfault_unlockall(&ufi, amap, uobj, 1124 anon); 1125 uvm_wait("flt_noram2"); 1126 goto ReFault; 1127 } 1128 1129 /* 1130 * copy data, kill loan, and drop uobj lock 1131 * (if any) 1132 */ 1133 /* copy old -> new */ 1134 uvm_pagecopy(anon->u.an_page, pg); 1135 1136 /* force reload */ 1137 pmap_page_protect(anon->u.an_page, 1138 VM_PROT_NONE); 1139 uvm_lock_pageq(); /* KILL loan */ 1140 if (uobj) 1141 /* if we were loaning */ 1142 anon->u.an_page->loan_count--; 1143 anon->u.an_page->uanon = NULL; 1144 /* in case we owned */ 1145 anon->u.an_page->pqflags &= ~PQ_ANON; 1146 uvm_unlock_pageq(); 1147 if (uobj) { 1148 simple_unlock(&uobj->vmobjlock); 1149 uobj = NULL; 1150 } 1151 1152 /* install new page in anon */ 1153 anon->u.an_page = pg; 1154 pg->uanon = anon; 1155 pg->pqflags |= PQ_ANON; 1156 pg->flags &= ~(PG_BUSY|PG_FAKE); 1157 UVM_PAGE_OWN(pg, NULL); 1158 1159 /* done! */ 1160 } /* ref == 1 */ 1161 } /* write fault */ 1162 } /* loan count */ 1163 1164 /* 1165 * if we are case 1B then we will need to allocate a new blank 1166 * anon to transfer the data into. note that we have a lock 1167 * on anon, so no one can busy or release the page until we are done. 1168 * also note that the ref count can't drop to zero here because 1169 * it is > 1 and we are only dropping one ref. 1170 * 1171 * in the (hopefully very rare) case that we are out of RAM we 1172 * will unlock, wait for more RAM, and refault. 1173 * 1174 * if we are out of anon VM we kill the process (XXX: could wait?). 1175 */ 1176 1177 if ((access_type & VM_PROT_WRITE) != 0 && anon->an_ref > 1) { 1178 1179 UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0); 1180 uvmexp.flt_acow++; 1181 oanon = anon; /* oanon = old, locked anon */ 1182 anon = uvm_analloc(); 1183 if (anon) { 1184 /* new anon is locked! */ 1185 pg = uvm_pagealloc(NULL, 0, anon, 0); 1186 } 1187 1188 /* check for out of RAM */ 1189 if (anon == NULL || pg == NULL) { 1190 if (anon) { 1191 anon->an_ref--; 1192 simple_unlock(&anon->an_lock); 1193 uvm_anfree(anon); 1194 } 1195 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1196 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1197 if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) { 1198 UVMHIST_LOG(maphist, 1199 "<- failed. out of VM",0,0,0,0); 1200 uvmexp.fltnoanon++; 1201 return (KERN_RESOURCE_SHORTAGE); 1202 } 1203 1204 uvmexp.fltnoram++; 1205 uvm_wait("flt_noram3"); /* out of RAM, wait for more */ 1206 goto ReFault; 1207 } 1208 1209 /* got all resources, replace anon with nanon */ 1210 1211 uvm_pagecopy(oanon->u.an_page, pg); /* pg now !PG_CLEAN */ 1212 pg->flags &= ~(PG_BUSY|PG_FAKE); /* un-busy! new page */ 1213 UVM_PAGE_OWN(pg, NULL); 1214 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start, 1215 anon, 1); 1216 1217 /* deref: can not drop to zero here by defn! */ 1218 oanon->an_ref--; 1219 1220 /* 1221 * note: oanon is still locked, as is the new anon. we 1222 * need to check for this later when we unlock oanon; if 1223 * oanon != anon, we'll have to unlock anon, too. 1224 */ 1225 1226 } else { 1227 1228 uvmexp.flt_anon++; 1229 oanon = anon; /* old, locked anon is same as anon */ 1230 pg = anon->u.an_page; 1231 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */ 1232 enter_prot = enter_prot & ~VM_PROT_WRITE; 1233 1234 } 1235 1236 /* locked: maps(read), amap, oanon, anon (if different from oanon) */ 1237 1238 /* 1239 * now map the page in ... 1240 * XXX: old fault unlocks object before pmap_enter. this seems 1241 * suspect since some other thread could blast the page out from 1242 * under us between the unlock and the pmap_enter. 1243 */ 1244 1245 UVMHIST_LOG(maphist, " MAPPING: anon: pm=0x%x, va=0x%x, pg=0x%x", 1246 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, 0); 1247 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1248 enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) 1249 != KERN_SUCCESS) { 1250 /* 1251 * No need to undo what we did; we can simply think of 1252 * this as the pmap throwing away the mapping information. 1253 * 1254 * We do, however, have to go through the ReFault path, 1255 * as the map may change while we're asleep. 1256 */ 1257 if (anon != oanon) 1258 simple_unlock(&anon->an_lock); 1259 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1260 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1261 if (uvmexp.swpgonly == uvmexp.swpages) { 1262 UVMHIST_LOG(maphist, 1263 "<- failed. out of VM",0,0,0,0); 1264 /* XXX instrumentation */ 1265 return (KERN_RESOURCE_SHORTAGE); 1266 } 1267 /* XXX instrumentation */ 1268 uvm_wait("flt_pmfail1"); 1269 goto ReFault; 1270 } 1271 1272 /* 1273 * ... update the page queues. 1274 */ 1275 1276 uvm_lock_pageq(); 1277 1278 if (fault_type == VM_FAULT_WIRE) { 1279 uvm_pagewire(pg); 1280 1281 /* 1282 * since the now-wired page cannot be paged out, 1283 * release its swap resources for others to use. 1284 * since an anon with no swap cannot be PG_CLEAN, 1285 * clear its clean flag now. 1286 */ 1287 1288 pg->flags &= ~(PG_CLEAN); 1289 uvm_anon_dropswap(anon); 1290 } else { 1291 /* activate it */ 1292 uvm_pageactivate(pg); 1293 } 1294 1295 uvm_unlock_pageq(); 1296 1297 /* 1298 * done case 1! finish up by unlocking everything and returning success 1299 */ 1300 1301 if (anon != oanon) 1302 simple_unlock(&anon->an_lock); 1303 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1304 return (KERN_SUCCESS); 1305 1306 1307 Case2: 1308 /* 1309 * handle case 2: faulting on backing object or zero fill 1310 */ 1311 1312 /* 1313 * locked: 1314 * maps(read), amap(if there), uobj(if !null), uobjpage(if !null) 1315 */ 1316 1317 /* 1318 * note that uobjpage can not be PGO_DONTCARE at this point. we now 1319 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we 1320 * have a backing object, check and see if we are going to promote 1321 * the data up to an anon during the fault. 1322 */ 1323 1324 if (uobj == NULL) { 1325 uobjpage = PGO_DONTCARE; 1326 promote = TRUE; /* always need anon here */ 1327 } else { 1328 KASSERT(uobjpage != PGO_DONTCARE); 1329 promote = (access_type & VM_PROT_WRITE) && 1330 UVM_ET_ISCOPYONWRITE(ufi.entry); 1331 } 1332 UVMHIST_LOG(maphist, " case 2 fault: promote=%d, zfill=%d", 1333 promote, (uobj == NULL), 0,0); 1334 1335 /* 1336 * if uobjpage is not null then we do not need to do I/O to get the 1337 * uobjpage. 1338 * 1339 * if uobjpage is null, then we need to unlock and ask the pager to 1340 * get the data for us. once we have the data, we need to reverify 1341 * the state the world. we are currently not holding any resources. 1342 */ 1343 1344 if (uobjpage) { 1345 /* update rusage counters */ 1346 curproc->p_addr->u_stats.p_ru.ru_minflt++; 1347 } else { 1348 /* update rusage counters */ 1349 curproc->p_addr->u_stats.p_ru.ru_majflt++; 1350 1351 /* locked: maps(read), amap(if there), uobj */ 1352 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1353 /* locked: uobj */ 1354 1355 uvmexp.fltget++; 1356 gotpages = 1; 1357 uoff = (ufi.orig_rvaddr - ufi.entry->start) + ufi.entry->offset; 1358 result = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages, 1359 0, access_type & MASK(ufi.entry), ufi.entry->advice, 1360 PGO_SYNCIO); 1361 1362 /* locked: uobjpage(if result OK) */ 1363 1364 /* 1365 * recover from I/O 1366 */ 1367 1368 if (result != 0) { 1369 if (result == EAGAIN) { 1370 UVMHIST_LOG(maphist, 1371 " pgo_get says TRY AGAIN!",0,0,0,0); 1372 tsleep(&lbolt, PVM, "fltagain2", 0); 1373 goto ReFault; 1374 } 1375 1376 UVMHIST_LOG(maphist, "<- pgo_get failed (code %d)", 1377 result, 0,0,0); 1378 return (KERN_PROTECTION_FAILURE); /* XXX i/o error */ 1379 } 1380 1381 /* locked: uobjpage */ 1382 1383 /* 1384 * re-verify the state of the world by first trying to relock 1385 * the maps. always relock the object. 1386 */ 1387 1388 locked = uvmfault_relock(&ufi); 1389 if (locked && amap) 1390 amap_lock(amap); 1391 simple_lock(&uobj->vmobjlock); 1392 1393 /* locked(locked): maps(read), amap(if !null), uobj, uobjpage */ 1394 /* locked(!locked): uobj, uobjpage */ 1395 1396 /* 1397 * verify that the page has not be released and re-verify 1398 * that amap slot is still free. if there is a problem, 1399 * we unlock and clean up. 1400 */ 1401 1402 if ((uobjpage->flags & PG_RELEASED) != 0 || 1403 (locked && amap && 1404 amap_lookup(&ufi.entry->aref, 1405 ufi.orig_rvaddr - ufi.entry->start))) { 1406 if (locked) 1407 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1408 locked = FALSE; 1409 } 1410 1411 /* 1412 * didn't get the lock? release the page and retry. 1413 */ 1414 1415 if (locked == FALSE) { 1416 1417 UVMHIST_LOG(maphist, 1418 " wasn't able to relock after fault: retry", 1419 0,0,0,0); 1420 if (uobjpage->flags & PG_WANTED) 1421 /* still holding object lock */ 1422 wakeup(uobjpage); 1423 1424 if (uobjpage->flags & PG_RELEASED) { 1425 uvmexp.fltpgrele++; 1426 KASSERT(uobj->pgops->pgo_releasepg != NULL); 1427 1428 /* frees page */ 1429 if (uobj->pgops->pgo_releasepg(uobjpage,NULL)) 1430 /* unlock if still alive */ 1431 simple_unlock(&uobj->vmobjlock); 1432 goto ReFault; 1433 } 1434 1435 uvm_lock_pageq(); 1436 /* make sure it is in queues */ 1437 uvm_pageactivate(uobjpage); 1438 1439 uvm_unlock_pageq(); 1440 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1441 UVM_PAGE_OWN(uobjpage, NULL); 1442 simple_unlock(&uobj->vmobjlock); 1443 goto ReFault; 1444 1445 } 1446 1447 /* 1448 * we have the data in uobjpage which is PG_BUSY and 1449 * !PG_RELEASED. we are holding object lock (so the page 1450 * can't be released on us). 1451 */ 1452 1453 /* locked: maps(read), amap(if !null), uobj, uobjpage */ 1454 } 1455 1456 /* 1457 * locked: 1458 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj) 1459 */ 1460 1461 /* 1462 * notes: 1463 * - at this point uobjpage can not be NULL 1464 * - at this point uobjpage can not be PG_RELEASED (since we checked 1465 * for it above) 1466 * - at this point uobjpage could be PG_WANTED (handle later) 1467 */ 1468 1469 if (promote == FALSE) { 1470 1471 /* 1472 * we are not promoting. if the mapping is COW ensure that we 1473 * don't give more access than we should (e.g. when doing a read 1474 * fault on a COPYONWRITE mapping we want to map the COW page in 1475 * R/O even though the entry protection could be R/W). 1476 * 1477 * set "pg" to the page we want to map in (uobjpage, usually) 1478 */ 1479 1480 /* no anon in this case. */ 1481 anon = NULL; 1482 1483 uvmexp.flt_obj++; 1484 if (UVM_ET_ISCOPYONWRITE(ufi.entry)) 1485 enter_prot &= ~VM_PROT_WRITE; 1486 pg = uobjpage; /* map in the actual object */ 1487 1488 /* assert(uobjpage != PGO_DONTCARE) */ 1489 1490 /* 1491 * we are faulting directly on the page. be careful 1492 * about writing to loaned pages... 1493 */ 1494 if (uobjpage->loan_count) { 1495 1496 if ((access_type & VM_PROT_WRITE) == 0) { 1497 /* read fault: cap the protection at readonly */ 1498 /* cap! */ 1499 enter_prot = enter_prot & ~VM_PROT_WRITE; 1500 } else { 1501 /* write fault: must break the loan here */ 1502 1503 /* alloc new un-owned page */ 1504 pg = uvm_pagealloc(NULL, 0, NULL, 0); 1505 1506 if (pg == NULL) { 1507 /* 1508 * drop ownership of page, it can't 1509 * be released 1510 */ 1511 if (uobjpage->flags & PG_WANTED) 1512 wakeup(uobjpage); 1513 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1514 UVM_PAGE_OWN(uobjpage, NULL); 1515 1516 uvm_lock_pageq(); 1517 /* activate: we will need it later */ 1518 uvm_pageactivate(uobjpage); 1519 1520 uvm_unlock_pageq(); 1521 uvmfault_unlockall(&ufi, amap, uobj, 1522 NULL); 1523 UVMHIST_LOG(maphist, 1524 " out of RAM breaking loan, waiting", 1525 0,0,0,0); 1526 uvmexp.fltnoram++; 1527 uvm_wait("flt_noram4"); 1528 goto ReFault; 1529 } 1530 1531 /* 1532 * copy the data from the old page to the new 1533 * one and clear the fake/clean flags on the 1534 * new page (keep it busy). force a reload 1535 * of the old page by clearing it from all 1536 * pmaps. then lock the page queues to 1537 * rename the pages. 1538 */ 1539 uvm_pagecopy(uobjpage, pg); /* old -> new */ 1540 pg->flags &= ~(PG_FAKE|PG_CLEAN); 1541 pmap_page_protect(uobjpage, VM_PROT_NONE); 1542 if (uobjpage->flags & PG_WANTED) 1543 wakeup(uobjpage); 1544 /* uobj still locked */ 1545 uobjpage->flags &= ~(PG_WANTED|PG_BUSY); 1546 UVM_PAGE_OWN(uobjpage, NULL); 1547 1548 uvm_lock_pageq(); 1549 offset = uobjpage->offset; 1550 /* remove old page */ 1551 uvm_pagerealloc(uobjpage, NULL, 0); 1552 1553 /* 1554 * at this point we have absolutely no 1555 * control over uobjpage 1556 */ 1557 /* install new page */ 1558 uvm_pagerealloc(pg, uobj, offset); 1559 uvm_unlock_pageq(); 1560 1561 /* 1562 * done! loan is broken and "pg" is 1563 * PG_BUSY. it can now replace uobjpage. 1564 */ 1565 1566 uobjpage = pg; 1567 1568 } /* write fault case */ 1569 } /* if loan_count */ 1570 1571 } else { 1572 1573 /* 1574 * if we are going to promote the data to an anon we 1575 * allocate a blank anon here and plug it into our amap. 1576 */ 1577 #if DIAGNOSTIC 1578 if (amap == NULL) 1579 panic("uvm_fault: want to promote data, but no anon"); 1580 #endif 1581 1582 anon = uvm_analloc(); 1583 if (anon) { 1584 /* 1585 * The new anon is locked. 1586 * 1587 * In `Fill in data...' below, if 1588 * uobjpage == PGO_DONTCARE, we want 1589 * a zero'd, dirty page, so have 1590 * uvm_pagealloc() do that for us. 1591 */ 1592 pg = uvm_pagealloc(NULL, 0, anon, 1593 (uobjpage == PGO_DONTCARE) ? UVM_PGA_ZERO : 0); 1594 } 1595 1596 /* 1597 * out of memory resources? 1598 */ 1599 if (anon == NULL || pg == NULL) { 1600 1601 if (anon != NULL) { 1602 anon->an_ref--; 1603 simple_unlock(&anon->an_lock); 1604 uvm_anfree(anon); 1605 } 1606 1607 /* 1608 * arg! must unbusy our page and fail or sleep. 1609 */ 1610 if (uobjpage != PGO_DONTCARE) { 1611 if (uobjpage->flags & PG_WANTED) 1612 /* still holding object lock */ 1613 wakeup(uobjpage); 1614 1615 uvm_lock_pageq(); 1616 uvm_pageactivate(uobjpage); 1617 uvm_unlock_pageq(); 1618 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1619 UVM_PAGE_OWN(uobjpage, NULL); 1620 } 1621 1622 /* unlock and fail ... */ 1623 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1624 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1625 if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) { 1626 UVMHIST_LOG(maphist, " promote: out of VM", 1627 0,0,0,0); 1628 uvmexp.fltnoanon++; 1629 return (KERN_RESOURCE_SHORTAGE); 1630 } 1631 1632 UVMHIST_LOG(maphist, " out of RAM, waiting for more", 1633 0,0,0,0); 1634 uvmexp.fltnoram++; 1635 uvm_wait("flt_noram5"); 1636 goto ReFault; 1637 } 1638 1639 /* 1640 * fill in the data 1641 */ 1642 1643 if (uobjpage != PGO_DONTCARE) { 1644 uvmexp.flt_prcopy++; 1645 /* copy page [pg now dirty] */ 1646 uvm_pagecopy(uobjpage, pg); 1647 1648 /* 1649 * promote to shared amap? make sure all sharing 1650 * procs see it 1651 */ 1652 if ((amap_flags(amap) & AMAP_SHARED) != 0) { 1653 pmap_page_protect(uobjpage, VM_PROT_NONE); 1654 } 1655 1656 /* 1657 * dispose of uobjpage. it can't be PG_RELEASED 1658 * since we still hold the object lock. 1659 * drop handle to uobj as well. 1660 */ 1661 1662 if (uobjpage->flags & PG_WANTED) 1663 /* still have the obj lock */ 1664 wakeup(uobjpage); 1665 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1666 UVM_PAGE_OWN(uobjpage, NULL); 1667 uvm_lock_pageq(); 1668 uvm_pageactivate(uobjpage); 1669 uvm_unlock_pageq(); 1670 simple_unlock(&uobj->vmobjlock); 1671 uobj = NULL; 1672 1673 UVMHIST_LOG(maphist, 1674 " promote uobjpage 0x%x to anon/page 0x%x/0x%x", 1675 uobjpage, anon, pg, 0); 1676 1677 } else { 1678 uvmexp.flt_przero++; 1679 /* 1680 * Page is zero'd and marked dirty by uvm_pagealloc() 1681 * above. 1682 */ 1683 UVMHIST_LOG(maphist," zero fill anon/page 0x%x/0%x", 1684 anon, pg, 0, 0); 1685 } 1686 1687 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start, 1688 anon, 0); 1689 } 1690 1691 /* 1692 * locked: 1693 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj), 1694 * anon(if !null), pg(if anon) 1695 * 1696 * note: pg is either the uobjpage or the new page in the new anon 1697 */ 1698 1699 /* 1700 * all resources are present. we can now map it in and free our 1701 * resources. 1702 */ 1703 1704 UVMHIST_LOG(maphist, 1705 " MAPPING: case2: pm=0x%x, va=0x%x, pg=0x%x, promote=%d", 1706 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, promote); 1707 KASSERT(access_type == VM_PROT_READ || (pg->flags & PG_RDONLY) == 0); 1708 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1709 pg->flags & PG_RDONLY ? VM_PROT_READ : enter_prot, 1710 access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) 1711 != KERN_SUCCESS) { 1712 1713 /* 1714 * No need to undo what we did; we can simply think of 1715 * this as the pmap throwing away the mapping information. 1716 * 1717 * We do, however, have to go through the ReFault path, 1718 * as the map may change while we're asleep. 1719 */ 1720 1721 if (pg->flags & PG_WANTED) 1722 wakeup(pg); /* lock still held */ 1723 1724 /* 1725 * note that pg can't be PG_RELEASED since we did not drop 1726 * the object lock since the last time we checked. 1727 */ 1728 1729 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); 1730 UVM_PAGE_OWN(pg, NULL); 1731 uvmfault_unlockall(&ufi, amap, uobj, anon); 1732 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1733 if (uvmexp.swpgonly == uvmexp.swpages) { 1734 UVMHIST_LOG(maphist, 1735 "<- failed. out of VM",0,0,0,0); 1736 /* XXX instrumentation */ 1737 return (KERN_RESOURCE_SHORTAGE); 1738 } 1739 /* XXX instrumentation */ 1740 uvm_wait("flt_pmfail2"); 1741 goto ReFault; 1742 } 1743 1744 uvm_lock_pageq(); 1745 1746 if (fault_type == VM_FAULT_WIRE) { 1747 uvm_pagewire(pg); 1748 if (pg->pqflags & PQ_AOBJ) { 1749 1750 /* 1751 * since the now-wired page cannot be paged out, 1752 * release its swap resources for others to use. 1753 * since an aobj page with no swap cannot be PG_CLEAN, 1754 * clear its clean flag now. 1755 */ 1756 1757 pg->flags &= ~(PG_CLEAN); 1758 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT); 1759 } 1760 } else { 1761 /* activate it */ 1762 uvm_pageactivate(pg); 1763 } 1764 uvm_unlock_pageq(); 1765 1766 if (pg->flags & PG_WANTED) 1767 wakeup(pg); /* lock still held */ 1768 1769 /* 1770 * note that pg can't be PG_RELEASED since we did not drop the object 1771 * lock since the last time we checked. 1772 */ 1773 1774 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); 1775 UVM_PAGE_OWN(pg, NULL); 1776 uvmfault_unlockall(&ufi, amap, uobj, anon); 1777 1778 UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0); 1779 return (KERN_SUCCESS); 1780 } 1781 1782 1783 /* 1784 * uvm_fault_wire: wire down a range of virtual addresses in a map. 1785 * 1786 * => map may be read-locked by caller, but MUST NOT be write-locked. 1787 * => if map is read-locked, any operations which may cause map to 1788 * be write-locked in uvm_fault() must be taken care of by 1789 * the caller. See uvm_map_pageable(). 1790 */ 1791 1792 int 1793 uvm_fault_wire(map, start, end, access_type) 1794 vm_map_t map; 1795 vaddr_t start, end; 1796 vm_prot_t access_type; 1797 { 1798 vaddr_t va; 1799 pmap_t pmap; 1800 int rv; 1801 1802 pmap = vm_map_pmap(map); 1803 1804 /* 1805 * now fault it in a page at a time. if the fault fails then we have 1806 * to undo what we have done. note that in uvm_fault VM_PROT_NONE 1807 * is replaced with the max protection if fault_type is VM_FAULT_WIRE. 1808 */ 1809 1810 for (va = start ; va < end ; va += PAGE_SIZE) { 1811 rv = uvm_fault(map, va, VM_FAULT_WIRE, access_type); 1812 if (rv) { 1813 if (va != start) { 1814 uvm_fault_unwire(map, start, va); 1815 } 1816 return (rv); 1817 } 1818 } 1819 1820 return (KERN_SUCCESS); 1821 } 1822 1823 /* 1824 * uvm_fault_unwire(): unwire range of virtual space. 1825 */ 1826 1827 void 1828 uvm_fault_unwire(map, start, end) 1829 vm_map_t map; 1830 vaddr_t start, end; 1831 { 1832 1833 vm_map_lock_read(map); 1834 uvm_fault_unwire_locked(map, start, end); 1835 vm_map_unlock_read(map); 1836 } 1837 1838 /* 1839 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire(). 1840 * 1841 * => map must be at least read-locked. 1842 */ 1843 1844 void 1845 uvm_fault_unwire_locked(map, start, end) 1846 vm_map_t map; 1847 vaddr_t start, end; 1848 { 1849 vm_map_entry_t entry; 1850 pmap_t pmap = vm_map_pmap(map); 1851 vaddr_t va; 1852 paddr_t pa; 1853 struct vm_page *pg; 1854 1855 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0); 1856 1857 /* 1858 * we assume that the area we are unwiring has actually been wired 1859 * in the first place. this means that we should be able to extract 1860 * the PAs from the pmap. we also lock out the page daemon so that 1861 * we can call uvm_pageunwire. 1862 */ 1863 1864 uvm_lock_pageq(); 1865 1866 /* 1867 * find the beginning map entry for the region. 1868 */ 1869 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map)); 1870 if (uvm_map_lookup_entry(map, start, &entry) == FALSE) 1871 panic("uvm_fault_unwire_locked: address not in map"); 1872 1873 for (va = start; va < end ; va += PAGE_SIZE) { 1874 if (pmap_extract(pmap, va, &pa) == FALSE) 1875 panic("uvm_fault_unwire_locked: unwiring " 1876 "non-wired memory"); 1877 1878 /* 1879 * make sure the current entry is for the address we're 1880 * dealing with. if not, grab the next entry. 1881 */ 1882 1883 KASSERT(va >= entry->start); 1884 if (va >= entry->end) { 1885 KASSERT(entry->next != &map->header && 1886 entry->next->start <= entry->end); 1887 entry = entry->next; 1888 } 1889 1890 /* 1891 * if the entry is no longer wired, tell the pmap. 1892 */ 1893 if (VM_MAPENT_ISWIRED(entry) == 0) 1894 pmap_unwire(pmap, va); 1895 1896 pg = PHYS_TO_VM_PAGE(pa); 1897 if (pg) 1898 uvm_pageunwire(pg); 1899 } 1900 1901 uvm_unlock_pageq(); 1902 } 1903