1 /* $OpenBSD: uvm_fault.c,v 1.19 2001/09/19 20:50:59 mickey Exp $ */ 2 /* $NetBSD: uvm_fault.c,v 1.48 2000/04/10 01:17:41 thorpej Exp $ */ 3 4 /* 5 * 6 * Copyright (c) 1997 Charles D. Cranor and Washington University. 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed by Charles D. Cranor and 20 * Washington University. 21 * 4. The name of the author may not be used to endorse or promote products 22 * derived from this software without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 25 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 26 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 27 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 28 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 29 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 30 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 31 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 32 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 33 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 34 * 35 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp 36 */ 37 38 /* 39 * uvm_fault.c: fault handler 40 */ 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/kernel.h> 45 #include <sys/proc.h> 46 #include <sys/malloc.h> 47 #include <sys/mman.h> 48 #include <sys/user.h> 49 50 #include <vm/vm.h> 51 #include <vm/vm_page.h> 52 53 #include <uvm/uvm.h> 54 55 /* 56 * 57 * a word on page faults: 58 * 59 * types of page faults we handle: 60 * 61 * CASE 1: upper layer faults CASE 2: lower layer faults 62 * 63 * CASE 1A CASE 1B CASE 2A CASE 2B 64 * read/write1 write>1 read/write +-cow_write/zero 65 * | | | | 66 * +--|--+ +--|--+ +-----+ + | + | +-----+ 67 * amap | V | | ----------->new| | | | ^ | 68 * +-----+ +-----+ +-----+ + | + | +--|--+ 69 * | | | 70 * +-----+ +-----+ +--|--+ | +--|--+ 71 * uobj | d/c | | d/c | | V | +----| | 72 * +-----+ +-----+ +-----+ +-----+ 73 * 74 * d/c = don't care 75 * 76 * case [0]: layerless fault 77 * no amap or uobj is present. this is an error. 78 * 79 * case [1]: upper layer fault [anon active] 80 * 1A: [read] or [write with anon->an_ref == 1] 81 * I/O takes place in top level anon and uobj is not touched. 82 * 1B: [write with anon->an_ref > 1] 83 * new anon is alloc'd and data is copied off ["COW"] 84 * 85 * case [2]: lower layer fault [uobj] 86 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area] 87 * I/O takes place directly in object. 88 * 2B: [write to copy_on_write] or [read on NULL uobj] 89 * data is "promoted" from uobj to a new anon. 90 * if uobj is null, then we zero fill. 91 * 92 * we follow the standard UVM locking protocol ordering: 93 * 94 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ) 95 * we hold a PG_BUSY page if we unlock for I/O 96 * 97 * 98 * the code is structured as follows: 99 * 100 * - init the "IN" params in the ufi structure 101 * ReFault: 102 * - do lookups [locks maps], check protection, handle needs_copy 103 * - check for case 0 fault (error) 104 * - establish "range" of fault 105 * - if we have an amap lock it and extract the anons 106 * - if sequential advice deactivate pages behind us 107 * - at the same time check pmap for unmapped areas and anon for pages 108 * that we could map in (and do map it if found) 109 * - check object for resident pages that we could map in 110 * - if (case 2) goto Case2 111 * - >>> handle case 1 112 * - ensure source anon is resident in RAM 113 * - if case 1B alloc new anon and copy from source 114 * - map the correct page in 115 * Case2: 116 * - >>> handle case 2 117 * - ensure source page is resident (if uobj) 118 * - if case 2B alloc new anon and copy from source (could be zero 119 * fill if uobj == NULL) 120 * - map the correct page in 121 * - done! 122 * 123 * note on paging: 124 * if we have to do I/O we place a PG_BUSY page in the correct object, 125 * unlock everything, and do the I/O. when I/O is done we must reverify 126 * the state of the world before assuming that our data structures are 127 * valid. [because mappings could change while the map is unlocked] 128 * 129 * alternative 1: unbusy the page in question and restart the page fault 130 * from the top (ReFault). this is easy but does not take advantage 131 * of the information that we already have from our previous lookup, 132 * although it is possible that the "hints" in the vm_map will help here. 133 * 134 * alternative 2: the system already keeps track of a "version" number of 135 * a map. [i.e. every time you write-lock a map (e.g. to change a 136 * mapping) you bump the version number up by one...] so, we can save 137 * the version number of the map before we release the lock and start I/O. 138 * then when I/O is done we can relock and check the version numbers 139 * to see if anything changed. this might save us some over 1 because 140 * we don't have to unbusy the page and may be less compares(?). 141 * 142 * alternative 3: put in backpointers or a way to "hold" part of a map 143 * in place while I/O is in progress. this could be complex to 144 * implement (especially with structures like amap that can be referenced 145 * by multiple map entries, and figuring out what should wait could be 146 * complex as well...). 147 * 148 * given that we are not currently multiprocessor or multithreaded we might 149 * as well choose alternative 2 now. maybe alternative 3 would be useful 150 * in the future. XXX keep in mind for future consideration//rechecking. 151 */ 152 153 /* 154 * local data structures 155 */ 156 157 struct uvm_advice { 158 int advice; 159 int nback; 160 int nforw; 161 }; 162 163 /* 164 * page range array: 165 * note: index in array must match "advice" value 166 * XXX: borrowed numbers from freebsd. do they work well for us? 167 */ 168 169 static struct uvm_advice uvmadvice[] = { 170 { MADV_NORMAL, 3, 4 }, 171 { MADV_RANDOM, 0, 0 }, 172 { MADV_SEQUENTIAL, 8, 7}, 173 }; 174 175 #define UVM_MAXRANGE 16 /* must be max() of nback+nforw+1 */ 176 177 /* 178 * private prototypes 179 */ 180 181 static void uvmfault_amapcopy __P((struct uvm_faultinfo *)); 182 static __inline void uvmfault_anonflush __P((struct vm_anon **, int)); 183 184 /* 185 * inline functions 186 */ 187 188 /* 189 * uvmfault_anonflush: try and deactivate pages in specified anons 190 * 191 * => does not have to deactivate page if it is busy 192 */ 193 194 static __inline void 195 uvmfault_anonflush(anons, n) 196 struct vm_anon **anons; 197 int n; 198 { 199 int lcv; 200 struct vm_page *pg; 201 202 for (lcv = 0 ; lcv < n ; lcv++) { 203 if (anons[lcv] == NULL) 204 continue; 205 simple_lock(&anons[lcv]->an_lock); 206 pg = anons[lcv]->u.an_page; 207 if (pg && (pg->flags & PG_BUSY) == 0 && pg->loan_count == 0) { 208 uvm_lock_pageq(); 209 if (pg->wire_count == 0) { 210 pmap_page_protect(pg, VM_PROT_NONE); 211 uvm_pagedeactivate(pg); 212 } 213 uvm_unlock_pageq(); 214 } 215 simple_unlock(&anons[lcv]->an_lock); 216 } 217 } 218 219 /* 220 * normal functions 221 */ 222 223 /* 224 * uvmfault_amapcopy: clear "needs_copy" in a map. 225 * 226 * => called with VM data structures unlocked (usually, see below) 227 * => we get a write lock on the maps and clear needs_copy for a VA 228 * => if we are out of RAM we sleep (waiting for more) 229 */ 230 231 static void 232 uvmfault_amapcopy(ufi) 233 struct uvm_faultinfo *ufi; 234 { 235 236 /* 237 * while we haven't done the job 238 */ 239 240 while (1) { 241 242 /* 243 * no mapping? give up. 244 */ 245 246 if (uvmfault_lookup(ufi, TRUE) == FALSE) 247 return; 248 249 /* 250 * copy if needed. 251 */ 252 253 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) 254 amap_copy(ufi->map, ufi->entry, M_NOWAIT, TRUE, 255 ufi->orig_rvaddr, ufi->orig_rvaddr + 1); 256 257 /* 258 * didn't work? must be out of RAM. unlock and sleep. 259 */ 260 261 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 262 uvmfault_unlockmaps(ufi, TRUE); 263 uvm_wait("fltamapcopy"); 264 continue; 265 } 266 267 /* 268 * got it! unlock and return. 269 */ 270 271 uvmfault_unlockmaps(ufi, TRUE); 272 return; 273 } 274 /*NOTREACHED*/ 275 } 276 277 /* 278 * uvmfault_anonget: get data in an anon into a non-busy, non-released 279 * page in that anon. 280 * 281 * => maps, amap, and anon locked by caller. 282 * => if we fail (result != VM_PAGER_OK) we unlock everything except anon. 283 * => if we are successful, we return with everything still locked. 284 * => we don't move the page on the queues [gets moved later] 285 * => if we allocate a new page [we_own], it gets put on the queues. 286 * either way, the result is that the page is on the queues at return time 287 * => for pages which are on loan from a uvm_object (and thus are not 288 * owned by the anon): if successful, we return with the owning object 289 * locked. the caller must unlock this object when it unlocks everything 290 * else. 291 */ 292 293 int 294 uvmfault_anonget(ufi, amap, anon) 295 struct uvm_faultinfo *ufi; 296 struct vm_amap *amap; 297 struct vm_anon *anon; 298 { 299 boolean_t we_own; /* we own anon's page? */ 300 boolean_t locked; /* did we relock? */ 301 struct vm_page *pg; 302 int result; 303 UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist); 304 305 result = 0; /* XXX shut up gcc */ 306 uvmexp.fltanget++; 307 /* bump rusage counters */ 308 if (anon->u.an_page) 309 curproc->p_addr->u_stats.p_ru.ru_minflt++; 310 else 311 curproc->p_addr->u_stats.p_ru.ru_majflt++; 312 313 /* 314 * loop until we get it, or fail. 315 */ 316 317 while (1) { 318 319 we_own = FALSE; /* TRUE if we set PG_BUSY on a page */ 320 pg = anon->u.an_page; 321 322 /* 323 * if there is a resident page and it is loaned, then anon 324 * may not own it. call out to uvm_anon_lockpage() to ensure 325 * the real owner of the page has been identified and locked. 326 */ 327 328 if (pg && pg->loan_count) 329 pg = uvm_anon_lockloanpg(anon); 330 331 /* 332 * page there? make sure it is not busy/released. 333 */ 334 335 if (pg) { 336 337 /* 338 * at this point, if the page has a uobject [meaning 339 * we have it on loan], then that uobject is locked 340 * by us! if the page is busy, we drop all the 341 * locks (including uobject) and try again. 342 */ 343 344 if ((pg->flags & (PG_BUSY|PG_RELEASED)) == 0) { 345 UVMHIST_LOG(maphist, "<- OK",0,0,0,0); 346 return (VM_PAGER_OK); 347 } 348 pg->flags |= PG_WANTED; 349 uvmexp.fltpgwait++; 350 351 /* 352 * the last unlock must be an atomic unlock+wait on 353 * the owner of page 354 */ 355 if (pg->uobject) { /* owner is uobject ? */ 356 uvmfault_unlockall(ufi, amap, NULL, anon); 357 UVMHIST_LOG(maphist, " unlock+wait on uobj",0, 358 0,0,0); 359 UVM_UNLOCK_AND_WAIT(pg, 360 &pg->uobject->vmobjlock, 361 FALSE, "anonget1",0); 362 } else { 363 /* anon owns page */ 364 uvmfault_unlockall(ufi, amap, NULL, NULL); 365 UVMHIST_LOG(maphist, " unlock+wait on anon",0, 366 0,0,0); 367 UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0, 368 "anonget2",0); 369 } 370 /* ready to relock and try again */ 371 372 } else { 373 374 /* 375 * no page, we must try and bring it in. 376 */ 377 pg = uvm_pagealloc(NULL, 0, anon, 0); 378 379 if (pg == NULL) { /* out of RAM. */ 380 381 uvmfault_unlockall(ufi, amap, NULL, anon); 382 uvmexp.fltnoram++; 383 UVMHIST_LOG(maphist, " noram -- UVM_WAIT",0, 384 0,0,0); 385 uvm_wait("flt_noram1"); 386 /* ready to relock and try again */ 387 388 } else { 389 390 /* we set the PG_BUSY bit */ 391 we_own = TRUE; 392 uvmfault_unlockall(ufi, amap, NULL, anon); 393 394 /* 395 * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN 396 * page into the uvm_swap_get function with 397 * all data structures unlocked. note that 398 * it is ok to read an_swslot here because 399 * we hold PG_BUSY on the page. 400 */ 401 uvmexp.pageins++; 402 result = uvm_swap_get(pg, anon->an_swslot, 403 PGO_SYNCIO); 404 405 /* 406 * we clean up after the i/o below in the 407 * "we_own" case 408 */ 409 /* ready to relock and try again */ 410 } 411 } 412 413 /* 414 * now relock and try again 415 */ 416 417 locked = uvmfault_relock(ufi); 418 if (locked && amap != NULL) { 419 amap_lock(amap); 420 } 421 if (locked || we_own) 422 simple_lock(&anon->an_lock); 423 424 /* 425 * if we own the page (i.e. we set PG_BUSY), then we need 426 * to clean up after the I/O. there are three cases to 427 * consider: 428 * [1] page released during I/O: free anon and ReFault. 429 * [2] I/O not OK. free the page and cause the fault 430 * to fail. 431 * [3] I/O OK! activate the page and sync with the 432 * non-we_own case (i.e. drop anon lock if not locked). 433 */ 434 435 if (we_own) { 436 437 if (pg->flags & PG_WANTED) { 438 /* still holding object lock */ 439 wakeup(pg); 440 } 441 /* un-busy! */ 442 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE); 443 UVM_PAGE_OWN(pg, NULL); 444 445 /* 446 * if we were RELEASED during I/O, then our anon is 447 * no longer part of an amap. we need to free the 448 * anon and try again. 449 */ 450 if (pg->flags & PG_RELEASED) { 451 pmap_page_protect(pg, VM_PROT_NONE); 452 simple_unlock(&anon->an_lock); 453 uvm_anfree(anon); /* frees page for us */ 454 if (locked) 455 uvmfault_unlockall(ufi, amap, NULL, 456 NULL); 457 uvmexp.fltpgrele++; 458 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 459 return (VM_PAGER_REFAULT); /* refault! */ 460 } 461 462 if (result != VM_PAGER_OK) { 463 #ifdef DIAGNOSTIC 464 if (result == VM_PAGER_PEND) { 465 panic("uvmfault_anonget: " 466 "got PENDING for non-async I/O"); 467 } 468 #endif 469 /* remove page from anon */ 470 anon->u.an_page = NULL; 471 472 /* 473 * remove the swap slot from the anon 474 * and mark the anon as having no real slot. 475 * don't free the swap slot, thus preventing 476 * it from being used again. 477 */ 478 uvm_swap_markbad(anon->an_swslot, 1); 479 anon->an_swslot = SWSLOT_BAD; 480 481 /* 482 * note: page was never !PG_BUSY, so it 483 * can't be mapped and thus no need to 484 * pmap_page_protect it... 485 */ 486 uvm_lock_pageq(); 487 uvm_pagefree(pg); 488 uvm_unlock_pageq(); 489 490 if (locked) 491 uvmfault_unlockall(ufi, amap, NULL, 492 anon); 493 else 494 simple_unlock(&anon->an_lock); 495 UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0); 496 return (VM_PAGER_ERROR); 497 } 498 499 /* 500 * must be OK, clear modify (already PG_CLEAN) 501 * and activate 502 */ 503 pmap_clear_modify(pg); 504 uvm_lock_pageq(); 505 uvm_pageactivate(pg); 506 uvm_unlock_pageq(); 507 if (!locked) 508 simple_unlock(&anon->an_lock); 509 } 510 511 /* 512 * we were not able to relock. restart fault. 513 */ 514 515 if (!locked) { 516 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 517 return (VM_PAGER_REFAULT); 518 } 519 520 /* 521 * verify no one has touched the amap and moved the anon on us. 522 */ 523 524 if (ufi != NULL && 525 amap_lookup(&ufi->entry->aref, 526 ufi->orig_rvaddr - ufi->entry->start) != anon) { 527 528 uvmfault_unlockall(ufi, amap, NULL, anon); 529 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 530 return (VM_PAGER_REFAULT); 531 } 532 533 /* 534 * try it again! 535 */ 536 537 uvmexp.fltanretry++; 538 continue; 539 540 } /* while (1) */ 541 542 /*NOTREACHED*/ 543 } 544 545 /* 546 * F A U L T - m a i n e n t r y p o i n t 547 */ 548 549 /* 550 * uvm_fault: page fault handler 551 * 552 * => called from MD code to resolve a page fault 553 * => VM data structures usually should be unlocked. however, it is 554 * possible to call here with the main map locked if the caller 555 * gets a write lock, sets it recusive, and then calls us (c.f. 556 * uvm_map_pageable). this should be avoided because it keeps 557 * the map locked off during I/O. 558 */ 559 560 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 561 ~VM_PROT_WRITE : VM_PROT_ALL) 562 563 int 564 uvm_fault(orig_map, vaddr, fault_type, access_type) 565 vm_map_t orig_map; 566 vaddr_t vaddr; 567 vm_fault_t fault_type; 568 vm_prot_t access_type; 569 { 570 struct uvm_faultinfo ufi; 571 vm_prot_t enter_prot; 572 boolean_t wired, narrow, promote, locked, shadowed; 573 int npages, nback, nforw, centeridx, result, lcv, gotpages; 574 vaddr_t startva, objaddr, currva, offset; 575 paddr_t pa; 576 struct vm_amap *amap; 577 struct uvm_object *uobj; 578 struct vm_anon *anons_store[UVM_MAXRANGE], **anons, *anon, *oanon; 579 struct vm_page *pages[UVM_MAXRANGE], *pg, *uobjpage; 580 UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist); 581 582 UVMHIST_LOG(maphist, "(map=0x%x, vaddr=0x%x, ft=%d, at=%d)", 583 orig_map, vaddr, fault_type, access_type); 584 585 anon = NULL; /* XXX: shut up gcc */ 586 587 uvmexp.faults++; /* XXX: locking? */ 588 589 /* 590 * init the IN parameters in the ufi 591 */ 592 593 ufi.orig_map = orig_map; 594 ufi.orig_rvaddr = trunc_page(vaddr); 595 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ 596 if (fault_type == VM_FAULT_WIRE) 597 narrow = TRUE; /* don't look for neighborhood 598 * pages on wire */ 599 else 600 narrow = FALSE; /* normal fault */ 601 602 /* 603 * before we do anything else, if this is a fault on a kernel 604 * address, check to see if the address is managed by an 605 * interrupt-safe map. If it is, we fail immediately. Intrsafe 606 * maps are never pageable, and this approach avoids an evil 607 * locking mess. 608 */ 609 if (orig_map == kernel_map && uvmfault_check_intrsafe(&ufi)) { 610 UVMHIST_LOG(maphist, "<- VA 0x%lx in intrsafe map %p", 611 ufi.orig_rvaddr, ufi.map, 0, 0); 612 return (KERN_FAILURE); 613 } 614 615 /* 616 * "goto ReFault" means restart the page fault from ground zero. 617 */ 618 ReFault: 619 620 /* 621 * lookup and lock the maps 622 */ 623 624 if (uvmfault_lookup(&ufi, FALSE) == FALSE) { 625 UVMHIST_LOG(maphist, "<- no mapping @ 0x%x", vaddr, 0,0,0); 626 return (KERN_INVALID_ADDRESS); 627 } 628 /* locked: maps(read) */ 629 630 /* 631 * check protection 632 */ 633 634 if ((ufi.entry->protection & access_type) != access_type) { 635 UVMHIST_LOG(maphist, 636 "<- protection failure (prot=0x%x, access=0x%x)", 637 ufi.entry->protection, access_type, 0, 0); 638 uvmfault_unlockmaps(&ufi, FALSE); 639 return (KERN_PROTECTION_FAILURE); 640 } 641 642 /* 643 * if the map is not a pageable map, a page fault always fails. 644 */ 645 646 if ((ufi.map->flags & VM_MAP_PAGEABLE) == 0) { 647 UVMHIST_LOG(maphist, 648 "<- map %p not pageable", ufi.map, 0, 0, 0); 649 uvmfault_unlockmaps(&ufi, FALSE); 650 return (KERN_FAILURE); 651 } 652 653 /* 654 * "enter_prot" is the protection we want to enter the page in at. 655 * for certain pages (e.g. copy-on-write pages) this protection can 656 * be more strict than ufi.entry->protection. "wired" means either 657 * the entry is wired or we are fault-wiring the pg. 658 */ 659 660 enter_prot = ufi.entry->protection; 661 wired = VM_MAPENT_ISWIRED(ufi.entry) || (fault_type == VM_FAULT_WIRE); 662 if (wired) 663 access_type = enter_prot; /* full access for wired */ 664 665 /* 666 * handle "needs_copy" case. if we need to copy the amap we will 667 * have to drop our readlock and relock it with a write lock. (we 668 * need a write lock to change anything in a map entry [e.g. 669 * needs_copy]). 670 */ 671 672 if (UVM_ET_ISNEEDSCOPY(ufi.entry)) { 673 if ((access_type & VM_PROT_WRITE) || 674 (ufi.entry->object.uvm_obj == NULL)) { 675 /* need to clear */ 676 UVMHIST_LOG(maphist, 677 " need to clear needs_copy and refault",0,0,0,0); 678 uvmfault_unlockmaps(&ufi, FALSE); 679 uvmfault_amapcopy(&ufi); 680 uvmexp.fltamcopy++; 681 goto ReFault; 682 683 } else { 684 685 /* 686 * ensure that we pmap_enter page R/O since 687 * needs_copy is still true 688 */ 689 enter_prot &= ~VM_PROT_WRITE; 690 691 } 692 } 693 694 /* 695 * identify the players 696 */ 697 698 amap = ufi.entry->aref.ar_amap; /* top layer */ 699 uobj = ufi.entry->object.uvm_obj; /* bottom layer */ 700 701 /* 702 * check for a case 0 fault. if nothing backing the entry then 703 * error now. 704 */ 705 706 if (amap == NULL && uobj == NULL) { 707 uvmfault_unlockmaps(&ufi, FALSE); 708 UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0); 709 return (KERN_INVALID_ADDRESS); 710 } 711 712 /* 713 * establish range of interest based on advice from mapper 714 * and then clip to fit map entry. note that we only want 715 * to do this the first time through the fault. if we 716 * ReFault we will disable this by setting "narrow" to true. 717 */ 718 719 if (narrow == FALSE) { 720 721 /* wide fault (!narrow) */ 722 #ifdef DIAGNOSTIC 723 if (uvmadvice[ufi.entry->advice].advice != ufi.entry->advice) 724 panic("fault: advice mismatch!"); 725 #endif 726 nback = min(uvmadvice[ufi.entry->advice].nback, 727 (ufi.orig_rvaddr - ufi.entry->start) >> PAGE_SHIFT); 728 startva = ufi.orig_rvaddr - (nback << PAGE_SHIFT); 729 nforw = min(uvmadvice[ufi.entry->advice].nforw, 730 ((ufi.entry->end - ufi.orig_rvaddr) >> 731 PAGE_SHIFT) - 1); 732 /* 733 * note: "-1" because we don't want to count the 734 * faulting page as forw 735 */ 736 npages = nback + nforw + 1; 737 centeridx = nback; 738 739 narrow = TRUE; /* ensure only once per-fault */ 740 741 } else { 742 743 /* narrow fault! */ 744 nback = nforw = 0; 745 startva = ufi.orig_rvaddr; 746 npages = 1; 747 centeridx = 0; 748 749 } 750 751 /* locked: maps(read) */ 752 UVMHIST_LOG(maphist, " narrow=%d, back=%d, forw=%d, startva=0x%x", 753 narrow, nback, nforw, startva); 754 UVMHIST_LOG(maphist, " entry=0x%x, amap=0x%x, obj=0x%x", ufi.entry, 755 amap, uobj, 0); 756 757 /* 758 * if we've got an amap, lock it and extract current anons. 759 */ 760 761 if (amap) { 762 amap_lock(amap); 763 anons = anons_store; 764 amap_lookups(&ufi.entry->aref, startva - ufi.entry->start, 765 anons, npages); 766 } else { 767 anons = NULL; /* to be safe */ 768 } 769 770 /* locked: maps(read), amap(if there) */ 771 772 /* 773 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages 774 * now and then forget about them (for the rest of the fault). 775 */ 776 777 if (ufi.entry->advice == MADV_SEQUENTIAL) { 778 779 UVMHIST_LOG(maphist, " MADV_SEQUENTIAL: flushing backpages", 780 0,0,0,0); 781 /* flush back-page anons? */ 782 if (amap) 783 uvmfault_anonflush(anons, nback); 784 785 /* flush object? */ 786 if (uobj) { 787 objaddr = 788 (startva - ufi.entry->start) + ufi.entry->offset; 789 simple_lock(&uobj->vmobjlock); 790 (void) uobj->pgops->pgo_flush(uobj, objaddr, objaddr + 791 (nback << PAGE_SHIFT), PGO_DEACTIVATE); 792 simple_unlock(&uobj->vmobjlock); 793 } 794 795 /* now forget about the backpages */ 796 if (amap) 797 anons += nback; 798 startva = startva + (nback << PAGE_SHIFT); 799 npages -= nback; 800 nback = centeridx = 0; 801 } 802 803 /* locked: maps(read), amap(if there) */ 804 805 /* 806 * map in the backpages and frontpages we found in the amap in hopes 807 * of preventing future faults. we also init the pages[] array as 808 * we go. 809 */ 810 811 currva = startva; 812 shadowed = FALSE; 813 for (lcv = 0 ; lcv < npages ; lcv++, currva += PAGE_SIZE) { 814 815 /* 816 * dont play with VAs that are already mapped 817 * except for center) 818 */ 819 if (lcv != centeridx) { 820 if (pmap_extract(ufi.orig_map->pmap, currva, &pa) == 821 TRUE) { 822 pages[lcv] = PGO_DONTCARE; 823 continue; 824 } 825 } 826 827 /* 828 * unmapped or center page. check if any anon at this level. 829 */ 830 if (amap == NULL || anons[lcv] == NULL) { 831 pages[lcv] = NULL; 832 continue; 833 } 834 835 /* 836 * check for present page and map if possible. re-activate it. 837 */ 838 839 pages[lcv] = PGO_DONTCARE; 840 if (lcv == centeridx) { /* save center for later! */ 841 shadowed = TRUE; 842 continue; 843 } 844 anon = anons[lcv]; 845 simple_lock(&anon->an_lock); 846 /* ignore loaned pages */ 847 if (anon->u.an_page && anon->u.an_page->loan_count == 0 && 848 (anon->u.an_page->flags & (PG_RELEASED|PG_BUSY)) == 0) { 849 uvm_lock_pageq(); 850 uvm_pageactivate(anon->u.an_page); /* reactivate */ 851 uvm_unlock_pageq(); 852 UVMHIST_LOG(maphist, 853 " MAPPING: n anon: pm=0x%x, va=0x%x, pg=0x%x", 854 ufi.orig_map->pmap, currva, anon->u.an_page, 0); 855 uvmexp.fltnamap++; 856 /* 857 * Since this isn't the page that's actually faulting, 858 * ignore pmap_enter() failures; it's not critical 859 * that we enter these right now. 860 */ 861 (void) pmap_enter(ufi.orig_map->pmap, currva, 862 VM_PAGE_TO_PHYS(anon->u.an_page), 863 (anon->an_ref > 1) ? (enter_prot & ~VM_PROT_WRITE) : 864 enter_prot, 865 PMAP_CANFAIL | 866 (VM_MAPENT_ISWIRED(ufi.entry) ? PMAP_WIRED : 0)); 867 } 868 simple_unlock(&anon->an_lock); 869 } 870 871 /* locked: maps(read), amap(if there) */ 872 /* (shadowed == TRUE) if there is an anon at the faulting address */ 873 UVMHIST_LOG(maphist, " shadowed=%d, will_get=%d", shadowed, 874 (uobj && shadowed == FALSE),0,0); 875 876 /* 877 * note that if we are really short of RAM we could sleep in the above 878 * call to pmap_enter with everything locked. bad? 879 * 880 * XXX Actually, that is bad; pmap_enter() should just fail in that 881 * XXX case. --thorpej 882 */ 883 884 /* 885 * if the desired page is not shadowed by the amap and we have a 886 * backing object, then we check to see if the backing object would 887 * prefer to handle the fault itself (rather than letting us do it 888 * with the usual pgo_get hook). the backing object signals this by 889 * providing a pgo_fault routine. 890 */ 891 892 if (uobj && shadowed == FALSE && uobj->pgops->pgo_fault != NULL) { 893 894 simple_lock(&uobj->vmobjlock); 895 896 /* locked: maps(read), amap (if there), uobj */ 897 result = uobj->pgops->pgo_fault(&ufi, startva, pages, npages, 898 centeridx, fault_type, access_type, 899 PGO_LOCKED); 900 /* locked: nothing, pgo_fault has unlocked everything */ 901 902 if (result == VM_PAGER_OK) 903 return (KERN_SUCCESS); /* pgo_fault did pmap enter */ 904 else if (result == VM_PAGER_REFAULT) 905 goto ReFault; /* try again! */ 906 else 907 return (KERN_PROTECTION_FAILURE); 908 } 909 910 /* 911 * now, if the desired page is not shadowed by the amap and we have 912 * a backing object that does not have a special fault routine, then 913 * we ask (with pgo_get) the object for resident pages that we care 914 * about and attempt to map them in. we do not let pgo_get block 915 * (PGO_LOCKED). 916 * 917 * ("get" has the option of doing a pmap_enter for us) 918 */ 919 920 if (uobj && shadowed == FALSE) { 921 simple_lock(&uobj->vmobjlock); 922 923 /* locked (!shadowed): maps(read), amap (if there), uobj */ 924 /* 925 * the following call to pgo_get does _not_ change locking state 926 */ 927 928 uvmexp.fltlget++; 929 gotpages = npages; 930 result = uobj->pgops->pgo_get(uobj, ufi.entry->offset + 931 (startva - ufi.entry->start), 932 pages, &gotpages, centeridx, 933 access_type & MASK(ufi.entry), 934 ufi.entry->advice, PGO_LOCKED); 935 936 /* 937 * check for pages to map, if we got any 938 */ 939 940 uobjpage = NULL; 941 942 if (gotpages) { 943 currva = startva; 944 for (lcv = 0 ; lcv < npages ; 945 lcv++, currva += PAGE_SIZE) { 946 947 if (pages[lcv] == NULL || 948 pages[lcv] == PGO_DONTCARE) 949 continue; 950 951 #ifdef DIAGNOSTIC 952 /* 953 * pager sanity check: pgo_get with 954 * PGO_LOCKED should never return a 955 * released page to us. 956 */ 957 if (pages[lcv]->flags & PG_RELEASED) 958 panic("uvm_fault: pgo_get PGO_LOCKED gave us a RELEASED page"); 959 #endif 960 961 /* 962 * if center page is resident and not 963 * PG_BUSY|PG_RELEASED then pgo_get 964 * made it PG_BUSY for us and gave 965 * us a handle to it. remember this 966 * page as "uobjpage." (for later use). 967 */ 968 969 if (lcv == centeridx) { 970 uobjpage = pages[lcv]; 971 UVMHIST_LOG(maphist, " got uobjpage (0x%x) with locked get", 972 uobjpage, 0,0,0); 973 continue; 974 } 975 976 /* 977 * note: calling pgo_get with locked data 978 * structures returns us pages which are 979 * neither busy nor released, so we don't 980 * need to check for this. we can just 981 * directly enter the page (after moving it 982 * to the head of the active queue [useful?]). 983 */ 984 985 uvm_lock_pageq(); 986 uvm_pageactivate(pages[lcv]); /* reactivate */ 987 uvm_unlock_pageq(); 988 UVMHIST_LOG(maphist, 989 " MAPPING: n obj: pm=0x%x, va=0x%x, pg=0x%x", 990 ufi.orig_map->pmap, currva, pages[lcv], 0); 991 uvmexp.fltnomap++; 992 /* 993 * Since this page isn't the page that's 994 * actually fauling, ignore pmap_enter() 995 * failures; it's not critical that we 996 * enter these right now. 997 */ 998 (void) pmap_enter(ufi.orig_map->pmap, currva, 999 VM_PAGE_TO_PHYS(pages[lcv]), 1000 enter_prot & MASK(ufi.entry), 1001 PMAP_CANFAIL | 1002 (wired ? PMAP_WIRED : 0)); 1003 1004 /* 1005 * NOTE: page can't be PG_WANTED or PG_RELEASED 1006 * because we've held the lock the whole time 1007 * we've had the handle. 1008 */ 1009 pages[lcv]->flags &= ~(PG_BUSY); /* un-busy! */ 1010 UVM_PAGE_OWN(pages[lcv], NULL); 1011 1012 /* done! */ 1013 } /* for "lcv" loop */ 1014 } /* "gotpages" != 0 */ 1015 1016 /* note: object still _locked_ */ 1017 } else { 1018 1019 uobjpage = NULL; 1020 1021 } 1022 1023 /* locked (shadowed): maps(read), amap */ 1024 /* locked (!shadowed): maps(read), amap(if there), 1025 uobj(if !null), uobjpage(if !null) */ 1026 1027 /* 1028 * note that at this point we are done with any front or back pages. 1029 * we are now going to focus on the center page (i.e. the one we've 1030 * faulted on). if we have faulted on the top (anon) layer 1031 * [i.e. case 1], then the anon we want is anons[centeridx] (we have 1032 * not touched it yet). if we have faulted on the bottom (uobj) 1033 * layer [i.e. case 2] and the page was both present and available, 1034 * then we've got a pointer to it as "uobjpage" and we've already 1035 * made it BUSY. 1036 */ 1037 1038 /* 1039 * there are four possible cases we must address: 1A, 1B, 2A, and 2B 1040 */ 1041 1042 /* 1043 * redirect case 2: if we are not shadowed, go to case 2. 1044 */ 1045 1046 if (shadowed == FALSE) 1047 goto Case2; 1048 1049 /* locked: maps(read), amap */ 1050 1051 /* 1052 * handle case 1: fault on an anon in our amap 1053 */ 1054 1055 anon = anons[centeridx]; 1056 UVMHIST_LOG(maphist, " case 1 fault: anon=0x%x", anon, 0,0,0); 1057 simple_lock(&anon->an_lock); 1058 1059 /* locked: maps(read), amap, anon */ 1060 1061 /* 1062 * no matter if we have case 1A or case 1B we are going to need to 1063 * have the anon's memory resident. ensure that now. 1064 */ 1065 1066 /* 1067 * let uvmfault_anonget do the dirty work. 1068 * if it fails (!OK) it will unlock all but the anon for us. 1069 * if it succeeds, locks are still valid and locked. 1070 * also, if it is OK, then the anon's page is on the queues. 1071 * if the page is on loan from a uvm_object, then anonget will 1072 * lock that object for us if it does not fail. 1073 */ 1074 1075 result = uvmfault_anonget(&ufi, amap, anon); 1076 if (result != VM_PAGER_OK) { 1077 simple_unlock(&anon->an_lock); 1078 } 1079 1080 if (result == VM_PAGER_REFAULT) 1081 goto ReFault; 1082 1083 if (result == VM_PAGER_AGAIN) { 1084 tsleep((caddr_t)&lbolt, PVM, "fltagain1", 0); 1085 goto ReFault; 1086 } 1087 1088 if (result != VM_PAGER_OK) 1089 return (KERN_PROTECTION_FAILURE); /* XXX??? */ 1090 1091 /* 1092 * uobj is non null if the page is on loan from an object (i.e. uobj) 1093 */ 1094 1095 uobj = anon->u.an_page->uobject; /* locked by anonget if !NULL */ 1096 1097 /* locked: maps(read), amap, anon, uobj(if one) */ 1098 1099 /* 1100 * special handling for loaned pages 1101 */ 1102 if (anon->u.an_page->loan_count) { 1103 1104 if ((access_type & VM_PROT_WRITE) == 0) { 1105 1106 /* 1107 * for read faults on loaned pages we just cap the 1108 * protection at read-only. 1109 */ 1110 1111 enter_prot = enter_prot & ~VM_PROT_WRITE; 1112 1113 } else { 1114 /* 1115 * note that we can't allow writes into a loaned page! 1116 * 1117 * if we have a write fault on a loaned page in an 1118 * anon then we need to look at the anon's ref count. 1119 * if it is greater than one then we are going to do 1120 * a normal copy-on-write fault into a new anon (this 1121 * is not a problem). however, if the reference count 1122 * is one (a case where we would normally allow a 1123 * write directly to the page) then we need to kill 1124 * the loan before we continue. 1125 */ 1126 1127 /* >1 case is already ok */ 1128 if (anon->an_ref == 1) { 1129 1130 /* get new un-owned replacement page */ 1131 pg = uvm_pagealloc(NULL, 0, NULL, 0); 1132 if (pg == NULL) { 1133 uvmfault_unlockall(&ufi, amap, uobj, 1134 anon); 1135 uvm_wait("flt_noram2"); 1136 goto ReFault; 1137 } 1138 1139 /* 1140 * copy data, kill loan, and drop uobj lock 1141 * (if any) 1142 */ 1143 /* copy old -> new */ 1144 uvm_pagecopy(anon->u.an_page, pg); 1145 1146 /* force reload */ 1147 pmap_page_protect(anon->u.an_page, 1148 VM_PROT_NONE); 1149 uvm_lock_pageq(); /* KILL loan */ 1150 if (uobj) 1151 /* if we were loaning */ 1152 anon->u.an_page->loan_count--; 1153 anon->u.an_page->uanon = NULL; 1154 /* in case we owned */ 1155 anon->u.an_page->pqflags &= ~PQ_ANON; 1156 uvm_unlock_pageq(); 1157 if (uobj) { 1158 simple_unlock(&uobj->vmobjlock); 1159 uobj = NULL; 1160 } 1161 1162 /* install new page in anon */ 1163 anon->u.an_page = pg; 1164 pg->uanon = anon; 1165 pg->pqflags |= PQ_ANON; 1166 pg->flags &= ~(PG_BUSY|PG_FAKE); 1167 UVM_PAGE_OWN(pg, NULL); 1168 1169 /* done! */ 1170 } /* ref == 1 */ 1171 } /* write fault */ 1172 } /* loan count */ 1173 1174 /* 1175 * if we are case 1B then we will need to allocate a new blank 1176 * anon to transfer the data into. note that we have a lock 1177 * on anon, so no one can busy or release the page until we are done. 1178 * also note that the ref count can't drop to zero here because 1179 * it is > 1 and we are only dropping one ref. 1180 * 1181 * in the (hopefully very rare) case that we are out of RAM we 1182 * will unlock, wait for more RAM, and refault. 1183 * 1184 * if we are out of anon VM we kill the process (XXX: could wait?). 1185 */ 1186 1187 if ((access_type & VM_PROT_WRITE) != 0 && anon->an_ref > 1) { 1188 1189 UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0); 1190 uvmexp.flt_acow++; 1191 oanon = anon; /* oanon = old, locked anon */ 1192 anon = uvm_analloc(); 1193 if (anon) 1194 pg = uvm_pagealloc(NULL, 0, anon, 0); 1195 #ifdef __GNUC__ 1196 else 1197 pg = NULL; /* XXX: gcc */ 1198 #endif 1199 1200 /* check for out of RAM */ 1201 if (anon == NULL || pg == NULL) { 1202 if (anon) 1203 uvm_anfree(anon); 1204 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1205 #ifdef DIAGNOSTIC 1206 if (uvmexp.swpgonly > uvmexp.swpages) { 1207 panic("uvmexp.swpgonly botch"); 1208 } 1209 #endif 1210 if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) { 1211 UVMHIST_LOG(maphist, 1212 "<- failed. out of VM",0,0,0,0); 1213 uvmexp.fltnoanon++; 1214 return (KERN_RESOURCE_SHORTAGE); 1215 } 1216 1217 uvmexp.fltnoram++; 1218 uvm_wait("flt_noram3"); /* out of RAM, wait for more */ 1219 goto ReFault; 1220 } 1221 1222 /* got all resources, replace anon with nanon */ 1223 1224 uvm_pagecopy(oanon->u.an_page, pg); /* pg now !PG_CLEAN */ 1225 pg->flags &= ~(PG_BUSY|PG_FAKE); /* un-busy! new page */ 1226 UVM_PAGE_OWN(pg, NULL); 1227 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start, 1228 anon, 1); 1229 1230 /* deref: can not drop to zero here by defn! */ 1231 oanon->an_ref--; 1232 1233 /* 1234 * note: oanon still locked. anon is _not_ locked, but we 1235 * have the sole references to in from amap which _is_ locked. 1236 * thus, no one can get at it until we are done with it. 1237 */ 1238 1239 } else { 1240 1241 uvmexp.flt_anon++; 1242 oanon = anon; /* old, locked anon is same as anon */ 1243 pg = anon->u.an_page; 1244 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */ 1245 enter_prot = enter_prot & ~VM_PROT_WRITE; 1246 1247 } 1248 1249 /* locked: maps(read), amap, anon */ 1250 1251 /* 1252 * now map the page in ... 1253 * XXX: old fault unlocks object before pmap_enter. this seems 1254 * suspect since some other thread could blast the page out from 1255 * under us between the unlock and the pmap_enter. 1256 */ 1257 1258 UVMHIST_LOG(maphist, " MAPPING: anon: pm=0x%x, va=0x%x, pg=0x%x", 1259 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, 0); 1260 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1261 enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) 1262 != KERN_SUCCESS) { 1263 /* 1264 * No need to undo what we did; we can simply think of 1265 * this as the pmap throwing away the mapping information. 1266 * 1267 * We do, however, have to go through the ReFault path, 1268 * as the map may change while we're asleep. 1269 */ 1270 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1271 #ifdef DIAGNOSTIC 1272 if (uvmexp.swpgonly > uvmexp.swpages) 1273 panic("uvmexp.swpgonly botch"); 1274 #endif 1275 if (uvmexp.swpgonly == uvmexp.swpages) { 1276 UVMHIST_LOG(maphist, 1277 "<- failed. out of VM",0,0,0,0); 1278 /* XXX instrumentation */ 1279 return (KERN_RESOURCE_SHORTAGE); 1280 } 1281 /* XXX instrumentation */ 1282 uvm_wait("flt_pmfail1"); 1283 goto ReFault; 1284 } 1285 1286 /* 1287 * ... update the page queues. 1288 */ 1289 1290 uvm_lock_pageq(); 1291 1292 if (fault_type == VM_FAULT_WIRE) { 1293 uvm_pagewire(pg); 1294 1295 /* 1296 * since the now-wired page cannot be paged out, 1297 * release its swap resources for others to use. 1298 * since an anon with no swap cannot be PG_CLEAN, 1299 * clear its clean flag now. 1300 */ 1301 1302 pg->flags &= ~(PG_CLEAN); 1303 uvm_anon_dropswap(anon); 1304 } else { 1305 /* activate it */ 1306 uvm_pageactivate(pg); 1307 } 1308 1309 uvm_unlock_pageq(); 1310 1311 /* 1312 * done case 1! finish up by unlocking everything and returning success 1313 */ 1314 1315 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1316 return (KERN_SUCCESS); 1317 1318 1319 Case2: 1320 /* 1321 * handle case 2: faulting on backing object or zero fill 1322 */ 1323 1324 /* 1325 * locked: 1326 * maps(read), amap(if there), uobj(if !null), uobjpage(if !null) 1327 */ 1328 1329 /* 1330 * note that uobjpage can not be PGO_DONTCARE at this point. we now 1331 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we 1332 * have a backing object, check and see if we are going to promote 1333 * the data up to an anon during the fault. 1334 */ 1335 1336 if (uobj == NULL) { 1337 uobjpage = PGO_DONTCARE; 1338 promote = TRUE; /* always need anon here */ 1339 } else { 1340 /* assert(uobjpage != PGO_DONTCARE) */ 1341 promote = (access_type & VM_PROT_WRITE) && 1342 UVM_ET_ISCOPYONWRITE(ufi.entry); 1343 } 1344 UVMHIST_LOG(maphist, " case 2 fault: promote=%d, zfill=%d", 1345 promote, (uobj == NULL), 0,0); 1346 1347 /* 1348 * if uobjpage is not null then we do not need to do I/O to get the 1349 * uobjpage. 1350 * 1351 * if uobjpage is null, then we need to unlock and ask the pager to 1352 * get the data for us. once we have the data, we need to reverify 1353 * the state the world. we are currently not holding any resources. 1354 */ 1355 1356 if (uobjpage) { 1357 /* update rusage counters */ 1358 curproc->p_addr->u_stats.p_ru.ru_minflt++; 1359 } else { 1360 /* update rusage counters */ 1361 curproc->p_addr->u_stats.p_ru.ru_majflt++; 1362 1363 /* locked: maps(read), amap(if there), uobj */ 1364 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1365 /* locked: uobj */ 1366 1367 uvmexp.fltget++; 1368 gotpages = 1; 1369 result = uobj->pgops->pgo_get(uobj, 1370 (ufi.orig_rvaddr - ufi.entry->start) + ufi.entry->offset, 1371 &uobjpage, &gotpages, 0, 1372 access_type & MASK(ufi.entry), 1373 ufi.entry->advice, 0); 1374 1375 /* locked: uobjpage(if result OK) */ 1376 1377 /* 1378 * recover from I/O 1379 */ 1380 1381 if (result != VM_PAGER_OK) { 1382 #ifdef DIAGNOSTIC 1383 if (result == VM_PAGER_PEND) 1384 panic("uvm_fault: pgo_get got PENDing " 1385 "on non-async I/O"); 1386 #endif 1387 1388 if (result == VM_PAGER_AGAIN) { 1389 UVMHIST_LOG(maphist, 1390 " pgo_get says TRY AGAIN!",0,0,0,0); 1391 tsleep((caddr_t)&lbolt, PVM, "fltagain2", 0); 1392 goto ReFault; 1393 } 1394 1395 UVMHIST_LOG(maphist, "<- pgo_get failed (code %d)", 1396 result, 0,0,0); 1397 return (KERN_PROTECTION_FAILURE); /* XXX i/o error */ 1398 } 1399 1400 /* locked: uobjpage */ 1401 1402 /* 1403 * re-verify the state of the world by first trying to relock 1404 * the maps. always relock the object. 1405 */ 1406 1407 locked = uvmfault_relock(&ufi); 1408 if (locked && amap) 1409 amap_lock(amap); 1410 simple_lock(&uobj->vmobjlock); 1411 1412 /* locked(locked): maps(read), amap(if !null), uobj, uobjpage */ 1413 /* locked(!locked): uobj, uobjpage */ 1414 1415 /* 1416 * verify that the page has not be released and re-verify 1417 * that amap slot is still free. if there is a problem, 1418 * we unlock and clean up. 1419 */ 1420 1421 if ((uobjpage->flags & PG_RELEASED) != 0 || 1422 (locked && amap && 1423 amap_lookup(&ufi.entry->aref, 1424 ufi.orig_rvaddr - ufi.entry->start))) { 1425 if (locked) 1426 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1427 locked = FALSE; 1428 } 1429 1430 /* 1431 * didn't get the lock? release the page and retry. 1432 */ 1433 1434 if (locked == FALSE) { 1435 1436 UVMHIST_LOG(maphist, 1437 " wasn't able to relock after fault: retry", 1438 0,0,0,0); 1439 if (uobjpage->flags & PG_WANTED) 1440 /* still holding object lock */ 1441 wakeup(uobjpage); 1442 1443 if (uobjpage->flags & PG_RELEASED) { 1444 uvmexp.fltpgrele++; 1445 #ifdef DIAGNOSTIC 1446 if (uobj->pgops->pgo_releasepg == NULL) 1447 panic("uvm_fault: object has no " 1448 "releasepg function"); 1449 #endif 1450 /* frees page */ 1451 if (uobj->pgops->pgo_releasepg(uobjpage,NULL)) 1452 /* unlock if still alive */ 1453 simple_unlock(&uobj->vmobjlock); 1454 goto ReFault; 1455 } 1456 1457 uvm_lock_pageq(); 1458 /* make sure it is in queues */ 1459 uvm_pageactivate(uobjpage); 1460 1461 uvm_unlock_pageq(); 1462 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1463 UVM_PAGE_OWN(uobjpage, NULL); 1464 simple_unlock(&uobj->vmobjlock); 1465 goto ReFault; 1466 1467 } 1468 1469 /* 1470 * we have the data in uobjpage which is PG_BUSY and 1471 * !PG_RELEASED. we are holding object lock (so the page 1472 * can't be released on us). 1473 */ 1474 1475 /* locked: maps(read), amap(if !null), uobj, uobjpage */ 1476 1477 } 1478 1479 /* 1480 * locked: 1481 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj) 1482 */ 1483 1484 /* 1485 * notes: 1486 * - at this point uobjpage can not be NULL 1487 * - at this point uobjpage can not be PG_RELEASED (since we checked 1488 * for it above) 1489 * - at this point uobjpage could be PG_WANTED (handle later) 1490 */ 1491 1492 if (promote == FALSE) { 1493 1494 /* 1495 * we are not promoting. if the mapping is COW ensure that we 1496 * don't give more access than we should (e.g. when doing a read 1497 * fault on a COPYONWRITE mapping we want to map the COW page in 1498 * R/O even though the entry protection could be R/W). 1499 * 1500 * set "pg" to the page we want to map in (uobjpage, usually) 1501 */ 1502 1503 uvmexp.flt_obj++; 1504 if (UVM_ET_ISCOPYONWRITE(ufi.entry)) 1505 enter_prot &= ~VM_PROT_WRITE; 1506 pg = uobjpage; /* map in the actual object */ 1507 1508 /* assert(uobjpage != PGO_DONTCARE) */ 1509 1510 /* 1511 * we are faulting directly on the page. be careful 1512 * about writing to loaned pages... 1513 */ 1514 if (uobjpage->loan_count) { 1515 1516 if ((access_type & VM_PROT_WRITE) == 0) { 1517 /* read fault: cap the protection at readonly */ 1518 /* cap! */ 1519 enter_prot = enter_prot & ~VM_PROT_WRITE; 1520 } else { 1521 /* write fault: must break the loan here */ 1522 1523 /* alloc new un-owned page */ 1524 pg = uvm_pagealloc(NULL, 0, NULL, 0); 1525 1526 if (pg == NULL) { 1527 /* 1528 * drop ownership of page, it can't 1529 * be released 1530 */ 1531 if (uobjpage->flags & PG_WANTED) 1532 wakeup(uobjpage); 1533 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1534 UVM_PAGE_OWN(uobjpage, NULL); 1535 1536 uvm_lock_pageq(); 1537 /* activate: we will need it later */ 1538 uvm_pageactivate(uobjpage); 1539 1540 uvm_unlock_pageq(); 1541 uvmfault_unlockall(&ufi, amap, uobj, 1542 NULL); 1543 UVMHIST_LOG(maphist, 1544 " out of RAM breaking loan, waiting", 1545 0,0,0,0); 1546 uvmexp.fltnoram++; 1547 uvm_wait("flt_noram4"); 1548 goto ReFault; 1549 } 1550 1551 /* 1552 * copy the data from the old page to the new 1553 * one and clear the fake/clean flags on the 1554 * new page (keep it busy). force a reload 1555 * of the old page by clearing it from all 1556 * pmaps. then lock the page queues to 1557 * rename the pages. 1558 */ 1559 uvm_pagecopy(uobjpage, pg); /* old -> new */ 1560 pg->flags &= ~(PG_FAKE|PG_CLEAN); 1561 pmap_page_protect(uobjpage, VM_PROT_NONE); 1562 if (uobjpage->flags & PG_WANTED) 1563 wakeup(uobjpage); 1564 /* uobj still locked */ 1565 uobjpage->flags &= ~(PG_WANTED|PG_BUSY); 1566 UVM_PAGE_OWN(uobjpage, NULL); 1567 1568 uvm_lock_pageq(); 1569 offset = uobjpage->offset; 1570 /* remove old page */ 1571 uvm_pagerealloc(uobjpage, NULL, 0); 1572 1573 /* 1574 * at this point we have absolutely no 1575 * control over uobjpage 1576 */ 1577 /* install new page */ 1578 uvm_pagerealloc(pg, uobj, offset); 1579 uvm_unlock_pageq(); 1580 1581 /* 1582 * done! loan is broken and "pg" is 1583 * PG_BUSY. it can now replace uobjpage. 1584 */ 1585 1586 uobjpage = pg; 1587 1588 } /* write fault case */ 1589 } /* if loan_count */ 1590 1591 } else { 1592 1593 /* 1594 * if we are going to promote the data to an anon we 1595 * allocate a blank anon here and plug it into our amap. 1596 */ 1597 #if DIAGNOSTIC 1598 if (amap == NULL) 1599 panic("uvm_fault: want to promote data, but no anon"); 1600 #endif 1601 1602 anon = uvm_analloc(); 1603 if (anon) { 1604 /* 1605 * In `Fill in data...' below, if 1606 * uobjpage == PGO_DONTCARE, we want 1607 * a zero'd, dirty page, so have 1608 * uvm_pagealloc() do that for us. 1609 */ 1610 pg = uvm_pagealloc(NULL, 0, anon, 1611 (uobjpage == PGO_DONTCARE) ? UVM_PGA_ZERO : 0); 1612 } 1613 #ifdef __GNUC__ 1614 else 1615 pg = NULL; /* XXX: gcc */ 1616 #endif 1617 1618 /* 1619 * out of memory resources? 1620 */ 1621 if (anon == NULL || pg == NULL) { 1622 1623 /* 1624 * arg! must unbusy our page and fail or sleep. 1625 */ 1626 if (uobjpage != PGO_DONTCARE) { 1627 if (uobjpage->flags & PG_WANTED) 1628 /* still holding object lock */ 1629 wakeup(uobjpage); 1630 1631 uvm_lock_pageq(); 1632 /* make sure it is in queues */ 1633 uvm_pageactivate(uobjpage); 1634 uvm_unlock_pageq(); 1635 /* un-busy! (still locked) */ 1636 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1637 UVM_PAGE_OWN(uobjpage, NULL); 1638 } 1639 1640 /* unlock and fail ... */ 1641 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1642 #ifdef DIAGNOSTIC 1643 if (uvmexp.swpgonly > uvmexp.swpages) { 1644 panic("uvmexp.swpgonly botch"); 1645 } 1646 #endif 1647 if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) { 1648 UVMHIST_LOG(maphist, " promote: out of VM", 1649 0,0,0,0); 1650 uvmexp.fltnoanon++; 1651 return (KERN_RESOURCE_SHORTAGE); 1652 } 1653 1654 UVMHIST_LOG(maphist, " out of RAM, waiting for more", 1655 0,0,0,0); 1656 uvm_anfree(anon); 1657 uvmexp.fltnoram++; 1658 uvm_wait("flt_noram5"); 1659 goto ReFault; 1660 } 1661 1662 /* 1663 * fill in the data 1664 */ 1665 1666 if (uobjpage != PGO_DONTCARE) { 1667 uvmexp.flt_prcopy++; 1668 /* copy page [pg now dirty] */ 1669 uvm_pagecopy(uobjpage, pg); 1670 1671 /* 1672 * promote to shared amap? make sure all sharing 1673 * procs see it 1674 */ 1675 if ((amap_flags(amap) & AMAP_SHARED) != 0) { 1676 pmap_page_protect(uobjpage, VM_PROT_NONE); 1677 } 1678 1679 /* 1680 * dispose of uobjpage. it can't be PG_RELEASED 1681 * since we still hold the object lock. drop 1682 * handle to uobj as well. 1683 */ 1684 1685 if (uobjpage->flags & PG_WANTED) 1686 /* still have the obj lock */ 1687 wakeup(uobjpage); 1688 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1689 UVM_PAGE_OWN(uobjpage, NULL); 1690 uvm_lock_pageq(); 1691 uvm_pageactivate(uobjpage); /* put it back */ 1692 uvm_unlock_pageq(); 1693 simple_unlock(&uobj->vmobjlock); 1694 uobj = NULL; 1695 UVMHIST_LOG(maphist, 1696 " promote uobjpage 0x%x to anon/page 0x%x/0x%x", 1697 uobjpage, anon, pg, 0); 1698 1699 } else { 1700 uvmexp.flt_przero++; 1701 /* 1702 * Page is zero'd and marked dirty by uvm_pagealloc() 1703 * above. 1704 */ 1705 UVMHIST_LOG(maphist," zero fill anon/page 0x%x/0%x", 1706 anon, pg, 0, 0); 1707 } 1708 1709 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start, 1710 anon, 0); 1711 1712 } 1713 1714 /* 1715 * locked: 1716 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj) 1717 * 1718 * note: pg is either the uobjpage or the new page in the new anon 1719 */ 1720 1721 /* 1722 * all resources are present. we can now map it in and free our 1723 * resources. 1724 */ 1725 1726 UVMHIST_LOG(maphist, 1727 " MAPPING: case2: pm=0x%x, va=0x%x, pg=0x%x, promote=%d", 1728 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, promote); 1729 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1730 enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) 1731 != KERN_SUCCESS) { 1732 /* 1733 * No need to undo what we did; we can simply think of 1734 * this as the pmap throwing away the mapping information. 1735 * 1736 * We do, however, have to go through the ReFault path, 1737 * as the map may change while we're asleep. 1738 */ 1739 if (pg->flags & PG_WANTED) 1740 wakeup(pg); /* lock still held */ 1741 1742 /* 1743 * note that pg can't be PG_RELEASED since we did not drop 1744 * the object lock since the last time we checked. 1745 */ 1746 1747 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); 1748 UVM_PAGE_OWN(pg, NULL); 1749 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1750 #ifdef DIAGNOSTIC 1751 if (uvmexp.swpgonly > uvmexp.swpages) 1752 panic("uvmexp.swpgonly botch"); 1753 #endif 1754 if (uvmexp.swpgonly == uvmexp.swpages) { 1755 UVMHIST_LOG(maphist, 1756 "<- failed. out of VM",0,0,0,0); 1757 /* XXX instrumentation */ 1758 return (KERN_RESOURCE_SHORTAGE); 1759 } 1760 /* XXX instrumentation */ 1761 uvm_wait("flt_pmfail2"); 1762 goto ReFault; 1763 } 1764 1765 uvm_lock_pageq(); 1766 1767 if (fault_type == VM_FAULT_WIRE) { 1768 uvm_pagewire(pg); 1769 if (pg->pqflags & PQ_AOBJ) { 1770 1771 /* 1772 * since the now-wired page cannot be paged out, 1773 * release its swap resources for others to use. 1774 * since an aobj page with no swap cannot be PG_CLEAN, 1775 * clear its clean flag now. 1776 */ 1777 1778 pg->flags &= ~(PG_CLEAN); 1779 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT); 1780 } 1781 } else { 1782 /* activate it */ 1783 uvm_pageactivate(pg); 1784 } 1785 1786 uvm_unlock_pageq(); 1787 1788 if (pg->flags & PG_WANTED) 1789 wakeup(pg); /* lock still held */ 1790 1791 /* 1792 * note that pg can't be PG_RELEASED since we did not drop the object 1793 * lock since the last time we checked. 1794 */ 1795 1796 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); 1797 UVM_PAGE_OWN(pg, NULL); 1798 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1799 1800 UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0); 1801 return (KERN_SUCCESS); 1802 } 1803 1804 1805 /* 1806 * uvm_fault_wire: wire down a range of virtual addresses in a map. 1807 * 1808 * => map may be read-locked by caller, but MUST NOT be write-locked. 1809 * => if map is read-locked, any operations which may cause map to 1810 * be write-locked in uvm_fault() must be taken care of by 1811 * the caller. See uvm_map_pageable(). 1812 */ 1813 1814 int 1815 uvm_fault_wire(map, start, end, access_type) 1816 vm_map_t map; 1817 vaddr_t start, end; 1818 vm_prot_t access_type; 1819 { 1820 vaddr_t va; 1821 pmap_t pmap; 1822 int rv; 1823 1824 pmap = vm_map_pmap(map); 1825 1826 /* 1827 * now fault it in a page at a time. if the fault fails then we have 1828 * to undo what we have done. note that in uvm_fault VM_PROT_NONE 1829 * is replaced with the max protection if fault_type is VM_FAULT_WIRE. 1830 */ 1831 1832 for (va = start ; va < end ; va += PAGE_SIZE) { 1833 rv = uvm_fault(map, va, VM_FAULT_WIRE, access_type); 1834 if (rv) { 1835 if (va != start) { 1836 uvm_fault_unwire(map, start, va); 1837 } 1838 return (rv); 1839 } 1840 } 1841 1842 return (KERN_SUCCESS); 1843 } 1844 1845 /* 1846 * uvm_fault_unwire(): unwire range of virtual space. 1847 */ 1848 1849 void 1850 uvm_fault_unwire(map, start, end) 1851 vm_map_t map; 1852 vaddr_t start, end; 1853 { 1854 1855 vm_map_lock_read(map); 1856 uvm_fault_unwire_locked(map, start, end); 1857 vm_map_unlock_read(map); 1858 } 1859 1860 /* 1861 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire(). 1862 * 1863 * => map must be at least read-locked. 1864 */ 1865 1866 void 1867 uvm_fault_unwire_locked(map, start, end) 1868 vm_map_t map; 1869 vaddr_t start, end; 1870 { 1871 vm_map_entry_t entry; 1872 pmap_t pmap = vm_map_pmap(map); 1873 vaddr_t va; 1874 paddr_t pa; 1875 struct vm_page *pg; 1876 1877 #ifdef DIAGNOSTIC 1878 if (map->flags & VM_MAP_INTRSAFE) 1879 panic("uvm_fault_unwire_locked: intrsafe map"); 1880 #endif 1881 1882 /* 1883 * we assume that the area we are unwiring has actually been wired 1884 * in the first place. this means that we should be able to extract 1885 * the PAs from the pmap. we also lock out the page daemon so that 1886 * we can call uvm_pageunwire. 1887 */ 1888 1889 uvm_lock_pageq(); 1890 1891 /* 1892 * find the beginning map entry for the region. 1893 */ 1894 #ifdef DIAGNOSTIC 1895 if (start < vm_map_min(map) || end > vm_map_max(map)) 1896 panic("uvm_fault_unwire_locked: address out of range"); 1897 #endif 1898 if (uvm_map_lookup_entry(map, start, &entry) == FALSE) 1899 panic("uvm_fault_unwire_locked: address not in map"); 1900 1901 for (va = start; va < end ; va += PAGE_SIZE) { 1902 if (pmap_extract(pmap, va, &pa) == FALSE) 1903 panic("uvm_fault_unwire_locked: unwiring " 1904 "non-wired memory"); 1905 1906 /* 1907 * make sure the current entry is for the address we're 1908 * dealing with. if not, grab the next entry. 1909 */ 1910 #ifdef DIAGNOSTIC 1911 if (va < entry->start) 1912 panic("uvm_fault_unwire_locked: hole 1"); 1913 #endif 1914 if (va >= entry->end) { 1915 #ifdef DIAGNOSTIC 1916 if (entry->next == &map->header || 1917 entry->next->start > entry->end) 1918 panic("uvm_fault_unwire_locked: hole 2"); 1919 #endif 1920 entry = entry->next; 1921 } 1922 1923 /* 1924 * if the entry is no longer wired, tell the pmap. 1925 */ 1926 if (VM_MAPENT_ISWIRED(entry) == 0) 1927 pmap_unwire(pmap, va); 1928 1929 pg = PHYS_TO_VM_PAGE(pa); 1930 if (pg) 1931 uvm_pageunwire(pg); 1932 } 1933 1934 uvm_unlock_pageq(); 1935 } 1936