1 /* $NetBSD: uvm_fault.c,v 1.50 2000/06/27 17:29:21 mrg Exp $ */ 2 3 /* 4 * 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by Charles D. Cranor and 19 * Washington University. 20 * 4. The name of the author may not be used to endorse or promote products 21 * derived from this software without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 24 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 25 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 26 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 27 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 28 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 32 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 * 34 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp 35 */ 36 37 #include "opt_uvmhist.h" 38 39 /* 40 * uvm_fault.c: fault handler 41 */ 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/kernel.h> 46 #include <sys/proc.h> 47 #include <sys/malloc.h> 48 #include <sys/mman.h> 49 #include <sys/user.h> 50 51 #include <uvm/uvm.h> 52 53 /* 54 * 55 * a word on page faults: 56 * 57 * types of page faults we handle: 58 * 59 * CASE 1: upper layer faults CASE 2: lower layer faults 60 * 61 * CASE 1A CASE 1B CASE 2A CASE 2B 62 * read/write1 write>1 read/write +-cow_write/zero 63 * | | | | 64 * +--|--+ +--|--+ +-----+ + | + | +-----+ 65 * amap | V | | ----------->new| | | | ^ | 66 * +-----+ +-----+ +-----+ + | + | +--|--+ 67 * | | | 68 * +-----+ +-----+ +--|--+ | +--|--+ 69 * uobj | d/c | | d/c | | V | +----| | 70 * +-----+ +-----+ +-----+ +-----+ 71 * 72 * d/c = don't care 73 * 74 * case [0]: layerless fault 75 * no amap or uobj is present. this is an error. 76 * 77 * case [1]: upper layer fault [anon active] 78 * 1A: [read] or [write with anon->an_ref == 1] 79 * I/O takes place in top level anon and uobj is not touched. 80 * 1B: [write with anon->an_ref > 1] 81 * new anon is alloc'd and data is copied off ["COW"] 82 * 83 * case [2]: lower layer fault [uobj] 84 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area] 85 * I/O takes place directly in object. 86 * 2B: [write to copy_on_write] or [read on NULL uobj] 87 * data is "promoted" from uobj to a new anon. 88 * if uobj is null, then we zero fill. 89 * 90 * we follow the standard UVM locking protocol ordering: 91 * 92 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ) 93 * we hold a PG_BUSY page if we unlock for I/O 94 * 95 * 96 * the code is structured as follows: 97 * 98 * - init the "IN" params in the ufi structure 99 * ReFault: 100 * - do lookups [locks maps], check protection, handle needs_copy 101 * - check for case 0 fault (error) 102 * - establish "range" of fault 103 * - if we have an amap lock it and extract the anons 104 * - if sequential advice deactivate pages behind us 105 * - at the same time check pmap for unmapped areas and anon for pages 106 * that we could map in (and do map it if found) 107 * - check object for resident pages that we could map in 108 * - if (case 2) goto Case2 109 * - >>> handle case 1 110 * - ensure source anon is resident in RAM 111 * - if case 1B alloc new anon and copy from source 112 * - map the correct page in 113 * Case2: 114 * - >>> handle case 2 115 * - ensure source page is resident (if uobj) 116 * - if case 2B alloc new anon and copy from source (could be zero 117 * fill if uobj == NULL) 118 * - map the correct page in 119 * - done! 120 * 121 * note on paging: 122 * if we have to do I/O we place a PG_BUSY page in the correct object, 123 * unlock everything, and do the I/O. when I/O is done we must reverify 124 * the state of the world before assuming that our data structures are 125 * valid. [because mappings could change while the map is unlocked] 126 * 127 * alternative 1: unbusy the page in question and restart the page fault 128 * from the top (ReFault). this is easy but does not take advantage 129 * of the information that we already have from our previous lookup, 130 * although it is possible that the "hints" in the vm_map will help here. 131 * 132 * alternative 2: the system already keeps track of a "version" number of 133 * a map. [i.e. every time you write-lock a map (e.g. to change a 134 * mapping) you bump the version number up by one...] so, we can save 135 * the version number of the map before we release the lock and start I/O. 136 * then when I/O is done we can relock and check the version numbers 137 * to see if anything changed. this might save us some over 1 because 138 * we don't have to unbusy the page and may be less compares(?). 139 * 140 * alternative 3: put in backpointers or a way to "hold" part of a map 141 * in place while I/O is in progress. this could be complex to 142 * implement (especially with structures like amap that can be referenced 143 * by multiple map entries, and figuring out what should wait could be 144 * complex as well...). 145 * 146 * given that we are not currently multiprocessor or multithreaded we might 147 * as well choose alternative 2 now. maybe alternative 3 would be useful 148 * in the future. XXX keep in mind for future consideration//rechecking. 149 */ 150 151 /* 152 * local data structures 153 */ 154 155 struct uvm_advice { 156 int advice; 157 int nback; 158 int nforw; 159 }; 160 161 /* 162 * page range array: 163 * note: index in array must match "advice" value 164 * XXX: borrowed numbers from freebsd. do they work well for us? 165 */ 166 167 static struct uvm_advice uvmadvice[] = { 168 { MADV_NORMAL, 3, 4 }, 169 { MADV_RANDOM, 0, 0 }, 170 { MADV_SEQUENTIAL, 8, 7}, 171 }; 172 173 #define UVM_MAXRANGE 16 /* must be max() of nback+nforw+1 */ 174 175 /* 176 * private prototypes 177 */ 178 179 static void uvmfault_amapcopy __P((struct uvm_faultinfo *)); 180 static __inline void uvmfault_anonflush __P((struct vm_anon **, int)); 181 182 /* 183 * inline functions 184 */ 185 186 /* 187 * uvmfault_anonflush: try and deactivate pages in specified anons 188 * 189 * => does not have to deactivate page if it is busy 190 */ 191 192 static __inline void 193 uvmfault_anonflush(anons, n) 194 struct vm_anon **anons; 195 int n; 196 { 197 int lcv; 198 struct vm_page *pg; 199 200 for (lcv = 0 ; lcv < n ; lcv++) { 201 if (anons[lcv] == NULL) 202 continue; 203 simple_lock(&anons[lcv]->an_lock); 204 pg = anons[lcv]->u.an_page; 205 if (pg && (pg->flags & PG_BUSY) == 0 && pg->loan_count == 0) { 206 uvm_lock_pageq(); 207 if (pg->wire_count == 0) { 208 pmap_page_protect(pg, VM_PROT_NONE); 209 uvm_pagedeactivate(pg); 210 } 211 uvm_unlock_pageq(); 212 } 213 simple_unlock(&anons[lcv]->an_lock); 214 } 215 } 216 217 /* 218 * normal functions 219 */ 220 221 /* 222 * uvmfault_amapcopy: clear "needs_copy" in a map. 223 * 224 * => called with VM data structures unlocked (usually, see below) 225 * => we get a write lock on the maps and clear needs_copy for a VA 226 * => if we are out of RAM we sleep (waiting for more) 227 */ 228 229 static void 230 uvmfault_amapcopy(ufi) 231 struct uvm_faultinfo *ufi; 232 { 233 234 /* 235 * while we haven't done the job 236 */ 237 238 while (1) { 239 240 /* 241 * no mapping? give up. 242 */ 243 244 if (uvmfault_lookup(ufi, TRUE) == FALSE) 245 return; 246 247 /* 248 * copy if needed. 249 */ 250 251 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) 252 amap_copy(ufi->map, ufi->entry, M_NOWAIT, TRUE, 253 ufi->orig_rvaddr, ufi->orig_rvaddr + 1); 254 255 /* 256 * didn't work? must be out of RAM. unlock and sleep. 257 */ 258 259 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 260 uvmfault_unlockmaps(ufi, TRUE); 261 uvm_wait("fltamapcopy"); 262 continue; 263 } 264 265 /* 266 * got it! unlock and return. 267 */ 268 269 uvmfault_unlockmaps(ufi, TRUE); 270 return; 271 } 272 /*NOTREACHED*/ 273 } 274 275 /* 276 * uvmfault_anonget: get data in an anon into a non-busy, non-released 277 * page in that anon. 278 * 279 * => maps, amap, and anon locked by caller. 280 * => if we fail (result != VM_PAGER_OK) we unlock everything except anon. 281 * => if we are successful, we return with everything still locked. 282 * => we don't move the page on the queues [gets moved later] 283 * => if we allocate a new page [we_own], it gets put on the queues. 284 * either way, the result is that the page is on the queues at return time 285 * => for pages which are on loan from a uvm_object (and thus are not 286 * owned by the anon): if successful, we return with the owning object 287 * locked. the caller must unlock this object when it unlocks everything 288 * else. 289 */ 290 291 int 292 uvmfault_anonget(ufi, amap, anon) 293 struct uvm_faultinfo *ufi; 294 struct vm_amap *amap; 295 struct vm_anon *anon; 296 { 297 boolean_t we_own; /* we own anon's page? */ 298 boolean_t locked; /* did we relock? */ 299 struct vm_page *pg; 300 int result; 301 UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist); 302 303 result = 0; /* XXX shut up gcc */ 304 uvmexp.fltanget++; 305 /* bump rusage counters */ 306 if (anon->u.an_page) 307 curproc->p_addr->u_stats.p_ru.ru_minflt++; 308 else 309 curproc->p_addr->u_stats.p_ru.ru_majflt++; 310 311 /* 312 * loop until we get it, or fail. 313 */ 314 315 while (1) { 316 317 we_own = FALSE; /* TRUE if we set PG_BUSY on a page */ 318 pg = anon->u.an_page; 319 320 /* 321 * if there is a resident page and it is loaned, then anon 322 * may not own it. call out to uvm_anon_lockpage() to ensure 323 * the real owner of the page has been identified and locked. 324 */ 325 326 if (pg && pg->loan_count) 327 pg = uvm_anon_lockloanpg(anon); 328 329 /* 330 * page there? make sure it is not busy/released. 331 */ 332 333 if (pg) { 334 335 /* 336 * at this point, if the page has a uobject [meaning 337 * we have it on loan], then that uobject is locked 338 * by us! if the page is busy, we drop all the 339 * locks (including uobject) and try again. 340 */ 341 342 if ((pg->flags & (PG_BUSY|PG_RELEASED)) == 0) { 343 UVMHIST_LOG(maphist, "<- OK",0,0,0,0); 344 return (VM_PAGER_OK); 345 } 346 pg->flags |= PG_WANTED; 347 uvmexp.fltpgwait++; 348 349 /* 350 * the last unlock must be an atomic unlock+wait on 351 * the owner of page 352 */ 353 if (pg->uobject) { /* owner is uobject ? */ 354 uvmfault_unlockall(ufi, amap, NULL, anon); 355 UVMHIST_LOG(maphist, " unlock+wait on uobj",0, 356 0,0,0); 357 UVM_UNLOCK_AND_WAIT(pg, 358 &pg->uobject->vmobjlock, 359 FALSE, "anonget1",0); 360 } else { 361 /* anon owns page */ 362 uvmfault_unlockall(ufi, amap, NULL, NULL); 363 UVMHIST_LOG(maphist, " unlock+wait on anon",0, 364 0,0,0); 365 UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0, 366 "anonget2",0); 367 } 368 /* ready to relock and try again */ 369 370 } else { 371 372 /* 373 * no page, we must try and bring it in. 374 */ 375 pg = uvm_pagealloc(NULL, 0, anon, 0); 376 377 if (pg == NULL) { /* out of RAM. */ 378 379 uvmfault_unlockall(ufi, amap, NULL, anon); 380 uvmexp.fltnoram++; 381 UVMHIST_LOG(maphist, " noram -- UVM_WAIT",0, 382 0,0,0); 383 uvm_wait("flt_noram1"); 384 /* ready to relock and try again */ 385 386 } else { 387 388 /* we set the PG_BUSY bit */ 389 we_own = TRUE; 390 uvmfault_unlockall(ufi, amap, NULL, anon); 391 392 /* 393 * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN 394 * page into the uvm_swap_get function with 395 * all data structures unlocked. note that 396 * it is ok to read an_swslot here because 397 * we hold PG_BUSY on the page. 398 */ 399 uvmexp.pageins++; 400 result = uvm_swap_get(pg, anon->an_swslot, 401 PGO_SYNCIO); 402 403 /* 404 * we clean up after the i/o below in the 405 * "we_own" case 406 */ 407 /* ready to relock and try again */ 408 } 409 } 410 411 /* 412 * now relock and try again 413 */ 414 415 locked = uvmfault_relock(ufi); 416 if (locked && amap != NULL) { 417 amap_lock(amap); 418 } 419 if (locked || we_own) 420 simple_lock(&anon->an_lock); 421 422 /* 423 * if we own the page (i.e. we set PG_BUSY), then we need 424 * to clean up after the I/O. there are three cases to 425 * consider: 426 * [1] page released during I/O: free anon and ReFault. 427 * [2] I/O not OK. free the page and cause the fault 428 * to fail. 429 * [3] I/O OK! activate the page and sync with the 430 * non-we_own case (i.e. drop anon lock if not locked). 431 */ 432 433 if (we_own) { 434 435 if (pg->flags & PG_WANTED) { 436 /* still holding object lock */ 437 wakeup(pg); 438 } 439 /* un-busy! */ 440 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE); 441 UVM_PAGE_OWN(pg, NULL); 442 443 /* 444 * if we were RELEASED during I/O, then our anon is 445 * no longer part of an amap. we need to free the 446 * anon and try again. 447 */ 448 if (pg->flags & PG_RELEASED) { 449 pmap_page_protect(pg, VM_PROT_NONE); 450 simple_unlock(&anon->an_lock); 451 uvm_anfree(anon); /* frees page for us */ 452 if (locked) 453 uvmfault_unlockall(ufi, amap, NULL, 454 NULL); 455 uvmexp.fltpgrele++; 456 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 457 return (VM_PAGER_REFAULT); /* refault! */ 458 } 459 460 if (result != VM_PAGER_OK) { 461 #ifdef DIAGNOSTIC 462 if (result == VM_PAGER_PEND) { 463 panic("uvmfault_anonget: " 464 "got PENDING for non-async I/O"); 465 } 466 #endif 467 /* remove page from anon */ 468 anon->u.an_page = NULL; 469 470 /* 471 * remove the swap slot from the anon 472 * and mark the anon as having no real slot. 473 * don't free the swap slot, thus preventing 474 * it from being used again. 475 */ 476 uvm_swap_markbad(anon->an_swslot, 1); 477 anon->an_swslot = SWSLOT_BAD; 478 479 /* 480 * note: page was never !PG_BUSY, so it 481 * can't be mapped and thus no need to 482 * pmap_page_protect it... 483 */ 484 uvm_lock_pageq(); 485 uvm_pagefree(pg); 486 uvm_unlock_pageq(); 487 488 if (locked) 489 uvmfault_unlockall(ufi, amap, NULL, 490 anon); 491 else 492 simple_unlock(&anon->an_lock); 493 UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0); 494 return (VM_PAGER_ERROR); 495 } 496 497 /* 498 * must be OK, clear modify (already PG_CLEAN) 499 * and activate 500 */ 501 pmap_clear_modify(pg); 502 uvm_lock_pageq(); 503 uvm_pageactivate(pg); 504 uvm_unlock_pageq(); 505 if (!locked) 506 simple_unlock(&anon->an_lock); 507 } 508 509 /* 510 * we were not able to relock. restart fault. 511 */ 512 513 if (!locked) { 514 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 515 return (VM_PAGER_REFAULT); 516 } 517 518 /* 519 * verify no one has touched the amap and moved the anon on us. 520 */ 521 522 if (ufi != NULL && 523 amap_lookup(&ufi->entry->aref, 524 ufi->orig_rvaddr - ufi->entry->start) != anon) { 525 526 uvmfault_unlockall(ufi, amap, NULL, anon); 527 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 528 return (VM_PAGER_REFAULT); 529 } 530 531 /* 532 * try it again! 533 */ 534 535 uvmexp.fltanretry++; 536 continue; 537 538 } /* while (1) */ 539 540 /*NOTREACHED*/ 541 } 542 543 /* 544 * F A U L T - m a i n e n t r y p o i n t 545 */ 546 547 /* 548 * uvm_fault: page fault handler 549 * 550 * => called from MD code to resolve a page fault 551 * => VM data structures usually should be unlocked. however, it is 552 * possible to call here with the main map locked if the caller 553 * gets a write lock, sets it recusive, and then calls us (c.f. 554 * uvm_map_pageable). this should be avoided because it keeps 555 * the map locked off during I/O. 556 */ 557 558 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 559 ~VM_PROT_WRITE : VM_PROT_ALL) 560 561 int 562 uvm_fault(orig_map, vaddr, fault_type, access_type) 563 vm_map_t orig_map; 564 vaddr_t vaddr; 565 vm_fault_t fault_type; 566 vm_prot_t access_type; 567 { 568 struct uvm_faultinfo ufi; 569 vm_prot_t enter_prot; 570 boolean_t wired, narrow, promote, locked, shadowed; 571 int npages, nback, nforw, centeridx, result, lcv, gotpages; 572 vaddr_t startva, objaddr, currva, offset; 573 paddr_t pa; 574 struct vm_amap *amap; 575 struct uvm_object *uobj; 576 struct vm_anon *anons_store[UVM_MAXRANGE], **anons, *anon, *oanon; 577 struct vm_page *pages[UVM_MAXRANGE], *pg, *uobjpage; 578 UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist); 579 580 UVMHIST_LOG(maphist, "(map=0x%x, vaddr=0x%x, ft=%d, at=%d)", 581 orig_map, vaddr, fault_type, access_type); 582 583 anon = NULL; /* XXX: shut up gcc */ 584 585 uvmexp.faults++; /* XXX: locking? */ 586 587 /* 588 * init the IN parameters in the ufi 589 */ 590 591 ufi.orig_map = orig_map; 592 ufi.orig_rvaddr = trunc_page(vaddr); 593 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ 594 if (fault_type == VM_FAULT_WIRE) 595 narrow = TRUE; /* don't look for neighborhood 596 * pages on wire */ 597 else 598 narrow = FALSE; /* normal fault */ 599 600 /* 601 * before we do anything else, if this is a fault on a kernel 602 * address, check to see if the address is managed by an 603 * interrupt-safe map. If it is, we fail immediately. Intrsafe 604 * maps are never pageable, and this approach avoids an evil 605 * locking mess. 606 */ 607 if (orig_map == kernel_map && uvmfault_check_intrsafe(&ufi)) { 608 UVMHIST_LOG(maphist, "<- VA 0x%lx in intrsafe map %p", 609 ufi.orig_rvaddr, ufi.map, 0, 0); 610 return (KERN_FAILURE); 611 } 612 613 /* 614 * "goto ReFault" means restart the page fault from ground zero. 615 */ 616 ReFault: 617 618 /* 619 * lookup and lock the maps 620 */ 621 622 if (uvmfault_lookup(&ufi, FALSE) == FALSE) { 623 UVMHIST_LOG(maphist, "<- no mapping @ 0x%x", vaddr, 0,0,0); 624 return (KERN_INVALID_ADDRESS); 625 } 626 /* locked: maps(read) */ 627 628 /* 629 * check protection 630 */ 631 632 if ((ufi.entry->protection & access_type) != access_type) { 633 UVMHIST_LOG(maphist, 634 "<- protection failure (prot=0x%x, access=0x%x)", 635 ufi.entry->protection, access_type, 0, 0); 636 uvmfault_unlockmaps(&ufi, FALSE); 637 return (KERN_PROTECTION_FAILURE); 638 } 639 640 /* 641 * if the map is not a pageable map, a page fault always fails. 642 */ 643 644 if ((ufi.map->flags & VM_MAP_PAGEABLE) == 0) { 645 UVMHIST_LOG(maphist, 646 "<- map %p not pageable", ufi.map, 0, 0, 0); 647 uvmfault_unlockmaps(&ufi, FALSE); 648 return (KERN_FAILURE); 649 } 650 651 /* 652 * "enter_prot" is the protection we want to enter the page in at. 653 * for certain pages (e.g. copy-on-write pages) this protection can 654 * be more strict than ufi.entry->protection. "wired" means either 655 * the entry is wired or we are fault-wiring the pg. 656 */ 657 658 enter_prot = ufi.entry->protection; 659 wired = VM_MAPENT_ISWIRED(ufi.entry) || (fault_type == VM_FAULT_WIRE); 660 if (wired) 661 access_type = enter_prot; /* full access for wired */ 662 663 /* 664 * handle "needs_copy" case. if we need to copy the amap we will 665 * have to drop our readlock and relock it with a write lock. (we 666 * need a write lock to change anything in a map entry [e.g. 667 * needs_copy]). 668 */ 669 670 if (UVM_ET_ISNEEDSCOPY(ufi.entry)) { 671 if ((access_type & VM_PROT_WRITE) || 672 (ufi.entry->object.uvm_obj == NULL)) { 673 /* need to clear */ 674 UVMHIST_LOG(maphist, 675 " need to clear needs_copy and refault",0,0,0,0); 676 uvmfault_unlockmaps(&ufi, FALSE); 677 uvmfault_amapcopy(&ufi); 678 uvmexp.fltamcopy++; 679 goto ReFault; 680 681 } else { 682 683 /* 684 * ensure that we pmap_enter page R/O since 685 * needs_copy is still true 686 */ 687 enter_prot &= ~VM_PROT_WRITE; 688 689 } 690 } 691 692 /* 693 * identify the players 694 */ 695 696 amap = ufi.entry->aref.ar_amap; /* top layer */ 697 uobj = ufi.entry->object.uvm_obj; /* bottom layer */ 698 699 /* 700 * check for a case 0 fault. if nothing backing the entry then 701 * error now. 702 */ 703 704 if (amap == NULL && uobj == NULL) { 705 uvmfault_unlockmaps(&ufi, FALSE); 706 UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0); 707 return (KERN_INVALID_ADDRESS); 708 } 709 710 /* 711 * establish range of interest based on advice from mapper 712 * and then clip to fit map entry. note that we only want 713 * to do this the first time through the fault. if we 714 * ReFault we will disable this by setting "narrow" to true. 715 */ 716 717 if (narrow == FALSE) { 718 719 /* wide fault (!narrow) */ 720 #ifdef DIAGNOSTIC 721 if (uvmadvice[ufi.entry->advice].advice != ufi.entry->advice) 722 panic("fault: advice mismatch!"); 723 #endif 724 nback = min(uvmadvice[ufi.entry->advice].nback, 725 (ufi.orig_rvaddr - ufi.entry->start) >> PAGE_SHIFT); 726 startva = ufi.orig_rvaddr - (nback << PAGE_SHIFT); 727 nforw = min(uvmadvice[ufi.entry->advice].nforw, 728 ((ufi.entry->end - ufi.orig_rvaddr) >> 729 PAGE_SHIFT) - 1); 730 /* 731 * note: "-1" because we don't want to count the 732 * faulting page as forw 733 */ 734 npages = nback + nforw + 1; 735 centeridx = nback; 736 737 narrow = TRUE; /* ensure only once per-fault */ 738 739 } else { 740 741 /* narrow fault! */ 742 nback = nforw = 0; 743 startva = ufi.orig_rvaddr; 744 npages = 1; 745 centeridx = 0; 746 747 } 748 749 /* locked: maps(read) */ 750 UVMHIST_LOG(maphist, " narrow=%d, back=%d, forw=%d, startva=0x%x", 751 narrow, nback, nforw, startva); 752 UVMHIST_LOG(maphist, " entry=0x%x, amap=0x%x, obj=0x%x", ufi.entry, 753 amap, uobj, 0); 754 755 /* 756 * if we've got an amap, lock it and extract current anons. 757 */ 758 759 if (amap) { 760 amap_lock(amap); 761 anons = anons_store; 762 amap_lookups(&ufi.entry->aref, startva - ufi.entry->start, 763 anons, npages); 764 } else { 765 anons = NULL; /* to be safe */ 766 } 767 768 /* locked: maps(read), amap(if there) */ 769 770 /* 771 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages 772 * now and then forget about them (for the rest of the fault). 773 */ 774 775 if (ufi.entry->advice == MADV_SEQUENTIAL) { 776 777 UVMHIST_LOG(maphist, " MADV_SEQUENTIAL: flushing backpages", 778 0,0,0,0); 779 /* flush back-page anons? */ 780 if (amap) 781 uvmfault_anonflush(anons, nback); 782 783 /* flush object? */ 784 if (uobj) { 785 objaddr = 786 (startva - ufi.entry->start) + ufi.entry->offset; 787 simple_lock(&uobj->vmobjlock); 788 (void) uobj->pgops->pgo_flush(uobj, objaddr, objaddr + 789 (nback << PAGE_SHIFT), PGO_DEACTIVATE); 790 simple_unlock(&uobj->vmobjlock); 791 } 792 793 /* now forget about the backpages */ 794 if (amap) 795 anons += nback; 796 startva = startva + (nback << PAGE_SHIFT); 797 npages -= nback; 798 nback = centeridx = 0; 799 } 800 801 /* locked: maps(read), amap(if there) */ 802 803 /* 804 * map in the backpages and frontpages we found in the amap in hopes 805 * of preventing future faults. we also init the pages[] array as 806 * we go. 807 */ 808 809 currva = startva; 810 shadowed = FALSE; 811 for (lcv = 0 ; lcv < npages ; lcv++, currva += PAGE_SIZE) { 812 813 /* 814 * dont play with VAs that are already mapped 815 * except for center) 816 */ 817 if (lcv != centeridx) { 818 if (pmap_extract(ufi.orig_map->pmap, currva, &pa) == 819 TRUE) { 820 pages[lcv] = PGO_DONTCARE; 821 continue; 822 } 823 } 824 825 /* 826 * unmapped or center page. check if any anon at this level. 827 */ 828 if (amap == NULL || anons[lcv] == NULL) { 829 pages[lcv] = NULL; 830 continue; 831 } 832 833 /* 834 * check for present page and map if possible. re-activate it. 835 */ 836 837 pages[lcv] = PGO_DONTCARE; 838 if (lcv == centeridx) { /* save center for later! */ 839 shadowed = TRUE; 840 continue; 841 } 842 anon = anons[lcv]; 843 simple_lock(&anon->an_lock); 844 /* ignore loaned pages */ 845 if (anon->u.an_page && anon->u.an_page->loan_count == 0 && 846 (anon->u.an_page->flags & (PG_RELEASED|PG_BUSY)) == 0) { 847 uvm_lock_pageq(); 848 uvm_pageactivate(anon->u.an_page); /* reactivate */ 849 uvm_unlock_pageq(); 850 UVMHIST_LOG(maphist, 851 " MAPPING: n anon: pm=0x%x, va=0x%x, pg=0x%x", 852 ufi.orig_map->pmap, currva, anon->u.an_page, 0); 853 uvmexp.fltnamap++; 854 /* 855 * Since this isn't the page that's actually faulting, 856 * ignore pmap_enter() failures; it's not critical 857 * that we enter these right now. 858 */ 859 (void) pmap_enter(ufi.orig_map->pmap, currva, 860 VM_PAGE_TO_PHYS(anon->u.an_page), 861 (anon->an_ref > 1) ? (enter_prot & ~VM_PROT_WRITE) : 862 enter_prot, 863 PMAP_CANFAIL | 864 (VM_MAPENT_ISWIRED(ufi.entry) ? PMAP_WIRED : 0)); 865 } 866 simple_unlock(&anon->an_lock); 867 } 868 869 /* locked: maps(read), amap(if there) */ 870 /* (shadowed == TRUE) if there is an anon at the faulting address */ 871 UVMHIST_LOG(maphist, " shadowed=%d, will_get=%d", shadowed, 872 (uobj && shadowed == FALSE),0,0); 873 874 /* 875 * note that if we are really short of RAM we could sleep in the above 876 * call to pmap_enter with everything locked. bad? 877 * 878 * XXX Actually, that is bad; pmap_enter() should just fail in that 879 * XXX case. --thorpej 880 */ 881 882 /* 883 * if the desired page is not shadowed by the amap and we have a 884 * backing object, then we check to see if the backing object would 885 * prefer to handle the fault itself (rather than letting us do it 886 * with the usual pgo_get hook). the backing object signals this by 887 * providing a pgo_fault routine. 888 */ 889 890 if (uobj && shadowed == FALSE && uobj->pgops->pgo_fault != NULL) { 891 892 simple_lock(&uobj->vmobjlock); 893 894 /* locked: maps(read), amap (if there), uobj */ 895 result = uobj->pgops->pgo_fault(&ufi, startva, pages, npages, 896 centeridx, fault_type, access_type, 897 PGO_LOCKED); 898 /* locked: nothing, pgo_fault has unlocked everything */ 899 900 if (result == VM_PAGER_OK) 901 return (KERN_SUCCESS); /* pgo_fault did pmap enter */ 902 else if (result == VM_PAGER_REFAULT) 903 goto ReFault; /* try again! */ 904 else 905 return (KERN_PROTECTION_FAILURE); 906 } 907 908 /* 909 * now, if the desired page is not shadowed by the amap and we have 910 * a backing object that does not have a special fault routine, then 911 * we ask (with pgo_get) the object for resident pages that we care 912 * about and attempt to map them in. we do not let pgo_get block 913 * (PGO_LOCKED). 914 * 915 * ("get" has the option of doing a pmap_enter for us) 916 */ 917 918 if (uobj && shadowed == FALSE) { 919 simple_lock(&uobj->vmobjlock); 920 921 /* locked (!shadowed): maps(read), amap (if there), uobj */ 922 /* 923 * the following call to pgo_get does _not_ change locking state 924 */ 925 926 uvmexp.fltlget++; 927 gotpages = npages; 928 result = uobj->pgops->pgo_get(uobj, ufi.entry->offset + 929 (startva - ufi.entry->start), 930 pages, &gotpages, centeridx, 931 access_type & MASK(ufi.entry), 932 ufi.entry->advice, PGO_LOCKED); 933 934 /* 935 * check for pages to map, if we got any 936 */ 937 938 uobjpage = NULL; 939 940 if (gotpages) { 941 currva = startva; 942 for (lcv = 0 ; lcv < npages ; 943 lcv++, currva += PAGE_SIZE) { 944 945 if (pages[lcv] == NULL || 946 pages[lcv] == PGO_DONTCARE) 947 continue; 948 949 #ifdef DIAGNOSTIC 950 /* 951 * pager sanity check: pgo_get with 952 * PGO_LOCKED should never return a 953 * released page to us. 954 */ 955 if (pages[lcv]->flags & PG_RELEASED) 956 panic("uvm_fault: pgo_get PGO_LOCKED gave us a RELEASED page"); 957 #endif 958 959 /* 960 * if center page is resident and not 961 * PG_BUSY|PG_RELEASED then pgo_get 962 * made it PG_BUSY for us and gave 963 * us a handle to it. remember this 964 * page as "uobjpage." (for later use). 965 */ 966 967 if (lcv == centeridx) { 968 uobjpage = pages[lcv]; 969 UVMHIST_LOG(maphist, " got uobjpage (0x%x) with locked get", 970 uobjpage, 0,0,0); 971 continue; 972 } 973 974 /* 975 * note: calling pgo_get with locked data 976 * structures returns us pages which are 977 * neither busy nor released, so we don't 978 * need to check for this. we can just 979 * directly enter the page (after moving it 980 * to the head of the active queue [useful?]). 981 */ 982 983 uvm_lock_pageq(); 984 uvm_pageactivate(pages[lcv]); /* reactivate */ 985 uvm_unlock_pageq(); 986 UVMHIST_LOG(maphist, 987 " MAPPING: n obj: pm=0x%x, va=0x%x, pg=0x%x", 988 ufi.orig_map->pmap, currva, pages[lcv], 0); 989 uvmexp.fltnomap++; 990 /* 991 * Since this page isn't the page that's 992 * actually fauling, ignore pmap_enter() 993 * failures; it's not critical that we 994 * enter these right now. 995 */ 996 (void) pmap_enter(ufi.orig_map->pmap, currva, 997 VM_PAGE_TO_PHYS(pages[lcv]), 998 enter_prot & MASK(ufi.entry), 999 PMAP_CANFAIL | 1000 (wired ? PMAP_WIRED : 0)); 1001 1002 /* 1003 * NOTE: page can't be PG_WANTED or PG_RELEASED 1004 * because we've held the lock the whole time 1005 * we've had the handle. 1006 */ 1007 pages[lcv]->flags &= ~(PG_BUSY); /* un-busy! */ 1008 UVM_PAGE_OWN(pages[lcv], NULL); 1009 1010 /* done! */ 1011 } /* for "lcv" loop */ 1012 } /* "gotpages" != 0 */ 1013 1014 /* note: object still _locked_ */ 1015 } else { 1016 1017 uobjpage = NULL; 1018 1019 } 1020 1021 /* locked (shadowed): maps(read), amap */ 1022 /* locked (!shadowed): maps(read), amap(if there), 1023 uobj(if !null), uobjpage(if !null) */ 1024 1025 /* 1026 * note that at this point we are done with any front or back pages. 1027 * we are now going to focus on the center page (i.e. the one we've 1028 * faulted on). if we have faulted on the top (anon) layer 1029 * [i.e. case 1], then the anon we want is anons[centeridx] (we have 1030 * not touched it yet). if we have faulted on the bottom (uobj) 1031 * layer [i.e. case 2] and the page was both present and available, 1032 * then we've got a pointer to it as "uobjpage" and we've already 1033 * made it BUSY. 1034 */ 1035 1036 /* 1037 * there are four possible cases we must address: 1A, 1B, 2A, and 2B 1038 */ 1039 1040 /* 1041 * redirect case 2: if we are not shadowed, go to case 2. 1042 */ 1043 1044 if (shadowed == FALSE) 1045 goto Case2; 1046 1047 /* locked: maps(read), amap */ 1048 1049 /* 1050 * handle case 1: fault on an anon in our amap 1051 */ 1052 1053 anon = anons[centeridx]; 1054 UVMHIST_LOG(maphist, " case 1 fault: anon=0x%x", anon, 0,0,0); 1055 simple_lock(&anon->an_lock); 1056 1057 /* locked: maps(read), amap, anon */ 1058 1059 /* 1060 * no matter if we have case 1A or case 1B we are going to need to 1061 * have the anon's memory resident. ensure that now. 1062 */ 1063 1064 /* 1065 * let uvmfault_anonget do the dirty work. 1066 * if it fails (!OK) it will unlock all but the anon for us. 1067 * if it succeeds, locks are still valid and locked. 1068 * also, if it is OK, then the anon's page is on the queues. 1069 * if the page is on loan from a uvm_object, then anonget will 1070 * lock that object for us if it does not fail. 1071 */ 1072 1073 result = uvmfault_anonget(&ufi, amap, anon); 1074 if (result != VM_PAGER_OK) { 1075 simple_unlock(&anon->an_lock); 1076 } 1077 1078 if (result == VM_PAGER_REFAULT) 1079 goto ReFault; 1080 1081 if (result == VM_PAGER_AGAIN) { 1082 tsleep((caddr_t)&lbolt, PVM, "fltagain1", 0); 1083 goto ReFault; 1084 } 1085 1086 if (result != VM_PAGER_OK) 1087 return (KERN_PROTECTION_FAILURE); /* XXX??? */ 1088 1089 /* 1090 * uobj is non null if the page is on loan from an object (i.e. uobj) 1091 */ 1092 1093 uobj = anon->u.an_page->uobject; /* locked by anonget if !NULL */ 1094 1095 /* locked: maps(read), amap, anon, uobj(if one) */ 1096 1097 /* 1098 * special handling for loaned pages 1099 */ 1100 if (anon->u.an_page->loan_count) { 1101 1102 if ((access_type & VM_PROT_WRITE) == 0) { 1103 1104 /* 1105 * for read faults on loaned pages we just cap the 1106 * protection at read-only. 1107 */ 1108 1109 enter_prot = enter_prot & ~VM_PROT_WRITE; 1110 1111 } else { 1112 /* 1113 * note that we can't allow writes into a loaned page! 1114 * 1115 * if we have a write fault on a loaned page in an 1116 * anon then we need to look at the anon's ref count. 1117 * if it is greater than one then we are going to do 1118 * a normal copy-on-write fault into a new anon (this 1119 * is not a problem). however, if the reference count 1120 * is one (a case where we would normally allow a 1121 * write directly to the page) then we need to kill 1122 * the loan before we continue. 1123 */ 1124 1125 /* >1 case is already ok */ 1126 if (anon->an_ref == 1) { 1127 1128 /* get new un-owned replacement page */ 1129 pg = uvm_pagealloc(NULL, 0, NULL, 0); 1130 if (pg == NULL) { 1131 uvmfault_unlockall(&ufi, amap, uobj, 1132 anon); 1133 uvm_wait("flt_noram2"); 1134 goto ReFault; 1135 } 1136 1137 /* 1138 * copy data, kill loan, and drop uobj lock 1139 * (if any) 1140 */ 1141 /* copy old -> new */ 1142 uvm_pagecopy(anon->u.an_page, pg); 1143 1144 /* force reload */ 1145 pmap_page_protect(anon->u.an_page, 1146 VM_PROT_NONE); 1147 uvm_lock_pageq(); /* KILL loan */ 1148 if (uobj) 1149 /* if we were loaning */ 1150 anon->u.an_page->loan_count--; 1151 anon->u.an_page->uanon = NULL; 1152 /* in case we owned */ 1153 anon->u.an_page->pqflags &= ~PQ_ANON; 1154 uvm_unlock_pageq(); 1155 if (uobj) { 1156 simple_unlock(&uobj->vmobjlock); 1157 uobj = NULL; 1158 } 1159 1160 /* install new page in anon */ 1161 anon->u.an_page = pg; 1162 pg->uanon = anon; 1163 pg->pqflags |= PQ_ANON; 1164 pg->flags &= ~(PG_BUSY|PG_FAKE); 1165 UVM_PAGE_OWN(pg, NULL); 1166 1167 /* done! */ 1168 } /* ref == 1 */ 1169 } /* write fault */ 1170 } /* loan count */ 1171 1172 /* 1173 * if we are case 1B then we will need to allocate a new blank 1174 * anon to transfer the data into. note that we have a lock 1175 * on anon, so no one can busy or release the page until we are done. 1176 * also note that the ref count can't drop to zero here because 1177 * it is > 1 and we are only dropping one ref. 1178 * 1179 * in the (hopefully very rare) case that we are out of RAM we 1180 * will unlock, wait for more RAM, and refault. 1181 * 1182 * if we are out of anon VM we kill the process (XXX: could wait?). 1183 */ 1184 1185 if ((access_type & VM_PROT_WRITE) != 0 && anon->an_ref > 1) { 1186 1187 UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0); 1188 uvmexp.flt_acow++; 1189 oanon = anon; /* oanon = old, locked anon */ 1190 anon = uvm_analloc(); 1191 if (anon) 1192 pg = uvm_pagealloc(NULL, 0, anon, 0); 1193 #ifdef __GNUC__ 1194 else 1195 pg = NULL; /* XXX: gcc */ 1196 #endif 1197 1198 /* check for out of RAM */ 1199 if (anon == NULL || pg == NULL) { 1200 if (anon) 1201 uvm_anfree(anon); 1202 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1203 #ifdef DIAGNOSTIC 1204 if (uvmexp.swpgonly > uvmexp.swpages) { 1205 panic("uvmexp.swpgonly botch"); 1206 } 1207 #endif 1208 if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) { 1209 UVMHIST_LOG(maphist, 1210 "<- failed. out of VM",0,0,0,0); 1211 uvmexp.fltnoanon++; 1212 return (KERN_RESOURCE_SHORTAGE); 1213 } 1214 1215 uvmexp.fltnoram++; 1216 uvm_wait("flt_noram3"); /* out of RAM, wait for more */ 1217 goto ReFault; 1218 } 1219 1220 /* got all resources, replace anon with nanon */ 1221 1222 uvm_pagecopy(oanon->u.an_page, pg); /* pg now !PG_CLEAN */ 1223 pg->flags &= ~(PG_BUSY|PG_FAKE); /* un-busy! new page */ 1224 UVM_PAGE_OWN(pg, NULL); 1225 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start, 1226 anon, 1); 1227 1228 /* deref: can not drop to zero here by defn! */ 1229 oanon->an_ref--; 1230 1231 /* 1232 * note: oanon still locked. anon is _not_ locked, but we 1233 * have the sole references to in from amap which _is_ locked. 1234 * thus, no one can get at it until we are done with it. 1235 */ 1236 1237 } else { 1238 1239 uvmexp.flt_anon++; 1240 oanon = anon; /* old, locked anon is same as anon */ 1241 pg = anon->u.an_page; 1242 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */ 1243 enter_prot = enter_prot & ~VM_PROT_WRITE; 1244 1245 } 1246 1247 /* locked: maps(read), amap, anon */ 1248 1249 /* 1250 * now map the page in ... 1251 * XXX: old fault unlocks object before pmap_enter. this seems 1252 * suspect since some other thread could blast the page out from 1253 * under us between the unlock and the pmap_enter. 1254 */ 1255 1256 UVMHIST_LOG(maphist, " MAPPING: anon: pm=0x%x, va=0x%x, pg=0x%x", 1257 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, 0); 1258 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1259 enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) 1260 != KERN_SUCCESS) { 1261 /* 1262 * No need to undo what we did; we can simply think of 1263 * this as the pmap throwing away the mapping information. 1264 * 1265 * We do, however, have to go through the ReFault path, 1266 * as the map may change while we're asleep. 1267 */ 1268 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1269 #ifdef DIAGNOSTIC 1270 if (uvmexp.swpgonly > uvmexp.swpages) 1271 panic("uvmexp.swpgonly botch"); 1272 #endif 1273 if (uvmexp.swpgonly == uvmexp.swpages) { 1274 UVMHIST_LOG(maphist, 1275 "<- failed. out of VM",0,0,0,0); 1276 /* XXX instrumentation */ 1277 return (KERN_RESOURCE_SHORTAGE); 1278 } 1279 /* XXX instrumentation */ 1280 uvm_wait("flt_pmfail1"); 1281 goto ReFault; 1282 } 1283 1284 /* 1285 * ... update the page queues. 1286 */ 1287 1288 uvm_lock_pageq(); 1289 1290 if (fault_type == VM_FAULT_WIRE) { 1291 uvm_pagewire(pg); 1292 1293 /* 1294 * since the now-wired page cannot be paged out, 1295 * release its swap resources for others to use. 1296 * since an anon with no swap cannot be PG_CLEAN, 1297 * clear its clean flag now. 1298 */ 1299 1300 pg->flags &= ~(PG_CLEAN); 1301 uvm_anon_dropswap(anon); 1302 } else { 1303 /* activate it */ 1304 uvm_pageactivate(pg); 1305 } 1306 1307 uvm_unlock_pageq(); 1308 1309 /* 1310 * done case 1! finish up by unlocking everything and returning success 1311 */ 1312 1313 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1314 return (KERN_SUCCESS); 1315 1316 1317 Case2: 1318 /* 1319 * handle case 2: faulting on backing object or zero fill 1320 */ 1321 1322 /* 1323 * locked: 1324 * maps(read), amap(if there), uobj(if !null), uobjpage(if !null) 1325 */ 1326 1327 /* 1328 * note that uobjpage can not be PGO_DONTCARE at this point. we now 1329 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we 1330 * have a backing object, check and see if we are going to promote 1331 * the data up to an anon during the fault. 1332 */ 1333 1334 if (uobj == NULL) { 1335 uobjpage = PGO_DONTCARE; 1336 promote = TRUE; /* always need anon here */ 1337 } else { 1338 /* assert(uobjpage != PGO_DONTCARE) */ 1339 promote = (access_type & VM_PROT_WRITE) && 1340 UVM_ET_ISCOPYONWRITE(ufi.entry); 1341 } 1342 UVMHIST_LOG(maphist, " case 2 fault: promote=%d, zfill=%d", 1343 promote, (uobj == NULL), 0,0); 1344 1345 /* 1346 * if uobjpage is not null then we do not need to do I/O to get the 1347 * uobjpage. 1348 * 1349 * if uobjpage is null, then we need to unlock and ask the pager to 1350 * get the data for us. once we have the data, we need to reverify 1351 * the state the world. we are currently not holding any resources. 1352 */ 1353 1354 if (uobjpage) { 1355 /* update rusage counters */ 1356 curproc->p_addr->u_stats.p_ru.ru_minflt++; 1357 } else { 1358 /* update rusage counters */ 1359 curproc->p_addr->u_stats.p_ru.ru_majflt++; 1360 1361 /* locked: maps(read), amap(if there), uobj */ 1362 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1363 /* locked: uobj */ 1364 1365 uvmexp.fltget++; 1366 gotpages = 1; 1367 result = uobj->pgops->pgo_get(uobj, 1368 (ufi.orig_rvaddr - ufi.entry->start) + ufi.entry->offset, 1369 &uobjpage, &gotpages, 0, 1370 access_type & MASK(ufi.entry), 1371 ufi.entry->advice, 0); 1372 1373 /* locked: uobjpage(if result OK) */ 1374 1375 /* 1376 * recover from I/O 1377 */ 1378 1379 if (result != VM_PAGER_OK) { 1380 #ifdef DIAGNOSTIC 1381 if (result == VM_PAGER_PEND) 1382 panic("uvm_fault: pgo_get got PENDing " 1383 "on non-async I/O"); 1384 #endif 1385 1386 if (result == VM_PAGER_AGAIN) { 1387 UVMHIST_LOG(maphist, 1388 " pgo_get says TRY AGAIN!",0,0,0,0); 1389 tsleep((caddr_t)&lbolt, PVM, "fltagain2", 0); 1390 goto ReFault; 1391 } 1392 1393 UVMHIST_LOG(maphist, "<- pgo_get failed (code %d)", 1394 result, 0,0,0); 1395 return (KERN_PROTECTION_FAILURE); /* XXX i/o error */ 1396 } 1397 1398 /* locked: uobjpage */ 1399 1400 /* 1401 * re-verify the state of the world by first trying to relock 1402 * the maps. always relock the object. 1403 */ 1404 1405 locked = uvmfault_relock(&ufi); 1406 if (locked && amap) 1407 amap_lock(amap); 1408 simple_lock(&uobj->vmobjlock); 1409 1410 /* locked(locked): maps(read), amap(if !null), uobj, uobjpage */ 1411 /* locked(!locked): uobj, uobjpage */ 1412 1413 /* 1414 * verify that the page has not be released and re-verify 1415 * that amap slot is still free. if there is a problem, 1416 * we unlock and clean up. 1417 */ 1418 1419 if ((uobjpage->flags & PG_RELEASED) != 0 || 1420 (locked && amap && 1421 amap_lookup(&ufi.entry->aref, 1422 ufi.orig_rvaddr - ufi.entry->start))) { 1423 if (locked) 1424 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1425 locked = FALSE; 1426 } 1427 1428 /* 1429 * didn't get the lock? release the page and retry. 1430 */ 1431 1432 if (locked == FALSE) { 1433 1434 UVMHIST_LOG(maphist, 1435 " wasn't able to relock after fault: retry", 1436 0,0,0,0); 1437 if (uobjpage->flags & PG_WANTED) 1438 /* still holding object lock */ 1439 wakeup(uobjpage); 1440 1441 if (uobjpage->flags & PG_RELEASED) { 1442 uvmexp.fltpgrele++; 1443 #ifdef DIAGNOSTIC 1444 if (uobj->pgops->pgo_releasepg == NULL) 1445 panic("uvm_fault: object has no " 1446 "releasepg function"); 1447 #endif 1448 /* frees page */ 1449 if (uobj->pgops->pgo_releasepg(uobjpage,NULL)) 1450 /* unlock if still alive */ 1451 simple_unlock(&uobj->vmobjlock); 1452 goto ReFault; 1453 } 1454 1455 uvm_lock_pageq(); 1456 /* make sure it is in queues */ 1457 uvm_pageactivate(uobjpage); 1458 1459 uvm_unlock_pageq(); 1460 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1461 UVM_PAGE_OWN(uobjpage, NULL); 1462 simple_unlock(&uobj->vmobjlock); 1463 goto ReFault; 1464 1465 } 1466 1467 /* 1468 * we have the data in uobjpage which is PG_BUSY and 1469 * !PG_RELEASED. we are holding object lock (so the page 1470 * can't be released on us). 1471 */ 1472 1473 /* locked: maps(read), amap(if !null), uobj, uobjpage */ 1474 1475 } 1476 1477 /* 1478 * locked: 1479 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj) 1480 */ 1481 1482 /* 1483 * notes: 1484 * - at this point uobjpage can not be NULL 1485 * - at this point uobjpage can not be PG_RELEASED (since we checked 1486 * for it above) 1487 * - at this point uobjpage could be PG_WANTED (handle later) 1488 */ 1489 1490 if (promote == FALSE) { 1491 1492 /* 1493 * we are not promoting. if the mapping is COW ensure that we 1494 * don't give more access than we should (e.g. when doing a read 1495 * fault on a COPYONWRITE mapping we want to map the COW page in 1496 * R/O even though the entry protection could be R/W). 1497 * 1498 * set "pg" to the page we want to map in (uobjpage, usually) 1499 */ 1500 1501 uvmexp.flt_obj++; 1502 if (UVM_ET_ISCOPYONWRITE(ufi.entry)) 1503 enter_prot &= ~VM_PROT_WRITE; 1504 pg = uobjpage; /* map in the actual object */ 1505 1506 /* assert(uobjpage != PGO_DONTCARE) */ 1507 1508 /* 1509 * we are faulting directly on the page. be careful 1510 * about writing to loaned pages... 1511 */ 1512 if (uobjpage->loan_count) { 1513 1514 if ((access_type & VM_PROT_WRITE) == 0) { 1515 /* read fault: cap the protection at readonly */ 1516 /* cap! */ 1517 enter_prot = enter_prot & ~VM_PROT_WRITE; 1518 } else { 1519 /* write fault: must break the loan here */ 1520 1521 /* alloc new un-owned page */ 1522 pg = uvm_pagealloc(NULL, 0, NULL, 0); 1523 1524 if (pg == NULL) { 1525 /* 1526 * drop ownership of page, it can't 1527 * be released 1528 */ 1529 if (uobjpage->flags & PG_WANTED) 1530 wakeup(uobjpage); 1531 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1532 UVM_PAGE_OWN(uobjpage, NULL); 1533 1534 uvm_lock_pageq(); 1535 /* activate: we will need it later */ 1536 uvm_pageactivate(uobjpage); 1537 1538 uvm_unlock_pageq(); 1539 uvmfault_unlockall(&ufi, amap, uobj, 1540 NULL); 1541 UVMHIST_LOG(maphist, 1542 " out of RAM breaking loan, waiting", 1543 0,0,0,0); 1544 uvmexp.fltnoram++; 1545 uvm_wait("flt_noram4"); 1546 goto ReFault; 1547 } 1548 1549 /* 1550 * copy the data from the old page to the new 1551 * one and clear the fake/clean flags on the 1552 * new page (keep it busy). force a reload 1553 * of the old page by clearing it from all 1554 * pmaps. then lock the page queues to 1555 * rename the pages. 1556 */ 1557 uvm_pagecopy(uobjpage, pg); /* old -> new */ 1558 pg->flags &= ~(PG_FAKE|PG_CLEAN); 1559 pmap_page_protect(uobjpage, VM_PROT_NONE); 1560 if (uobjpage->flags & PG_WANTED) 1561 wakeup(uobjpage); 1562 /* uobj still locked */ 1563 uobjpage->flags &= ~(PG_WANTED|PG_BUSY); 1564 UVM_PAGE_OWN(uobjpage, NULL); 1565 1566 uvm_lock_pageq(); 1567 offset = uobjpage->offset; 1568 /* remove old page */ 1569 uvm_pagerealloc(uobjpage, NULL, 0); 1570 1571 /* 1572 * at this point we have absolutely no 1573 * control over uobjpage 1574 */ 1575 /* install new page */ 1576 uvm_pagerealloc(pg, uobj, offset); 1577 uvm_unlock_pageq(); 1578 1579 /* 1580 * done! loan is broken and "pg" is 1581 * PG_BUSY. it can now replace uobjpage. 1582 */ 1583 1584 uobjpage = pg; 1585 1586 } /* write fault case */ 1587 } /* if loan_count */ 1588 1589 } else { 1590 1591 /* 1592 * if we are going to promote the data to an anon we 1593 * allocate a blank anon here and plug it into our amap. 1594 */ 1595 #if DIAGNOSTIC 1596 if (amap == NULL) 1597 panic("uvm_fault: want to promote data, but no anon"); 1598 #endif 1599 1600 anon = uvm_analloc(); 1601 if (anon) { 1602 /* 1603 * In `Fill in data...' below, if 1604 * uobjpage == PGO_DONTCARE, we want 1605 * a zero'd, dirty page, so have 1606 * uvm_pagealloc() do that for us. 1607 */ 1608 pg = uvm_pagealloc(NULL, 0, anon, 1609 (uobjpage == PGO_DONTCARE) ? UVM_PGA_ZERO : 0); 1610 } 1611 #ifdef __GNUC__ 1612 else 1613 pg = NULL; /* XXX: gcc */ 1614 #endif 1615 1616 /* 1617 * out of memory resources? 1618 */ 1619 if (anon == NULL || pg == NULL) { 1620 1621 /* 1622 * arg! must unbusy our page and fail or sleep. 1623 */ 1624 if (uobjpage != PGO_DONTCARE) { 1625 if (uobjpage->flags & PG_WANTED) 1626 /* still holding object lock */ 1627 wakeup(uobjpage); 1628 1629 uvm_lock_pageq(); 1630 /* make sure it is in queues */ 1631 uvm_pageactivate(uobjpage); 1632 uvm_unlock_pageq(); 1633 /* un-busy! (still locked) */ 1634 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1635 UVM_PAGE_OWN(uobjpage, NULL); 1636 } 1637 1638 /* unlock and fail ... */ 1639 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1640 #ifdef DIAGNOSTIC 1641 if (uvmexp.swpgonly > uvmexp.swpages) { 1642 panic("uvmexp.swpgonly botch"); 1643 } 1644 #endif 1645 if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) { 1646 UVMHIST_LOG(maphist, " promote: out of VM", 1647 0,0,0,0); 1648 uvmexp.fltnoanon++; 1649 return (KERN_RESOURCE_SHORTAGE); 1650 } 1651 1652 UVMHIST_LOG(maphist, " out of RAM, waiting for more", 1653 0,0,0,0); 1654 uvm_anfree(anon); 1655 uvmexp.fltnoram++; 1656 uvm_wait("flt_noram5"); 1657 goto ReFault; 1658 } 1659 1660 /* 1661 * fill in the data 1662 */ 1663 1664 if (uobjpage != PGO_DONTCARE) { 1665 uvmexp.flt_prcopy++; 1666 /* copy page [pg now dirty] */ 1667 uvm_pagecopy(uobjpage, pg); 1668 1669 /* 1670 * promote to shared amap? make sure all sharing 1671 * procs see it 1672 */ 1673 if ((amap_flags(amap) & AMAP_SHARED) != 0) { 1674 pmap_page_protect(uobjpage, VM_PROT_NONE); 1675 } 1676 1677 /* 1678 * dispose of uobjpage. it can't be PG_RELEASED 1679 * since we still hold the object lock. drop 1680 * handle to uobj as well. 1681 */ 1682 1683 if (uobjpage->flags & PG_WANTED) 1684 /* still have the obj lock */ 1685 wakeup(uobjpage); 1686 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1687 UVM_PAGE_OWN(uobjpage, NULL); 1688 uvm_lock_pageq(); 1689 uvm_pageactivate(uobjpage); /* put it back */ 1690 uvm_unlock_pageq(); 1691 simple_unlock(&uobj->vmobjlock); 1692 uobj = NULL; 1693 UVMHIST_LOG(maphist, 1694 " promote uobjpage 0x%x to anon/page 0x%x/0x%x", 1695 uobjpage, anon, pg, 0); 1696 1697 } else { 1698 uvmexp.flt_przero++; 1699 /* 1700 * Page is zero'd and marked dirty by uvm_pagealloc() 1701 * above. 1702 */ 1703 UVMHIST_LOG(maphist," zero fill anon/page 0x%x/0%x", 1704 anon, pg, 0, 0); 1705 } 1706 1707 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start, 1708 anon, 0); 1709 1710 } 1711 1712 /* 1713 * locked: 1714 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj) 1715 * 1716 * note: pg is either the uobjpage or the new page in the new anon 1717 */ 1718 1719 /* 1720 * all resources are present. we can now map it in and free our 1721 * resources. 1722 */ 1723 1724 UVMHIST_LOG(maphist, 1725 " MAPPING: case2: pm=0x%x, va=0x%x, pg=0x%x, promote=%d", 1726 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, promote); 1727 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1728 enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) 1729 != KERN_SUCCESS) { 1730 /* 1731 * No need to undo what we did; we can simply think of 1732 * this as the pmap throwing away the mapping information. 1733 * 1734 * We do, however, have to go through the ReFault path, 1735 * as the map may change while we're asleep. 1736 */ 1737 if (pg->flags & PG_WANTED) 1738 wakeup(pg); /* lock still held */ 1739 1740 /* 1741 * note that pg can't be PG_RELEASED since we did not drop 1742 * the object lock since the last time we checked. 1743 */ 1744 1745 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); 1746 UVM_PAGE_OWN(pg, NULL); 1747 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1748 #ifdef DIAGNOSTIC 1749 if (uvmexp.swpgonly > uvmexp.swpages) 1750 panic("uvmexp.swpgonly botch"); 1751 #endif 1752 if (uvmexp.swpgonly == uvmexp.swpages) { 1753 UVMHIST_LOG(maphist, 1754 "<- failed. out of VM",0,0,0,0); 1755 /* XXX instrumentation */ 1756 return (KERN_RESOURCE_SHORTAGE); 1757 } 1758 /* XXX instrumentation */ 1759 uvm_wait("flt_pmfail2"); 1760 goto ReFault; 1761 } 1762 1763 uvm_lock_pageq(); 1764 1765 if (fault_type == VM_FAULT_WIRE) { 1766 uvm_pagewire(pg); 1767 if (pg->pqflags & PQ_AOBJ) { 1768 1769 /* 1770 * since the now-wired page cannot be paged out, 1771 * release its swap resources for others to use. 1772 * since an aobj page with no swap cannot be PG_CLEAN, 1773 * clear its clean flag now. 1774 */ 1775 1776 pg->flags &= ~(PG_CLEAN); 1777 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT); 1778 } 1779 } else { 1780 /* activate it */ 1781 uvm_pageactivate(pg); 1782 } 1783 1784 uvm_unlock_pageq(); 1785 1786 if (pg->flags & PG_WANTED) 1787 wakeup(pg); /* lock still held */ 1788 1789 /* 1790 * note that pg can't be PG_RELEASED since we did not drop the object 1791 * lock since the last time we checked. 1792 */ 1793 1794 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); 1795 UVM_PAGE_OWN(pg, NULL); 1796 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1797 1798 UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0); 1799 return (KERN_SUCCESS); 1800 } 1801 1802 1803 /* 1804 * uvm_fault_wire: wire down a range of virtual addresses in a map. 1805 * 1806 * => map may be read-locked by caller, but MUST NOT be write-locked. 1807 * => if map is read-locked, any operations which may cause map to 1808 * be write-locked in uvm_fault() must be taken care of by 1809 * the caller. See uvm_map_pageable(). 1810 */ 1811 1812 int 1813 uvm_fault_wire(map, start, end, access_type) 1814 vm_map_t map; 1815 vaddr_t start, end; 1816 vm_prot_t access_type; 1817 { 1818 vaddr_t va; 1819 pmap_t pmap; 1820 int rv; 1821 1822 pmap = vm_map_pmap(map); 1823 1824 /* 1825 * now fault it in a page at a time. if the fault fails then we have 1826 * to undo what we have done. note that in uvm_fault VM_PROT_NONE 1827 * is replaced with the max protection if fault_type is VM_FAULT_WIRE. 1828 */ 1829 1830 for (va = start ; va < end ; va += PAGE_SIZE) { 1831 rv = uvm_fault(map, va, VM_FAULT_WIRE, access_type); 1832 if (rv) { 1833 if (va != start) { 1834 uvm_fault_unwire(map, start, va); 1835 } 1836 return (rv); 1837 } 1838 } 1839 1840 return (KERN_SUCCESS); 1841 } 1842 1843 /* 1844 * uvm_fault_unwire(): unwire range of virtual space. 1845 */ 1846 1847 void 1848 uvm_fault_unwire(map, start, end) 1849 vm_map_t map; 1850 vaddr_t start, end; 1851 { 1852 1853 vm_map_lock_read(map); 1854 uvm_fault_unwire_locked(map, start, end); 1855 vm_map_unlock_read(map); 1856 } 1857 1858 /* 1859 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire(). 1860 * 1861 * => map must be at least read-locked. 1862 */ 1863 1864 void 1865 uvm_fault_unwire_locked(map, start, end) 1866 vm_map_t map; 1867 vaddr_t start, end; 1868 { 1869 vm_map_entry_t entry; 1870 pmap_t pmap = vm_map_pmap(map); 1871 vaddr_t va; 1872 paddr_t pa; 1873 struct vm_page *pg; 1874 1875 #ifdef DIAGNOSTIC 1876 if (map->flags & VM_MAP_INTRSAFE) 1877 panic("uvm_fault_unwire_locked: intrsafe map"); 1878 #endif 1879 1880 /* 1881 * we assume that the area we are unwiring has actually been wired 1882 * in the first place. this means that we should be able to extract 1883 * the PAs from the pmap. we also lock out the page daemon so that 1884 * we can call uvm_pageunwire. 1885 */ 1886 1887 uvm_lock_pageq(); 1888 1889 /* 1890 * find the beginning map entry for the region. 1891 */ 1892 #ifdef DIAGNOSTIC 1893 if (start < vm_map_min(map) || end > vm_map_max(map)) 1894 panic("uvm_fault_unwire_locked: address out of range"); 1895 #endif 1896 if (uvm_map_lookup_entry(map, start, &entry) == FALSE) 1897 panic("uvm_fault_unwire_locked: address not in map"); 1898 1899 for (va = start; va < end ; va += PAGE_SIZE) { 1900 if (pmap_extract(pmap, va, &pa) == FALSE) 1901 panic("uvm_fault_unwire_locked: unwiring " 1902 "non-wired memory"); 1903 1904 /* 1905 * make sure the current entry is for the address we're 1906 * dealing with. if not, grab the next entry. 1907 */ 1908 #ifdef DIAGNOSTIC 1909 if (va < entry->start) 1910 panic("uvm_fault_unwire_locked: hole 1"); 1911 #endif 1912 if (va >= entry->end) { 1913 #ifdef DIAGNOSTIC 1914 if (entry->next == &map->header || 1915 entry->next->start > entry->end) 1916 panic("uvm_fault_unwire_locked: hole 2"); 1917 #endif 1918 entry = entry->next; 1919 } 1920 1921 /* 1922 * if the entry is no longer wired, tell the pmap. 1923 */ 1924 if (VM_MAPENT_ISWIRED(entry) == 0) 1925 pmap_unwire(pmap, va); 1926 1927 pg = PHYS_TO_VM_PAGE(pa); 1928 if (pg) 1929 uvm_pageunwire(pg); 1930 } 1931 1932 uvm_unlock_pageq(); 1933 } 1934