1 /* $NetBSD: uvm_fault.c,v 1.18 1998/11/20 19:37:06 chuck Exp $ */ 2 3 /* 4 * XXXCDC: "ROUGH DRAFT" QUALITY UVM PRE-RELEASE FILE! 5 * >>>USE AT YOUR OWN RISK, WORK IS NOT FINISHED<<< 6 */ 7 /* 8 * 9 * Copyright (c) 1997 Charles D. Cranor and Washington University. 10 * All rights reserved. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. All advertising materials mentioning features or use of this software 21 * must display the following acknowledgement: 22 * This product includes software developed by Charles D. Cranor and 23 * Washington University. 24 * 4. The name of the author may not be used to endorse or promote products 25 * derived from this software without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 28 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 29 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 30 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 31 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 32 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 33 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 34 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 35 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 36 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 37 * 38 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp 39 */ 40 41 #include "opt_uvmhist.h" 42 43 /* 44 * uvm_fault.c: fault handler 45 */ 46 47 #include <sys/param.h> 48 #include <sys/systm.h> 49 #include <sys/kernel.h> 50 #include <sys/proc.h> 51 #include <sys/malloc.h> 52 #include <sys/mman.h> 53 #include <sys/user.h> 54 55 #include <vm/vm.h> 56 #include <vm/vm_page.h> 57 #include <vm/vm_kern.h> 58 59 #include <uvm/uvm.h> 60 61 /* 62 * 63 * a word on page faults: 64 * 65 * types of page faults we handle: 66 * 67 * CASE 1: upper layer faults CASE 2: lower layer faults 68 * 69 * CASE 1A CASE 1B CASE 2A CASE 2B 70 * read/write1 write>1 read/write +-cow_write/zero 71 * | | | | 72 * +--|--+ +--|--+ +-----+ + | + | +-----+ 73 * amap | V | | ----------->new| | | | ^ | 74 * +-----+ +-----+ +-----+ + | + | +--|--+ 75 * | | | 76 * +-----+ +-----+ +--|--+ | +--|--+ 77 * uobj | d/c | | d/c | | V | +----| | 78 * +-----+ +-----+ +-----+ +-----+ 79 * 80 * d/c = don't care 81 * 82 * case [0]: layerless fault 83 * no amap or uobj is present. this is an error. 84 * 85 * case [1]: upper layer fault [anon active] 86 * 1A: [read] or [write with anon->an_ref == 1] 87 * I/O takes place in top level anon and uobj is not touched. 88 * 1B: [write with anon->an_ref > 1] 89 * new anon is alloc'd and data is copied off ["COW"] 90 * 91 * case [2]: lower layer fault [uobj] 92 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area] 93 * I/O takes place directly in object. 94 * 2B: [write to copy_on_write] or [read on NULL uobj] 95 * data is "promoted" from uobj to a new anon. 96 * if uobj is null, then we zero fill. 97 * 98 * we follow the standard UVM locking protocol ordering: 99 * 100 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ) 101 * we hold a PG_BUSY page if we unlock for I/O 102 * 103 * 104 * the code is structured as follows: 105 * 106 * - init the "IN" params in the ufi structure 107 * ReFault: 108 * - do lookups [locks maps], check protection, handle needs_copy 109 * - check for case 0 fault (error) 110 * - establish "range" of fault 111 * - if we have an amap lock it and extract the anons 112 * - if sequential advice deactivate pages behind us 113 * - at the same time check pmap for unmapped areas and anon for pages 114 * that we could map in (and do map it if found) 115 * - check object for resident pages that we could map in 116 * - if (case 2) goto Case2 117 * - >>> handle case 1 118 * - ensure source anon is resident in RAM 119 * - if case 1B alloc new anon and copy from source 120 * - map the correct page in 121 * Case2: 122 * - >>> handle case 2 123 * - ensure source page is resident (if uobj) 124 * - if case 2B alloc new anon and copy from source (could be zero 125 * fill if uobj == NULL) 126 * - map the correct page in 127 * - done! 128 * 129 * note on paging: 130 * if we have to do I/O we place a PG_BUSY page in the correct object, 131 * unlock everything, and do the I/O. when I/O is done we must reverify 132 * the state of the world before assuming that our data structures are 133 * valid. [because mappings could change while the map is unlocked] 134 * 135 * alternative 1: unbusy the page in question and restart the page fault 136 * from the top (ReFault). this is easy but does not take advantage 137 * of the information that we already have from our previous lookup, 138 * although it is possible that the "hints" in the vm_map will help here. 139 * 140 * alternative 2: the system already keeps track of a "version" number of 141 * a map. [i.e. every time you write-lock a map (e.g. to change a 142 * mapping) you bump the version number up by one...] so, we can save 143 * the version number of the map before we release the lock and start I/O. 144 * then when I/O is done we can relock and check the version numbers 145 * to see if anything changed. this might save us some over 1 because 146 * we don't have to unbusy the page and may be less compares(?). 147 * 148 * alternative 3: put in backpointers or a way to "hold" part of a map 149 * in place while I/O is in progress. this could be complex to 150 * implement (especially with structures like amap that can be referenced 151 * by multiple map entries, and figuring out what should wait could be 152 * complex as well...). 153 * 154 * given that we are not currently multiprocessor or multithreaded we might 155 * as well choose alternative 2 now. maybe alternative 3 would be useful 156 * in the future. XXX keep in mind for future consideration//rechecking. 157 */ 158 159 /* 160 * local data structures 161 */ 162 163 struct uvm_advice { 164 int advice; 165 int nback; 166 int nforw; 167 }; 168 169 /* 170 * page range array: 171 * note: index in array must match "advice" value 172 * XXX: borrowed numbers from freebsd. do they work well for us? 173 */ 174 175 static struct uvm_advice uvmadvice[] = { 176 { MADV_NORMAL, 3, 4 }, 177 { MADV_RANDOM, 0, 0 }, 178 { MADV_SEQUENTIAL, 8, 7}, 179 }; 180 181 #define UVM_MAXRANGE 16 /* must be max() of nback+nforw+1 */ 182 183 /* 184 * private prototypes 185 */ 186 187 static void uvmfault_amapcopy __P((struct uvm_faultinfo *)); 188 static __inline void uvmfault_anonflush __P((struct vm_anon **, int)); 189 190 /* 191 * inline functions 192 */ 193 194 /* 195 * uvmfault_anonflush: try and deactivate pages in specified anons 196 * 197 * => does not have to deactivate page if it is busy 198 */ 199 200 static __inline void 201 uvmfault_anonflush(anons, n) 202 struct vm_anon **anons; 203 int n; 204 { 205 int lcv; 206 struct vm_page *pg; 207 208 for (lcv = 0 ; lcv < n ; lcv++) { 209 if (anons[lcv] == NULL) 210 continue; 211 simple_lock(&anons[lcv]->an_lock); 212 pg = anons[lcv]->u.an_page; 213 if (pg && (pg->flags & PG_BUSY) == 0 && pg->loan_count == 0) { 214 uvm_lock_pageq(); 215 if (pg->wire_count == 0) { 216 pmap_page_protect(PMAP_PGARG(pg), VM_PROT_NONE); 217 uvm_pagedeactivate(pg); 218 } 219 uvm_unlock_pageq(); 220 } 221 simple_unlock(&anons[lcv]->an_lock); 222 } 223 } 224 225 /* 226 * normal functions 227 */ 228 229 /* 230 * uvmfault_amapcopy: clear "needs_copy" in a map. 231 * 232 * => called with VM data structures unlocked (usually, see below) 233 * => we get a write lock on the maps and clear needs_copy for a VA 234 * => if we are out of RAM we sleep (waiting for more) 235 */ 236 237 static void 238 uvmfault_amapcopy(ufi) 239 struct uvm_faultinfo *ufi; 240 { 241 242 /* 243 * while we haven't done the job 244 */ 245 246 while (1) { 247 248 /* 249 * no mapping? give up. 250 */ 251 252 if (uvmfault_lookup(ufi, TRUE) == FALSE) 253 return; 254 255 /* 256 * copy if needed. 257 */ 258 259 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) 260 amap_copy(ufi->map, ufi->entry, M_NOWAIT, TRUE, 261 ufi->orig_rvaddr, ufi->orig_rvaddr + 1); 262 263 /* 264 * didn't work? must be out of RAM. unlock and sleep. 265 */ 266 267 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 268 uvmfault_unlockmaps(ufi, TRUE); 269 uvm_wait("fltamapcopy"); 270 continue; 271 } 272 273 /* 274 * got it! unlock and return. 275 */ 276 277 uvmfault_unlockmaps(ufi, TRUE); 278 return; 279 } 280 /*NOTREACHED*/ 281 } 282 283 /* 284 * uvmfault_anonget: get data in an anon into a non-busy, non-released 285 * page in that anon. 286 * 287 * => maps, amap, and anon locked by caller. 288 * => if we fail (result != VM_PAGER_OK) we unlock everything. 289 * => if we are successful, we return with everything still locked. 290 * => we don't move the page on the queues [gets moved later] 291 * => if we allocate a new page [we_own], it gets put on the queues. 292 * either way, the result is that the page is on the queues at return time 293 * => for pages which are on loan from a uvm_object (and thus are not 294 * owned by the anon): if successful, we return with the owning object 295 * locked. the caller must unlock this object when it unlocks everything 296 * else. 297 */ 298 299 int uvmfault_anonget(ufi, amap, anon) 300 struct uvm_faultinfo *ufi; 301 struct vm_amap *amap; 302 struct vm_anon *anon; 303 { 304 boolean_t we_own; /* we own anon's page? */ 305 boolean_t locked; /* did we relock? */ 306 struct vm_page *pg; 307 int result; 308 UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist); 309 310 result = 0; /* XXX shut up gcc */ 311 uvmexp.fltanget++; 312 /* bump rusage counters */ 313 if (anon->u.an_page) 314 curproc->p_addr->u_stats.p_ru.ru_minflt++; 315 else 316 curproc->p_addr->u_stats.p_ru.ru_majflt++; 317 318 /* 319 * loop until we get it, or fail. 320 */ 321 322 while (1) { 323 324 we_own = FALSE; /* TRUE if we set PG_BUSY on a page */ 325 pg = anon->u.an_page; 326 327 /* 328 * if there is a resident page and it is loaned, then anon 329 * may not own it. call out to uvm_anon_lockpage() to ensure 330 * the real owner of the page has been identified and locked. 331 */ 332 333 if (pg && pg->loan_count) 334 pg = uvm_anon_lockloanpg(anon); 335 336 /* 337 * page there? make sure it is not busy/released. 338 */ 339 340 if (pg) { 341 342 /* 343 * at this point, if the page has a uobject [meaning 344 * we have it on loan], then that uobject is locked 345 * by us! if the page is busy, we drop all the 346 * locks (including uobject) and try again. 347 */ 348 349 if ((pg->flags & (PG_BUSY|PG_RELEASED)) == 0) { 350 UVMHIST_LOG(maphist, "<- OK",0,0,0,0); 351 return (VM_PAGER_OK); 352 } 353 pg->flags |= PG_WANTED; 354 uvmexp.fltpgwait++; 355 356 /* 357 * the last unlock must be an atomic unlock+wait on 358 * the owner of page 359 */ 360 if (pg->uobject) { /* owner is uobject ? */ 361 uvmfault_unlockall(ufi, amap, NULL, anon); 362 UVMHIST_LOG(maphist, " unlock+wait on uobj",0, 363 0,0,0); 364 UVM_UNLOCK_AND_WAIT(pg, 365 &pg->uobject->vmobjlock, 366 FALSE, "anonget1",0); 367 } else { 368 /* anon owns page */ 369 uvmfault_unlockall(ufi, amap, NULL, NULL); 370 UVMHIST_LOG(maphist, " unlock+wait on anon",0, 371 0,0,0); 372 UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0, 373 "anonget2",0); 374 } 375 /* ready to relock and try again */ 376 377 } else { 378 379 /* 380 * no page, we must try and bring it in. 381 */ 382 pg = uvm_pagealloc(NULL, 0, anon); 383 384 if (pg == NULL) { /* out of RAM. */ 385 386 uvmfault_unlockall(ufi, amap, NULL, anon); 387 uvmexp.fltnoram++; 388 UVMHIST_LOG(maphist, " noram -- UVM_WAIT",0, 389 0,0,0); 390 uvm_wait("flt_noram1"); 391 /* ready to relock and try again */ 392 393 } else { 394 395 /* we set the PG_BUSY bit */ 396 we_own = TRUE; 397 uvmfault_unlockall(ufi, amap, NULL, anon); 398 399 /* 400 * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN 401 * page into the uvm_swap_get function with 402 * all data structures unlocked. note that 403 * it is ok to read an_swslot here because 404 * we hold PG_BUSY on the page. 405 */ 406 uvmexp.pageins++; 407 result = uvm_swap_get(pg, anon->an_swslot, 408 PGO_SYNCIO); 409 410 /* 411 * we clean up after the i/o below in the 412 * "we_own" case 413 */ 414 /* ready to relock and try again */ 415 } 416 } 417 418 /* 419 * now relock and try again 420 */ 421 422 locked = uvmfault_relock(ufi); 423 if (locked) { 424 simple_lock(&amap->am_l); 425 } 426 if (locked || we_own) 427 simple_lock(&anon->an_lock); 428 429 /* 430 * if we own the page (i.e. we set PG_BUSY), then we need 431 * to clean up after the I/O. there are three cases to 432 * consider: 433 * [1] page released during I/O: free anon and ReFault. 434 * [2] I/O not OK. free the page and cause the fault 435 * to fail. 436 * [3] I/O OK! activate the page and sync with the 437 * non-we_own case (i.e. drop anon lock if not locked). 438 */ 439 440 if (we_own) { 441 442 if (pg->flags & PG_WANTED) { 443 /* still holding object lock */ 444 thread_wakeup(pg); 445 } 446 /* un-busy! */ 447 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE); 448 UVM_PAGE_OWN(pg, NULL); 449 450 /* 451 * if we were RELEASED during I/O, then our anon is 452 * no longer part of an amap. we need to free the 453 * anon and try again. 454 */ 455 if (pg->flags & PG_RELEASED) { 456 pmap_page_protect(PMAP_PGARG(pg), 457 VM_PROT_NONE); /* to be safe */ 458 simple_unlock(&anon->an_lock); 459 uvm_anfree(anon); /* frees page for us */ 460 if (locked) 461 uvmfault_unlockall(ufi, amap, NULL, NULL); 462 uvmexp.fltpgrele++; 463 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 464 return (VM_PAGER_REFAULT); /* refault! */ 465 } 466 467 if (result != VM_PAGER_OK) { 468 #ifdef DIAGNOSTIC 469 if (result == VM_PAGER_PEND) 470 panic("uvmfault_anonget: got PENDING for non-async I/O"); 471 #endif 472 /* remove page from anon */ 473 anon->u.an_page = NULL; 474 475 /* 476 * note: page was never !PG_BUSY, so it 477 * can't be mapped and thus no need to 478 * pmap_page_protect it... 479 */ 480 uvm_lock_pageq(); 481 uvm_pagefree(pg); 482 uvm_unlock_pageq(); 483 484 if (locked) 485 uvmfault_unlockall(ufi, amap, NULL, 486 anon); 487 else 488 simple_unlock(&anon->an_lock); 489 UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0); 490 return (VM_PAGER_ERROR); 491 } 492 493 /* 494 * must be OK, clear modify (already PG_CLEAN) 495 * and activate 496 */ 497 pmap_clear_modify(PMAP_PGARG(pg)); 498 uvm_lock_pageq(); 499 uvm_pageactivate(pg); 500 uvm_unlock_pageq(); 501 if (!locked) 502 simple_unlock(&anon->an_lock); 503 } 504 505 /* 506 * we were not able to relock. restart fault. 507 */ 508 509 if (!locked) { 510 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 511 return (VM_PAGER_REFAULT); 512 } 513 514 /* 515 * verify no one has touched the amap and moved the anon on us. 516 */ 517 518 if (amap_lookup(&ufi->entry->aref, 519 ufi->orig_rvaddr - ufi->entry->start) != anon) { 520 521 uvmfault_unlockall(ufi, amap, NULL, anon); 522 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 523 return (VM_PAGER_REFAULT); 524 } 525 526 /* 527 * try it again! 528 */ 529 530 uvmexp.fltanretry++; 531 continue; 532 533 } /* while (1) */ 534 535 /*NOTREACHED*/ 536 } 537 538 /* 539 * F A U L T - m a i n e n t r y p o i n t 540 */ 541 542 /* 543 * uvm_fault: page fault handler 544 * 545 * => called from MD code to resolve a page fault 546 * => VM data structures usually should be unlocked. however, it is 547 * possible to call here with the main map locked if the caller 548 * gets a write lock, sets it recusive, and then calls us (c.f. 549 * uvm_map_pageable). this should be avoided because it keeps 550 * the map locked off during I/O. 551 */ 552 553 int 554 uvm_fault(orig_map, vaddr, fault_type, access_type) 555 vm_map_t orig_map; 556 vaddr_t vaddr; 557 vm_fault_t fault_type; 558 vm_prot_t access_type; 559 { 560 struct uvm_faultinfo ufi; 561 vm_prot_t enter_prot; 562 boolean_t wired, narrow, promote, locked, shadowed; 563 int npages, nback, nforw, centeridx, result, lcv, gotpages; 564 vaddr_t startva, objaddr, currva, offset; 565 paddr_t pa; 566 struct vm_amap *amap; 567 struct uvm_object *uobj; 568 struct vm_anon *anons_store[UVM_MAXRANGE], **anons, *anon, *oanon; 569 struct vm_page *pages[UVM_MAXRANGE], *pg, *uobjpage; 570 UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist); 571 572 UVMHIST_LOG(maphist, "(map=0x%x, vaddr=0x%x, ft=%d, at=%d)", 573 orig_map, vaddr, fault_type, access_type); 574 575 anon = NULL; /* XXX: shut up gcc */ 576 577 uvmexp.faults++; /* XXX: locking? */ 578 579 /* 580 * init the IN parameters in the ufi 581 */ 582 583 ufi.orig_map = orig_map; 584 ufi.orig_rvaddr = trunc_page(vaddr); 585 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ 586 if (fault_type == VM_FAULT_WIRE) 587 narrow = TRUE; /* don't look for neighborhood 588 * pages on wire */ 589 else 590 narrow = FALSE; /* normal fault */ 591 592 /* 593 * "goto ReFault" means restart the page fault from ground zero. 594 */ 595 ReFault: 596 597 /* 598 * lookup and lock the maps 599 */ 600 601 if (uvmfault_lookup(&ufi, FALSE) == FALSE) { 602 UVMHIST_LOG(maphist, "<- no mapping @ 0x%x", vaddr, 0,0,0); 603 return (KERN_INVALID_ADDRESS); 604 } 605 /* locked: maps(read) */ 606 607 /* 608 * check protection 609 */ 610 611 if ((ufi.entry->protection & access_type) != access_type) { 612 UVMHIST_LOG(maphist, 613 "<- protection failure (prot=0x%x, access=0x%x)", 614 ufi.entry->protection, access_type, 0, 0); 615 uvmfault_unlockmaps(&ufi, FALSE); 616 return (KERN_PROTECTION_FAILURE); 617 } 618 619 /* 620 * "enter_prot" is the protection we want to enter the page in at. 621 * for certain pages (e.g. copy-on-write pages) this protection can 622 * be more strict than ufi.entry->protection. "wired" means either 623 * the entry is wired or we are fault-wiring the pg. 624 */ 625 626 enter_prot = ufi.entry->protection; 627 wired = (ufi.entry->wired_count != 0) || (fault_type == VM_FAULT_WIRE); 628 if (wired) 629 access_type = enter_prot; /* full access for wired */ 630 631 /* 632 * handle "needs_copy" case. if we need to copy the amap we will 633 * have to drop our readlock and relock it with a write lock. (we 634 * need a write lock to change anything in a map entry [e.g. 635 * needs_copy]). 636 */ 637 638 if (UVM_ET_ISNEEDSCOPY(ufi.entry)) { 639 if ((access_type & VM_PROT_WRITE) || 640 (ufi.entry->object.uvm_obj == NULL)) { 641 /* need to clear */ 642 UVMHIST_LOG(maphist, 643 " need to clear needs_copy and refault",0,0,0,0); 644 uvmfault_unlockmaps(&ufi, FALSE); 645 uvmfault_amapcopy(&ufi); 646 uvmexp.fltamcopy++; 647 goto ReFault; 648 649 } else { 650 651 /* 652 * ensure that we pmap_enter page R/O since 653 * needs_copy is still true 654 */ 655 enter_prot = enter_prot & ~VM_PROT_WRITE; 656 657 } 658 } 659 660 /* 661 * identify the players 662 */ 663 664 amap = ufi.entry->aref.ar_amap; /* top layer */ 665 uobj = ufi.entry->object.uvm_obj; /* bottom layer */ 666 667 /* 668 * check for a case 0 fault. if nothing backing the entry then 669 * error now. 670 */ 671 672 if (amap == NULL && uobj == NULL) { 673 uvmfault_unlockmaps(&ufi, FALSE); 674 UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0); 675 return (KERN_INVALID_ADDRESS); 676 } 677 678 /* 679 * establish range of interest based on advice from mapper 680 * and then clip to fit map entry. note that we only want 681 * to do this the first time through the fault. if we 682 * ReFault we will disable this by setting "narrow" to true. 683 */ 684 685 if (narrow == FALSE) { 686 687 /* wide fault (!narrow) */ 688 #ifdef DIAGNOSTIC 689 if (uvmadvice[ufi.entry->advice].advice != ufi.entry->advice) 690 panic("fault: advice mismatch!"); 691 #endif 692 nback = min(uvmadvice[ufi.entry->advice].nback, 693 (ufi.orig_rvaddr - ufi.entry->start) >> PAGE_SHIFT); 694 startva = ufi.orig_rvaddr - (nback << PAGE_SHIFT); 695 nforw = min(uvmadvice[ufi.entry->advice].nforw, 696 ((ufi.entry->end - ufi.orig_rvaddr) >> 697 PAGE_SHIFT) - 1); 698 /* 699 * note: "-1" because we don't want to count the 700 * faulting page as forw 701 */ 702 npages = nback + nforw + 1; 703 centeridx = nback; 704 705 narrow = FALSE; /* ensure only once per-fault */ 706 707 } else { 708 709 /* narrow fault! */ 710 nback = nforw = 0; 711 startva = ufi.orig_rvaddr; 712 npages = 1; 713 centeridx = 0; 714 715 } 716 717 /* locked: maps(read) */ 718 UVMHIST_LOG(maphist, " narrow=%d, back=%d, forw=%d, startva=0x%x", 719 narrow, nback, nforw, startva); 720 UVMHIST_LOG(maphist, " entry=0x%x, amap=0x%x, obj=0x%x", ufi.entry, 721 amap, uobj, 0); 722 723 /* 724 * if we've got an amap, lock it and extract current anons. 725 */ 726 727 if (amap) { 728 simple_lock(&amap->am_l); 729 anons = anons_store; 730 amap_lookups(&ufi.entry->aref, startva - ufi.entry->start, 731 anons, npages); 732 } else { 733 anons = NULL; /* to be safe */ 734 } 735 736 /* locked: maps(read), amap(if there) */ 737 738 /* 739 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages 740 * now and then forget about them (for the rest of the fault). 741 */ 742 743 if (ufi.entry->advice == MADV_SEQUENTIAL) { 744 745 UVMHIST_LOG(maphist, " MADV_SEQUENTIAL: flushing backpages", 746 0,0,0,0); 747 /* flush back-page anons? */ 748 if (amap) 749 uvmfault_anonflush(anons, nback); 750 751 /* flush object? */ 752 if (uobj) { 753 objaddr = 754 (startva - ufi.entry->start) + ufi.entry->offset; 755 simple_lock(&uobj->vmobjlock); 756 (void) uobj->pgops->pgo_flush(uobj, objaddr, objaddr + 757 (nback << PAGE_SHIFT), PGO_DEACTIVATE); 758 simple_unlock(&uobj->vmobjlock); 759 } 760 761 /* now forget about the backpages */ 762 if (amap) 763 anons += nback; 764 startva = startva + (nback << PAGE_SHIFT); 765 npages -= nback; 766 nback = centeridx = 0; 767 } 768 769 /* locked: maps(read), amap(if there) */ 770 771 /* 772 * map in the backpages and frontpages we found in the amap in hopes 773 * of preventing future faults. we also init the pages[] array as 774 * we go. 775 */ 776 777 currva = startva; 778 shadowed = FALSE; 779 for (lcv = 0 ; lcv < npages ; lcv++, currva += PAGE_SIZE) { 780 781 /* 782 * dont play with VAs that are already mapped 783 * except for center) 784 * XXX: return value of pmap_extract disallows PA 0 785 */ 786 if (lcv != centeridx) { 787 pa = pmap_extract(ufi.orig_map->pmap, currva); 788 if (pa != NULL) { 789 pages[lcv] = PGO_DONTCARE; 790 continue; 791 } 792 } 793 794 /* 795 * unmapped or center page. check if any anon at this level. 796 */ 797 if (amap == NULL || anons[lcv] == NULL) { 798 pages[lcv] = NULL; 799 continue; 800 } 801 802 /* 803 * check for present page and map if possible. re-activate it. 804 */ 805 806 pages[lcv] = PGO_DONTCARE; 807 if (lcv == centeridx) { /* save center for later! */ 808 shadowed = TRUE; 809 continue; 810 } 811 anon = anons[lcv]; 812 simple_lock(&anon->an_lock); 813 /* ignore loaned pages */ 814 if (anon->u.an_page && anon->u.an_page->loan_count == 0 && 815 (anon->u.an_page->flags & (PG_RELEASED|PG_BUSY)) == 0) { 816 uvm_lock_pageq(); 817 uvm_pageactivate(anon->u.an_page); /* reactivate */ 818 uvm_unlock_pageq(); 819 UVMHIST_LOG(maphist, 820 " MAPPING: n anon: pm=0x%x, va=0x%x, pg=0x%x", 821 ufi.orig_map->pmap, currva, anon->u.an_page, 0); 822 uvmexp.fltnamap++; 823 pmap_enter(ufi.orig_map->pmap, currva, 824 VM_PAGE_TO_PHYS(anon->u.an_page), 825 (anon->an_ref > 1) ? VM_PROT_READ : enter_prot, 826 (ufi.entry->wired_count != 0)); 827 } 828 simple_unlock(&anon->an_lock); 829 } 830 831 /* locked: maps(read), amap(if there) */ 832 /* (shadowed == TRUE) if there is an anon at the faulting address */ 833 UVMHIST_LOG(maphist, " shadowed=%d, will_get=%d", shadowed, 834 (uobj && shadowed == FALSE),0,0); 835 836 /* 837 * note that if we are really short of RAM we could sleep in the above 838 * call to pmap_enter with everything locked. bad? 839 * XXXCDC: this is fixed in PMAP_NEW (no sleep alloc's in pmap) 840 */ 841 842 /* 843 * if the desired page is not shadowed by the amap and we have a 844 * backing object, then we check to see if the backing object would 845 * prefer to handle the fault itself (rather than letting us do it 846 * with the usual pgo_get hook). the backing object signals this by 847 * providing a pgo_fault routine. 848 */ 849 850 if (uobj && shadowed == FALSE && uobj->pgops->pgo_fault != NULL) { 851 852 simple_lock(&uobj->vmobjlock); 853 854 /* locked: maps(read), amap (if there), uobj */ 855 result = uobj->pgops->pgo_fault(&ufi, startva, pages, npages, 856 centeridx, fault_type, access_type, 857 PGO_LOCKED); 858 /* locked: nothing, pgo_fault has unlocked everything */ 859 860 if (result == VM_PAGER_OK) 861 return (KERN_SUCCESS); /* pgo_fault did pmap enter */ 862 else if (result == VM_PAGER_REFAULT) 863 goto ReFault; /* try again! */ 864 else 865 return (KERN_PROTECTION_FAILURE); 866 } 867 868 /* 869 * now, if the desired page is not shadowed by the amap and we have 870 * a backing object that does not have a special fault routine, then 871 * we ask (with pgo_get) the object for resident pages that we care 872 * about and attempt to map them in. we do not let pgo_get block 873 * (PGO_LOCKED). 874 * 875 * ("get" has the option of doing a pmap_enter for us) 876 */ 877 878 if (uobj && shadowed == FALSE) { 879 simple_lock(&uobj->vmobjlock); 880 881 /* locked (!shadowed): maps(read), amap (if there), uobj */ 882 /* 883 * the following call to pgo_get does _not_ change locking state 884 */ 885 886 uvmexp.fltlget++; 887 gotpages = npages; 888 result = uobj->pgops->pgo_get(uobj, ufi.entry->offset + 889 (startva - ufi.entry->start), 890 pages, &gotpages, centeridx, 891 UVM_ET_ISCOPYONWRITE(ufi.entry) ? 892 VM_PROT_READ : access_type, 893 ufi.entry->advice, PGO_LOCKED); 894 895 /* 896 * check for pages to map, if we got any 897 */ 898 899 uobjpage = NULL; 900 901 if (gotpages) { 902 currva = startva; 903 for (lcv = 0 ; lcv < npages ; 904 lcv++, currva += PAGE_SIZE) { 905 906 if (pages[lcv] == NULL || 907 pages[lcv] == PGO_DONTCARE) 908 continue; 909 910 #ifdef DIAGNOSTIC 911 /* 912 * pager sanity check: pgo_get with 913 * PGO_LOCKED should never return a 914 * released page to us. 915 */ 916 if (pages[lcv]->flags & PG_RELEASED) 917 panic("uvm_fault: pgo_get PGO_LOCKED gave us a RELEASED page"); 918 #endif 919 920 /* 921 * if center page is resident and not 922 * PG_BUSY|PG_RELEASED then pgo_get 923 * made it PG_BUSY for us and gave 924 * us a handle to it. remember this 925 * page as "uobjpage." (for later use). 926 */ 927 928 if (lcv == centeridx) { 929 uobjpage = pages[lcv]; 930 UVMHIST_LOG(maphist, " got uobjpage (0x%x) with locked get", 931 uobjpage, 0,0,0); 932 continue; 933 } 934 935 /* 936 * note: calling pgo_get with locked data 937 * structures returns us pages which are 938 * neither busy nor released, so we don't 939 * need to check for this. we can just 940 * directly enter the page (after moving it 941 * to the head of the active queue [useful?]). 942 */ 943 944 uvm_lock_pageq(); 945 uvm_pageactivate(pages[lcv]); /* reactivate */ 946 uvm_unlock_pageq(); 947 UVMHIST_LOG(maphist, 948 " MAPPING: n obj: pm=0x%x, va=0x%x, pg=0x%x", 949 ufi.orig_map->pmap, currva, pages[lcv], 0); 950 uvmexp.fltnomap++; 951 pmap_enter(ufi.orig_map->pmap, currva, 952 VM_PAGE_TO_PHYS(pages[lcv]), 953 UVM_ET_ISCOPYONWRITE(ufi.entry) ? 954 VM_PROT_READ : enter_prot, wired); 955 956 /* 957 * NOTE: page can't be PG_WANTED or PG_RELEASED 958 * because we've held the lock the whole time 959 * we've had the handle. 960 */ 961 pages[lcv]->flags &= ~(PG_BUSY); /* un-busy! */ 962 UVM_PAGE_OWN(pages[lcv], NULL); 963 964 /* done! */ 965 } /* for "lcv" loop */ 966 } /* "gotpages" != 0 */ 967 968 /* note: object still _locked_ */ 969 } else { 970 971 uobjpage = NULL; 972 973 } 974 975 /* locked (shadowed): maps(read), amap */ 976 /* locked (!shadowed): maps(read), amap(if there), 977 uobj(if !null), uobjpage(if !null) */ 978 979 /* 980 * note that at this point we are done with any front or back pages. 981 * we are now going to focus on the center page (i.e. the one we've 982 * faulted on). if we have faulted on the top (anon) layer 983 * [i.e. case 1], then the anon we want is anons[centeridx] (we have 984 * not touched it yet). if we have faulted on the bottom (uobj) 985 * layer [i.e. case 2] and the page was both present and available, 986 * then we've got a pointer to it as "uobjpage" and we've already 987 * made it BUSY. 988 */ 989 990 /* 991 * there are four possible cases we must address: 1A, 1B, 2A, and 2B 992 */ 993 994 /* 995 * redirect case 2: if we are not shadowed, go to case 2. 996 */ 997 998 if (shadowed == FALSE) 999 goto Case2; 1000 1001 /* locked: maps(read), amap */ 1002 1003 /* 1004 * handle case 1: fault on an anon in our amap 1005 */ 1006 1007 anon = anons[centeridx]; 1008 UVMHIST_LOG(maphist, " case 1 fault: anon=0x%x", anon, 0,0,0); 1009 simple_lock(&anon->an_lock); 1010 1011 /* locked: maps(read), amap, anon */ 1012 1013 /* 1014 * no matter if we have case 1A or case 1B we are going to need to 1015 * have the anon's memory resident. ensure that now. 1016 */ 1017 1018 /* 1019 * let uvmfault_anonget do the dirty work. if it fails (!OK) it will 1020 * unlock for us. if it is OK, locks are still valid and locked. 1021 * also, if it is OK, then the anon's page is on the queues. 1022 * if the page is on loan from a uvm_object, then anonget will 1023 * lock that object for us if it does not fail. 1024 */ 1025 1026 result = uvmfault_anonget(&ufi, amap, anon); 1027 1028 if (result == VM_PAGER_REFAULT) 1029 goto ReFault; 1030 1031 if (result == VM_PAGER_AGAIN) { 1032 tsleep((caddr_t)&lbolt, PVM, "fltagain1", 0); 1033 goto ReFault; 1034 } 1035 1036 if (result != VM_PAGER_OK) 1037 return (KERN_PROTECTION_FAILURE); /* XXX??? */ 1038 1039 /* 1040 * uobj is non null if the page is on loan from an object (i.e. uobj) 1041 */ 1042 1043 uobj = anon->u.an_page->uobject; /* locked by anonget if !NULL */ 1044 1045 /* locked: maps(read), amap, anon, uobj(if one) */ 1046 1047 /* 1048 * special handling for loaned pages 1049 */ 1050 if (anon->u.an_page->loan_count) { 1051 1052 if ((access_type & VM_PROT_WRITE) == 0) { 1053 1054 /* 1055 * for read faults on loaned pages we just cap the 1056 * protection at read-only. 1057 */ 1058 1059 enter_prot = enter_prot & ~VM_PROT_WRITE; 1060 1061 } else { 1062 /* 1063 * note that we can't allow writes into a loaned page! 1064 * 1065 * if we have a write fault on a loaned page in an 1066 * anon then we need to look at the anon's ref count. 1067 * if it is greater than one then we are going to do 1068 * a normal copy-on-write fault into a new anon (this 1069 * is not a problem). however, if the reference count 1070 * is one (a case where we would normally allow a 1071 * write directly to the page) then we need to kill 1072 * the loan before we continue. 1073 */ 1074 1075 /* >1 case is already ok */ 1076 if (anon->an_ref == 1) { 1077 1078 /* get new un-owned replacement page */ 1079 pg = uvm_pagealloc(NULL, 0, NULL); 1080 if (pg == NULL) { 1081 uvmfault_unlockall(&ufi, amap, uobj, 1082 anon); 1083 uvm_wait("flt_noram2"); 1084 goto ReFault; 1085 } 1086 1087 /* 1088 * copy data, kill loan, and drop uobj lock 1089 * (if any) 1090 */ 1091 /* copy old -> new */ 1092 uvm_pagecopy(anon->u.an_page, pg); 1093 1094 /* force reload */ 1095 pmap_page_protect(PMAP_PGARG(anon->u.an_page), 1096 VM_PROT_NONE); 1097 uvm_lock_pageq(); /* KILL loan */ 1098 if (uobj) 1099 /* if we were loaning */ 1100 anon->u.an_page->loan_count--; 1101 anon->u.an_page->uanon = NULL; 1102 /* in case we owned */ 1103 anon->u.an_page->pqflags &= ~PQ_ANON; 1104 uvm_unlock_pageq(); 1105 if (uobj) { 1106 simple_unlock(&uobj->vmobjlock); 1107 uobj = NULL; 1108 } 1109 1110 /* install new page in anon */ 1111 anon->u.an_page = pg; 1112 pg->uanon = anon; 1113 pg->pqflags |= PQ_ANON; 1114 pg->flags &= ~(PG_BUSY|PG_FAKE); 1115 UVM_PAGE_OWN(pg, NULL); 1116 1117 /* done! */ 1118 } /* ref == 1 */ 1119 } /* write fault */ 1120 } /* loan count */ 1121 1122 /* 1123 * if we are case 1B then we will need to allocate a new blank 1124 * anon to transfer the data into. note that we have a lock 1125 * on anon, so no one can busy or release the page until we are done. 1126 * also note that the ref count can't drop to zero here because 1127 * it is > 1 and we are only dropping one ref. 1128 * 1129 * in the (hopefully very rare) case that we are out of RAM we 1130 * will unlock, wait for more RAM, and refault. 1131 * 1132 * if we are out of anon VM we kill the process (XXX: could wait?). 1133 */ 1134 1135 if ((access_type & VM_PROT_WRITE) != 0 && anon->an_ref > 1) { 1136 1137 UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0); 1138 uvmexp.flt_acow++; 1139 oanon = anon; /* oanon = old, locked anon */ 1140 anon = uvm_analloc(); 1141 if (anon) 1142 pg = uvm_pagealloc(NULL, 0, anon); 1143 #ifdef __GNUC__ 1144 else 1145 pg = NULL; /* XXX: gcc */ 1146 #endif 1147 1148 /* check for out of RAM */ 1149 if (anon == NULL || pg == NULL) { 1150 if (anon) 1151 uvm_anfree(anon); 1152 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1153 if (anon == NULL) { 1154 UVMHIST_LOG(maphist, 1155 "<- failed. out of VM",0,0,0,0); 1156 uvmexp.fltnoanon++; 1157 /* XXX: OUT OF VM, ??? */ 1158 return (KERN_RESOURCE_SHORTAGE); 1159 } 1160 uvmexp.fltnoram++; 1161 uvm_wait("flt_noram3"); /* out of RAM, wait for more */ 1162 goto ReFault; 1163 } 1164 1165 /* got all resources, replace anon with nanon */ 1166 1167 uvm_pagecopy(oanon->u.an_page, pg); /* pg now !PG_CLEAN */ 1168 pg->flags &= ~(PG_BUSY|PG_FAKE); /* un-busy! new page */ 1169 UVM_PAGE_OWN(pg, NULL); 1170 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start, 1171 anon, 1); 1172 1173 /* deref: can not drop to zero here by defn! */ 1174 oanon->an_ref--; 1175 1176 /* 1177 * note: oanon still locked. anon is _not_ locked, but we 1178 * have the sole references to in from amap which _is_ locked. 1179 * thus, no one can get at it until we are done with it. 1180 */ 1181 1182 } else { 1183 1184 uvmexp.flt_anon++; 1185 oanon = anon; /* old, locked anon is same as anon */ 1186 pg = anon->u.an_page; 1187 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */ 1188 enter_prot = enter_prot & ~VM_PROT_WRITE; 1189 1190 } 1191 1192 /* locked: maps(read), amap, anon */ 1193 1194 /* 1195 * now map the page in ... 1196 * XXX: old fault unlocks object before pmap_enter. this seems 1197 * suspect since some other thread could blast the page out from 1198 * under us between the unlock and the pmap_enter. 1199 */ 1200 1201 UVMHIST_LOG(maphist, " MAPPING: anon: pm=0x%x, va=0x%x, pg=0x%x", 1202 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, 0); 1203 pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1204 enter_prot, wired); 1205 1206 /* 1207 * ... and update the page queues. 1208 */ 1209 1210 uvm_lock_pageq(); 1211 1212 if (fault_type == VM_FAULT_WIRE) { 1213 uvm_pagewire(pg); 1214 } else { 1215 /* activate it */ 1216 uvm_pageactivate(pg); 1217 1218 } 1219 1220 uvm_unlock_pageq(); 1221 1222 /* 1223 * done case 1! finish up by unlocking everything and returning success 1224 */ 1225 1226 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1227 return (KERN_SUCCESS); 1228 1229 1230 Case2: 1231 /* 1232 * handle case 2: faulting on backing object or zero fill 1233 */ 1234 1235 /* 1236 * locked: 1237 * maps(read), amap(if there), uobj(if !null), uobjpage(if !null) 1238 */ 1239 1240 /* 1241 * note that uobjpage can not be PGO_DONTCARE at this point. we now 1242 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we 1243 * have a backing object, check and see if we are going to promote 1244 * the data up to an anon during the fault. 1245 */ 1246 1247 if (uobj == NULL) { 1248 uobjpage = PGO_DONTCARE; 1249 promote = TRUE; /* always need anon here */ 1250 } else { 1251 /* assert(uobjpage != PGO_DONTCARE) */ 1252 promote = (access_type & VM_PROT_WRITE) && 1253 UVM_ET_ISCOPYONWRITE(ufi.entry); 1254 } 1255 UVMHIST_LOG(maphist, " case 2 fault: promote=%d, zfill=%d", 1256 promote, (uobj == NULL), 0,0); 1257 1258 /* 1259 * if uobjpage is not null then we do not need to do I/O to get the 1260 * uobjpage. 1261 * 1262 * if uobjpage is null, then we need to unlock and ask the pager to 1263 * get the data for us. once we have the data, we need to reverify 1264 * the state the world. we are currently not holding any resources. 1265 */ 1266 1267 if (uobjpage) { 1268 /* update rusage counters */ 1269 curproc->p_addr->u_stats.p_ru.ru_minflt++; 1270 } else { 1271 /* update rusage counters */ 1272 curproc->p_addr->u_stats.p_ru.ru_majflt++; 1273 1274 /* locked: maps(read), amap(if there), uobj */ 1275 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1276 /* locked: uobj */ 1277 1278 uvmexp.fltget++; 1279 gotpages = 1; 1280 result = uobj->pgops->pgo_get(uobj, 1281 (ufi.orig_rvaddr - ufi.entry->start) + ufi.entry->offset, 1282 &uobjpage, &gotpages, 0, 1283 UVM_ET_ISCOPYONWRITE(ufi.entry) ? 1284 VM_PROT_READ : access_type, 1285 ufi.entry->advice, 0); 1286 1287 /* locked: uobjpage(if result OK) */ 1288 1289 /* 1290 * recover from I/O 1291 */ 1292 1293 if (result != VM_PAGER_OK) { 1294 1295 #ifdef DIAGNOSTIC 1296 if (result == VM_PAGER_PEND) 1297 panic("uvm_fault: pgo_get got PENDing on non-async I/O"); 1298 #endif 1299 1300 if (result == VM_PAGER_AGAIN) { 1301 UVMHIST_LOG(maphist, " pgo_get says TRY AGAIN!",0,0,0,0); 1302 tsleep((caddr_t)&lbolt, PVM, "fltagain2", 0); 1303 goto ReFault; 1304 } 1305 1306 UVMHIST_LOG(maphist, "<- pgo_get failed (code %d)", 1307 result, 0,0,0); 1308 return (KERN_PROTECTION_FAILURE); /* XXX i/o error */ 1309 } 1310 1311 /* locked: uobjpage */ 1312 1313 /* 1314 * re-verify the state of the world by first trying to relock 1315 * the maps. always relock the object. 1316 */ 1317 1318 locked = uvmfault_relock(&ufi); 1319 if (locked && amap) 1320 simple_lock(&amap->am_l); 1321 simple_lock(&uobj->vmobjlock); 1322 1323 /* locked(locked): maps(read), amap(if !null), uobj, uobjpage */ 1324 /* locked(!locked): uobj, uobjpage */ 1325 1326 /* 1327 * verify that the page has not be released and re-verify 1328 * that amap slot is still free. if there is a problem, 1329 * we unlock and clean up. 1330 */ 1331 1332 if ((uobjpage->flags & PG_RELEASED) != 0 || 1333 (locked && amap && 1334 amap_lookup(&ufi.entry->aref, 1335 ufi.orig_rvaddr - ufi.entry->start))) { 1336 if (locked) 1337 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1338 locked = FALSE; 1339 } 1340 1341 /* 1342 * didn't get the lock? release the page and retry. 1343 */ 1344 1345 if (locked == FALSE) { 1346 1347 UVMHIST_LOG(maphist, 1348 " wasn't able to relock after fault: retry", 1349 0,0,0,0); 1350 if (uobjpage->flags & PG_WANTED) 1351 /* still holding object lock */ 1352 thread_wakeup(uobjpage); 1353 1354 if (uobjpage->flags & PG_RELEASED) { 1355 uvmexp.fltpgrele++; 1356 #ifdef DIAGNOSTIC 1357 if (uobj->pgops->pgo_releasepg == NULL) 1358 panic("uvm_fault: object has no releasepg function"); 1359 #endif 1360 /* frees page */ 1361 if (uobj->pgops->pgo_releasepg(uobjpage,NULL)) 1362 /* unlock if still alive */ 1363 simple_unlock(&uobj->vmobjlock); 1364 goto ReFault; 1365 } 1366 1367 uvm_lock_pageq(); 1368 /* make sure it is in queues */ 1369 uvm_pageactivate(uobjpage); 1370 1371 uvm_unlock_pageq(); 1372 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1373 UVM_PAGE_OWN(uobjpage, NULL); 1374 simple_unlock(&uobj->vmobjlock); 1375 goto ReFault; 1376 1377 } 1378 1379 /* 1380 * we have the data in uobjpage which is PG_BUSY and 1381 * !PG_RELEASED. we are holding object lock (so the page 1382 * can't be released on us). 1383 */ 1384 1385 /* locked: maps(read), amap(if !null), uobj, uobjpage */ 1386 1387 } 1388 1389 /* 1390 * locked: 1391 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj) 1392 */ 1393 1394 /* 1395 * notes: 1396 * - at this point uobjpage can not be NULL 1397 * - at this point uobjpage can not be PG_RELEASED (since we checked 1398 * for it above) 1399 * - at this point uobjpage could be PG_WANTED (handle later) 1400 */ 1401 1402 if (promote == FALSE) { 1403 1404 /* 1405 * we are not promoting. if the mapping is COW ensure that we 1406 * don't give more access than we should (e.g. when doing a read 1407 * fault on a COPYONWRITE mapping we want to map the COW page in 1408 * R/O even though the entry protection could be R/W). 1409 * 1410 * set "pg" to the page we want to map in (uobjpage, usually) 1411 */ 1412 1413 uvmexp.flt_obj++; 1414 if (UVM_ET_ISCOPYONWRITE(ufi.entry)) 1415 enter_prot = enter_prot & ~VM_PROT_WRITE; 1416 pg = uobjpage; /* map in the actual object */ 1417 1418 /* assert(uobjpage != PGO_DONTCARE) */ 1419 1420 /* 1421 * we are faulting directly on the page. be careful 1422 * about writing to loaned pages... 1423 */ 1424 if (uobjpage->loan_count) { 1425 1426 if ((access_type & VM_PROT_WRITE) == 0) { 1427 /* read fault: cap the protection at readonly */ 1428 /* cap! */ 1429 enter_prot = enter_prot & ~VM_PROT_WRITE; 1430 } else { 1431 /* write fault: must break the loan here */ 1432 1433 /* alloc new un-owned page */ 1434 pg = uvm_pagealloc(NULL, 0, NULL); 1435 1436 if (pg == NULL) { 1437 /* 1438 * drop ownership of page, it can't 1439 * be released 1440 * */ 1441 if (uobjpage->flags & PG_WANTED) 1442 thread_wakeup(uobjpage); 1443 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1444 UVM_PAGE_OWN(uobjpage, NULL); 1445 1446 uvm_lock_pageq(); 1447 /* activate: we will need it later */ 1448 uvm_pageactivate(uobjpage); 1449 1450 uvm_unlock_pageq(); 1451 uvmfault_unlockall(&ufi, amap, uobj, 1452 NULL); 1453 UVMHIST_LOG(maphist, 1454 " out of RAM breaking loan, waiting", 0,0,0,0); 1455 uvmexp.fltnoram++; 1456 uvm_wait("flt_noram4"); 1457 goto ReFault; 1458 } 1459 1460 /* 1461 * copy the data from the old page to the new 1462 * one and clear the fake/clean flags on the 1463 * new page (keep it busy). force a reload 1464 * of the old page by clearing it from all 1465 * pmaps. then lock the page queues to 1466 * rename the pages. 1467 */ 1468 uvm_pagecopy(uobjpage, pg); /* old -> new */ 1469 pg->flags &= ~(PG_FAKE|PG_CLEAN); 1470 pmap_page_protect(PMAP_PGARG(uobjpage), 1471 VM_PROT_NONE); 1472 if (uobjpage->flags & PG_WANTED) 1473 thread_wakeup(uobjpage); 1474 /* uobj still locked */ 1475 uobjpage->flags &= ~(PG_WANTED|PG_BUSY); 1476 UVM_PAGE_OWN(uobjpage, NULL); 1477 1478 uvm_lock_pageq(); 1479 offset = uobjpage->offset; 1480 /* remove old page */ 1481 uvm_pagerealloc(uobjpage, NULL, 0); 1482 1483 /* 1484 * at this point we have absolutely no 1485 * control over uobjpage 1486 */ 1487 /* install new page */ 1488 uvm_pagerealloc(pg, uobj, offset); 1489 uvm_unlock_pageq(); 1490 1491 /* 1492 * done! loan is broken and "pg" is 1493 * PG_BUSY. it can now replace uobjpage. 1494 */ 1495 1496 uobjpage = pg; 1497 1498 } /* write fault case */ 1499 } /* if loan_count */ 1500 1501 } else { 1502 1503 /* 1504 * if we are going to promote the data to an anon we 1505 * allocate a blank anon here and plug it into our amap. 1506 */ 1507 #if DIAGNOSTIC 1508 if (amap == NULL) 1509 panic("uvm_fault: want to promote data, but no anon"); 1510 #endif 1511 1512 anon = uvm_analloc(); 1513 if (anon) 1514 pg = uvm_pagealloc(NULL, 0, anon); /* BUSY+CLEAN+FAKE */ 1515 #ifdef __GNUC__ 1516 else 1517 pg = NULL; /* XXX: gcc */ 1518 #endif 1519 1520 /* 1521 * out of memory resources? 1522 */ 1523 if (anon == NULL || pg == NULL) { 1524 1525 /* 1526 * arg! must unbusy our page and fail or sleep. 1527 */ 1528 if (uobjpage != PGO_DONTCARE) { 1529 if (uobjpage->flags & PG_WANTED) 1530 /* still holding object lock */ 1531 thread_wakeup(uobjpage); 1532 1533 uvm_lock_pageq(); 1534 /* make sure it is in queues */ 1535 uvm_pageactivate(uobjpage); 1536 uvm_unlock_pageq(); 1537 /* un-busy! (still locked) */ 1538 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1539 UVM_PAGE_OWN(uobjpage, NULL); 1540 } 1541 1542 /* unlock and fail ... */ 1543 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1544 if (anon == NULL) { 1545 UVMHIST_LOG(maphist, " promote: out of VM", 1546 0,0,0,0); 1547 uvmexp.fltnoanon++; 1548 /* XXX: out of VM */ 1549 return (KERN_RESOURCE_SHORTAGE); 1550 } 1551 UVMHIST_LOG(maphist, " out of RAM, waiting for more", 1552 0,0,0,0); 1553 uvm_anfree(anon); 1554 uvmexp.fltnoram++; 1555 uvm_wait("flt_noram5"); 1556 goto ReFault; 1557 } 1558 1559 /* 1560 * fill in the data 1561 */ 1562 1563 if (uobjpage != PGO_DONTCARE) { 1564 uvmexp.flt_prcopy++; 1565 /* copy page [pg now dirty] */ 1566 uvm_pagecopy(uobjpage, pg); 1567 1568 /* 1569 * promote to shared amap? make sure all sharing 1570 * procs see it 1571 */ 1572 if ((amap->am_flags & AMAP_SHARED) != 0) { 1573 pmap_page_protect(PMAP_PGARG(uobjpage), 1574 VM_PROT_NONE); 1575 } 1576 1577 /* 1578 * dispose of uobjpage. it can't be PG_RELEASED 1579 * since we still hold the object lock. drop 1580 * handle to uobj as well. 1581 */ 1582 1583 if (uobjpage->flags & PG_WANTED) 1584 /* still have the obj lock */ 1585 thread_wakeup(uobjpage); 1586 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1587 UVM_PAGE_OWN(uobjpage, NULL); 1588 uvm_lock_pageq(); 1589 uvm_pageactivate(uobjpage); /* put it back */ 1590 uvm_unlock_pageq(); 1591 simple_unlock(&uobj->vmobjlock); 1592 uobj = NULL; 1593 UVMHIST_LOG(maphist, 1594 " promote uobjpage 0x%x to anon/page 0x%x/0x%x", 1595 uobjpage, anon, pg, 0); 1596 1597 } else { 1598 uvmexp.flt_przero++; 1599 uvm_pagezero(pg); /* zero page [pg now dirty] */ 1600 UVMHIST_LOG(maphist," zero fill anon/page 0x%x/0%x", 1601 anon, pg, 0, 0); 1602 } 1603 1604 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start, 1605 anon, 0); 1606 1607 } 1608 1609 /* 1610 * locked: 1611 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj) 1612 * 1613 * note: pg is either the uobjpage or the new page in the new anon 1614 */ 1615 1616 /* 1617 * all resources are present. we can now map it in and free our 1618 * resources. 1619 */ 1620 1621 UVMHIST_LOG(maphist, 1622 " MAPPING: case2: pm=0x%x, va=0x%x, pg=0x%x, promote=%d", 1623 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, promote); 1624 pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1625 enter_prot, wired); 1626 1627 uvm_lock_pageq(); 1628 1629 if (fault_type == VM_FAULT_WIRE) { 1630 uvm_pagewire(pg); 1631 } else { 1632 1633 /* activate it */ 1634 uvm_pageactivate(pg); 1635 1636 } 1637 1638 uvm_unlock_pageq(); 1639 1640 if (pg->flags & PG_WANTED) 1641 thread_wakeup(pg); /* lock still held */ 1642 1643 /* 1644 * note that pg can't be PG_RELEASED since we did not drop the object 1645 * lock since the last time we checked. 1646 */ 1647 1648 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); 1649 UVM_PAGE_OWN(pg, NULL); 1650 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1651 1652 UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0); 1653 return (KERN_SUCCESS); 1654 } 1655 1656 1657 /* 1658 * uvm_fault_wire: wire down a range of virtual addresses in a map. 1659 * 1660 * => map should be locked by caller? If so how can we call 1661 * uvm_fault? WRONG. 1662 * => XXXCDC: locking here is all screwed up!!! start with 1663 * uvm_map_pageable and fix it. 1664 */ 1665 1666 int 1667 uvm_fault_wire(map, start, end) 1668 vm_map_t map; 1669 vaddr_t start, end; 1670 { 1671 vaddr_t va; 1672 pmap_t pmap; 1673 int rv; 1674 1675 pmap = vm_map_pmap(map); 1676 1677 /* 1678 * call pmap pageable: this tells the pmap layer to lock down these 1679 * page tables. 1680 */ 1681 1682 pmap_pageable(pmap, start, end, FALSE); 1683 1684 /* 1685 * now fault it in page at a time. if the fault fails then we have 1686 * to undo what we have done. note that in uvm_fault VM_PROT_NONE 1687 * is replaced with the max protection if fault_type is VM_FAULT_WIRE. 1688 */ 1689 1690 for (va = start ; va < end ; va += PAGE_SIZE) { 1691 rv = uvm_fault(map, va, VM_FAULT_WIRE, VM_PROT_NONE); 1692 if (rv) { 1693 if (va != start) { 1694 uvm_fault_unwire(map->pmap, start, va); 1695 } 1696 return (rv); 1697 } 1698 } 1699 1700 return (KERN_SUCCESS); 1701 } 1702 1703 /* 1704 * uvm_fault_unwire(): unwire range of virtual space. 1705 * 1706 * => caller holds reference to pmap (via its map) 1707 */ 1708 1709 void 1710 uvm_fault_unwire(pmap, start, end) 1711 struct pmap *pmap; 1712 vaddr_t start, end; 1713 { 1714 vaddr_t va; 1715 paddr_t pa; 1716 struct vm_page *pg; 1717 1718 /* 1719 * we assume that the area we are unwiring has actually been wired 1720 * in the first place. this means that we should be able to extract 1721 * the PAs from the pmap. we also lock out the page daemon so that 1722 * we can call uvm_pageunwire. 1723 */ 1724 1725 uvm_lock_pageq(); 1726 1727 for (va = start; va < end ; va += PAGE_SIZE) { 1728 pa = pmap_extract(pmap, va); 1729 1730 /* XXX: assumes PA 0 cannot be in map */ 1731 if (pa == (paddr_t) 0) { 1732 panic("uvm_fault_unwire: unwiring non-wired memory"); 1733 } 1734 pmap_change_wiring(pmap, va, FALSE); /* tell the pmap */ 1735 pg = PHYS_TO_VM_PAGE(pa); 1736 if (pg) 1737 uvm_pageunwire(pg); 1738 } 1739 1740 uvm_unlock_pageq(); 1741 1742 /* 1743 * now we call pmap_pageable to let the pmap know that the page tables 1744 * in this space no longer need to be wired. 1745 */ 1746 1747 pmap_pageable(pmap, start, end, TRUE); 1748 1749 } 1750