1 /* $OpenBSD: uvm_fault.c,v 1.41 2006/07/31 11:51:29 mickey Exp $ */ 2 /* $NetBSD: uvm_fault.c,v 1.51 2000/08/06 00:22:53 thorpej Exp $ */ 3 4 /* 5 * 6 * Copyright (c) 1997 Charles D. Cranor and Washington University. 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed by Charles D. Cranor and 20 * Washington University. 21 * 4. The name of the author may not be used to endorse or promote products 22 * derived from this software without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 25 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 26 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 27 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 28 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 29 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 30 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 31 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 32 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 33 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 34 * 35 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp 36 */ 37 38 /* 39 * uvm_fault.c: fault handler 40 */ 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/kernel.h> 45 #include <sys/proc.h> 46 #include <sys/malloc.h> 47 #include <sys/mman.h> 48 #include <sys/user.h> 49 50 #include <uvm/uvm.h> 51 52 /* 53 * 54 * a word on page faults: 55 * 56 * types of page faults we handle: 57 * 58 * CASE 1: upper layer faults CASE 2: lower layer faults 59 * 60 * CASE 1A CASE 1B CASE 2A CASE 2B 61 * read/write1 write>1 read/write +-cow_write/zero 62 * | | | | 63 * +--|--+ +--|--+ +-----+ + | + | +-----+ 64 * amap | V | | ----------->new| | | | ^ | 65 * +-----+ +-----+ +-----+ + | + | +--|--+ 66 * | | | 67 * +-----+ +-----+ +--|--+ | +--|--+ 68 * uobj | d/c | | d/c | | V | +----| | 69 * +-----+ +-----+ +-----+ +-----+ 70 * 71 * d/c = don't care 72 * 73 * case [0]: layerless fault 74 * no amap or uobj is present. this is an error. 75 * 76 * case [1]: upper layer fault [anon active] 77 * 1A: [read] or [write with anon->an_ref == 1] 78 * I/O takes place in top level anon and uobj is not touched. 79 * 1B: [write with anon->an_ref > 1] 80 * new anon is alloc'd and data is copied off ["COW"] 81 * 82 * case [2]: lower layer fault [uobj] 83 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area] 84 * I/O takes place directly in object. 85 * 2B: [write to copy_on_write] or [read on NULL uobj] 86 * data is "promoted" from uobj to a new anon. 87 * if uobj is null, then we zero fill. 88 * 89 * we follow the standard UVM locking protocol ordering: 90 * 91 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ) 92 * we hold a PG_BUSY page if we unlock for I/O 93 * 94 * 95 * the code is structured as follows: 96 * 97 * - init the "IN" params in the ufi structure 98 * ReFault: 99 * - do lookups [locks maps], check protection, handle needs_copy 100 * - check for case 0 fault (error) 101 * - establish "range" of fault 102 * - if we have an amap lock it and extract the anons 103 * - if sequential advice deactivate pages behind us 104 * - at the same time check pmap for unmapped areas and anon for pages 105 * that we could map in (and do map it if found) 106 * - check object for resident pages that we could map in 107 * - if (case 2) goto Case2 108 * - >>> handle case 1 109 * - ensure source anon is resident in RAM 110 * - if case 1B alloc new anon and copy from source 111 * - map the correct page in 112 * Case2: 113 * - >>> handle case 2 114 * - ensure source page is resident (if uobj) 115 * - if case 2B alloc new anon and copy from source (could be zero 116 * fill if uobj == NULL) 117 * - map the correct page in 118 * - done! 119 * 120 * note on paging: 121 * if we have to do I/O we place a PG_BUSY page in the correct object, 122 * unlock everything, and do the I/O. when I/O is done we must reverify 123 * the state of the world before assuming that our data structures are 124 * valid. [because mappings could change while the map is unlocked] 125 * 126 * alternative 1: unbusy the page in question and restart the page fault 127 * from the top (ReFault). this is easy but does not take advantage 128 * of the information that we already have from our previous lookup, 129 * although it is possible that the "hints" in the vm_map will help here. 130 * 131 * alternative 2: the system already keeps track of a "version" number of 132 * a map. [i.e. every time you write-lock a map (e.g. to change a 133 * mapping) you bump the version number up by one...] so, we can save 134 * the version number of the map before we release the lock and start I/O. 135 * then when I/O is done we can relock and check the version numbers 136 * to see if anything changed. this might save us some over 1 because 137 * we don't have to unbusy the page and may be less compares(?). 138 * 139 * alternative 3: put in backpointers or a way to "hold" part of a map 140 * in place while I/O is in progress. this could be complex to 141 * implement (especially with structures like amap that can be referenced 142 * by multiple map entries, and figuring out what should wait could be 143 * complex as well...). 144 * 145 * given that we are not currently multiprocessor or multithreaded we might 146 * as well choose alternative 2 now. maybe alternative 3 would be useful 147 * in the future. XXX keep in mind for future consideration//rechecking. 148 */ 149 150 /* 151 * local data structures 152 */ 153 154 struct uvm_advice { 155 int advice; 156 int nback; 157 int nforw; 158 }; 159 160 /* 161 * page range array: 162 * note: index in array must match "advice" value 163 * XXX: borrowed numbers from freebsd. do they work well for us? 164 */ 165 166 static struct uvm_advice uvmadvice[] = { 167 { MADV_NORMAL, 3, 4 }, 168 { MADV_RANDOM, 0, 0 }, 169 { MADV_SEQUENTIAL, 8, 7}, 170 }; 171 172 #define UVM_MAXRANGE 16 /* must be max() of nback+nforw+1 */ 173 174 /* 175 * private prototypes 176 */ 177 178 static void uvmfault_amapcopy(struct uvm_faultinfo *); 179 static __inline void uvmfault_anonflush(struct vm_anon **, int); 180 181 /* 182 * inline functions 183 */ 184 185 /* 186 * uvmfault_anonflush: try and deactivate pages in specified anons 187 * 188 * => does not have to deactivate page if it is busy 189 */ 190 191 static __inline void 192 uvmfault_anonflush(anons, n) 193 struct vm_anon **anons; 194 int n; 195 { 196 int lcv; 197 struct vm_page *pg; 198 199 for (lcv = 0 ; lcv < n ; lcv++) { 200 if (anons[lcv] == NULL) 201 continue; 202 simple_lock(&anons[lcv]->an_lock); 203 pg = anons[lcv]->u.an_page; 204 if (pg && (pg->flags & PG_BUSY) == 0 && pg->loan_count == 0) { 205 uvm_lock_pageq(); 206 if (pg->wire_count == 0) { 207 #ifdef UBC 208 pmap_clear_reference(pg); 209 #else 210 pmap_page_protect(pg, VM_PROT_NONE); 211 #endif 212 uvm_pagedeactivate(pg); 213 } 214 uvm_unlock_pageq(); 215 } 216 simple_unlock(&anons[lcv]->an_lock); 217 } 218 } 219 220 /* 221 * normal functions 222 */ 223 224 /* 225 * uvmfault_amapcopy: clear "needs_copy" in a map. 226 * 227 * => called with VM data structures unlocked (usually, see below) 228 * => we get a write lock on the maps and clear needs_copy for a VA 229 * => if we are out of RAM we sleep (waiting for more) 230 */ 231 232 static void 233 uvmfault_amapcopy(ufi) 234 struct uvm_faultinfo *ufi; 235 { 236 237 /* 238 * while we haven't done the job 239 */ 240 241 while (1) { 242 243 /* 244 * no mapping? give up. 245 */ 246 247 if (uvmfault_lookup(ufi, TRUE) == FALSE) 248 return; 249 250 /* 251 * copy if needed. 252 */ 253 254 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) 255 amap_copy(ufi->map, ufi->entry, M_NOWAIT, TRUE, 256 ufi->orig_rvaddr, ufi->orig_rvaddr + 1); 257 258 /* 259 * didn't work? must be out of RAM. unlock and sleep. 260 */ 261 262 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 263 uvmfault_unlockmaps(ufi, TRUE); 264 uvm_wait("fltamapcopy"); 265 continue; 266 } 267 268 /* 269 * got it! unlock and return. 270 */ 271 272 uvmfault_unlockmaps(ufi, TRUE); 273 return; 274 } 275 /*NOTREACHED*/ 276 } 277 278 /* 279 * uvmfault_anonget: get data in an anon into a non-busy, non-released 280 * page in that anon. 281 * 282 * => maps, amap, and anon locked by caller. 283 * => if we fail (result != VM_PAGER_OK) we unlock everything. 284 * => if we are successful, we return with everything still locked. 285 * => we don't move the page on the queues [gets moved later] 286 * => if we allocate a new page [we_own], it gets put on the queues. 287 * either way, the result is that the page is on the queues at return time 288 * => for pages which are on loan from a uvm_object (and thus are not 289 * owned by the anon): if successful, we return with the owning object 290 * locked. the caller must unlock this object when it unlocks everything 291 * else. 292 */ 293 294 int 295 uvmfault_anonget(ufi, amap, anon) 296 struct uvm_faultinfo *ufi; 297 struct vm_amap *amap; 298 struct vm_anon *anon; 299 { 300 boolean_t we_own; /* we own anon's page? */ 301 boolean_t locked; /* did we relock? */ 302 struct vm_page *pg; 303 int result; 304 UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist); 305 306 result = 0; /* XXX shut up gcc */ 307 uvmexp.fltanget++; 308 /* bump rusage counters */ 309 if (anon->u.an_page) 310 curproc->p_addr->u_stats.p_ru.ru_minflt++; 311 else 312 curproc->p_addr->u_stats.p_ru.ru_majflt++; 313 314 /* 315 * loop until we get it, or fail. 316 */ 317 318 while (1) { 319 320 we_own = FALSE; /* TRUE if we set PG_BUSY on a page */ 321 pg = anon->u.an_page; 322 323 /* 324 * if there is a resident page and it is loaned, then anon 325 * may not own it. call out to uvm_anon_lockpage() to ensure 326 * the real owner of the page has been identified and locked. 327 */ 328 329 if (pg && pg->loan_count) 330 pg = uvm_anon_lockloanpg(anon); 331 332 /* 333 * page there? make sure it is not busy/released. 334 */ 335 336 if (pg) { 337 338 /* 339 * at this point, if the page has a uobject [meaning 340 * we have it on loan], then that uobject is locked 341 * by us! if the page is busy, we drop all the 342 * locks (including uobject) and try again. 343 */ 344 345 if ((pg->flags & (PG_BUSY|PG_RELEASED)) == 0) { 346 UVMHIST_LOG(maphist, "<- OK",0,0,0,0); 347 return (VM_PAGER_OK); 348 } 349 pg->flags |= PG_WANTED; 350 uvmexp.fltpgwait++; 351 352 /* 353 * the last unlock must be an atomic unlock+wait on 354 * the owner of page 355 */ 356 if (pg->uobject) { /* owner is uobject ? */ 357 uvmfault_unlockall(ufi, amap, NULL, anon); 358 UVMHIST_LOG(maphist, " unlock+wait on uobj",0, 359 0,0,0); 360 UVM_UNLOCK_AND_WAIT(pg, 361 &pg->uobject->vmobjlock, 362 FALSE, "anonget1",0); 363 } else { 364 /* anon owns page */ 365 uvmfault_unlockall(ufi, amap, NULL, NULL); 366 UVMHIST_LOG(maphist, " unlock+wait on anon",0, 367 0,0,0); 368 UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0, 369 "anonget2",0); 370 } 371 /* ready to relock and try again */ 372 373 } else { 374 375 /* 376 * no page, we must try and bring it in. 377 */ 378 pg = uvm_pagealloc(NULL, 0, anon, 0); 379 380 if (pg == NULL) { /* out of RAM. */ 381 382 uvmfault_unlockall(ufi, amap, NULL, anon); 383 uvmexp.fltnoram++; 384 UVMHIST_LOG(maphist, " noram -- UVM_WAIT",0, 385 0,0,0); 386 uvm_wait("flt_noram1"); 387 /* ready to relock and try again */ 388 389 } else { 390 391 /* we set the PG_BUSY bit */ 392 we_own = TRUE; 393 uvmfault_unlockall(ufi, amap, NULL, anon); 394 395 /* 396 * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN 397 * page into the uvm_swap_get function with 398 * all data structures unlocked. note that 399 * it is ok to read an_swslot here because 400 * we hold PG_BUSY on the page. 401 */ 402 uvmexp.pageins++; 403 result = uvm_swap_get(pg, anon->an_swslot, 404 PGO_SYNCIO); 405 406 /* 407 * we clean up after the i/o below in the 408 * "we_own" case 409 */ 410 /* ready to relock and try again */ 411 } 412 } 413 414 /* 415 * now relock and try again 416 */ 417 418 locked = uvmfault_relock(ufi); 419 if (locked && amap != NULL) { 420 amap_lock(amap); 421 } 422 if (locked || we_own) 423 simple_lock(&anon->an_lock); 424 425 /* 426 * if we own the page (i.e. we set PG_BUSY), then we need 427 * to clean up after the I/O. there are three cases to 428 * consider: 429 * [1] page released during I/O: free anon and ReFault. 430 * [2] I/O not OK. free the page and cause the fault 431 * to fail. 432 * [3] I/O OK! activate the page and sync with the 433 * non-we_own case (i.e. drop anon lock if not locked). 434 */ 435 436 if (we_own) { 437 438 if (pg->flags & PG_WANTED) { 439 /* still holding object lock */ 440 wakeup(pg); 441 } 442 /* un-busy! */ 443 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE); 444 UVM_PAGE_OWN(pg, NULL); 445 446 /* 447 * if we were RELEASED during I/O, then our anon is 448 * no longer part of an amap. we need to free the 449 * anon and try again. 450 */ 451 if (pg->flags & PG_RELEASED) { 452 pmap_page_protect(pg, VM_PROT_NONE); 453 simple_unlock(&anon->an_lock); 454 uvm_anfree(anon); /* frees page for us */ 455 if (locked) 456 uvmfault_unlockall(ufi, amap, NULL, 457 NULL); 458 uvmexp.fltpgrele++; 459 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 460 return (VM_PAGER_REFAULT); /* refault! */ 461 } 462 463 if (result != VM_PAGER_OK) { 464 KASSERT(result != VM_PAGER_PEND); 465 466 /* remove page from anon */ 467 anon->u.an_page = NULL; 468 469 /* 470 * remove the swap slot from the anon 471 * and mark the anon as having no real slot. 472 * don't free the swap slot, thus preventing 473 * it from being used again. 474 */ 475 uvm_swap_markbad(anon->an_swslot, 1); 476 anon->an_swslot = SWSLOT_BAD; 477 478 /* 479 * note: page was never !PG_BUSY, so it 480 * can't be mapped and thus no need to 481 * pmap_page_protect it... 482 */ 483 uvm_lock_pageq(); 484 uvm_pagefree(pg); 485 uvm_unlock_pageq(); 486 487 if (locked) 488 uvmfault_unlockall(ufi, amap, NULL, 489 anon); 490 else 491 simple_unlock(&anon->an_lock); 492 UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0); 493 return (VM_PAGER_ERROR); 494 } 495 496 /* 497 * must be OK, clear modify (already PG_CLEAN) 498 * and activate 499 */ 500 pmap_clear_modify(pg); 501 uvm_lock_pageq(); 502 uvm_pageactivate(pg); 503 uvm_unlock_pageq(); 504 if (!locked) 505 simple_unlock(&anon->an_lock); 506 } 507 508 /* 509 * we were not able to relock. restart fault. 510 */ 511 512 if (!locked) { 513 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 514 return (VM_PAGER_REFAULT); 515 } 516 517 /* 518 * verify no one has touched the amap and moved the anon on us. 519 */ 520 521 if (ufi != NULL && 522 amap_lookup(&ufi->entry->aref, 523 ufi->orig_rvaddr - ufi->entry->start) != anon) { 524 525 uvmfault_unlockall(ufi, amap, NULL, anon); 526 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 527 return (VM_PAGER_REFAULT); 528 } 529 530 /* 531 * try it again! 532 */ 533 534 uvmexp.fltanretry++; 535 continue; 536 537 } /* while (1) */ 538 539 /*NOTREACHED*/ 540 } 541 542 /* 543 * F A U L T - m a i n e n t r y p o i n t 544 */ 545 546 /* 547 * uvm_fault: page fault handler 548 * 549 * => called from MD code to resolve a page fault 550 * => VM data structures usually should be unlocked. however, it is 551 * possible to call here with the main map locked if the caller 552 * gets a write lock, sets it recursive, and then calls us (c.f. 553 * uvm_map_pageable). this should be avoided because it keeps 554 * the map locked off during I/O. 555 */ 556 557 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 558 ~VM_PROT_WRITE : VM_PROT_ALL) 559 560 int 561 uvm_fault(orig_map, vaddr, fault_type, access_type) 562 vm_map_t orig_map; 563 vaddr_t vaddr; 564 vm_fault_t fault_type; 565 vm_prot_t access_type; 566 { 567 struct uvm_faultinfo ufi; 568 vm_prot_t enter_prot; 569 boolean_t wired, narrow, promote, locked, shadowed; 570 int npages, nback, nforw, centeridx, result, lcv, gotpages; 571 vaddr_t startva, currva; 572 voff_t uoff; 573 paddr_t pa; 574 struct vm_amap *amap; 575 struct uvm_object *uobj; 576 struct vm_anon *anons_store[UVM_MAXRANGE], **anons, *anon, *oanon; 577 struct vm_page *pages[UVM_MAXRANGE], *pg, *uobjpage; 578 UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist); 579 580 UVMHIST_LOG(maphist, "(map=%p, vaddr=0x%lx, ft=%ld, at=%ld)", 581 orig_map, vaddr, fault_type, access_type); 582 583 anon = NULL; 584 pg = NULL; 585 586 uvmexp.faults++; /* XXX: locking? */ 587 588 /* 589 * init the IN parameters in the ufi 590 */ 591 592 ufi.orig_map = orig_map; 593 ufi.orig_rvaddr = trunc_page(vaddr); 594 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ 595 if (fault_type == VM_FAULT_WIRE) 596 narrow = TRUE; /* don't look for neighborhood 597 * pages on wire */ 598 else 599 narrow = FALSE; /* normal fault */ 600 601 /* 602 * before we do anything else, if this is a fault on a kernel 603 * address, check to see if the address is managed by an 604 * interrupt-safe map. If it is, we fail immediately. Intrsafe 605 * maps are never pageable, and this approach avoids an evil 606 * locking mess. 607 */ 608 if (orig_map == kernel_map && uvmfault_check_intrsafe(&ufi)) { 609 UVMHIST_LOG(maphist, "<- VA 0x%lx in intrsafe map %p", 610 ufi.orig_rvaddr, ufi.map, 0, 0); 611 return (KERN_FAILURE); 612 } 613 614 /* 615 * "goto ReFault" means restart the page fault from ground zero. 616 */ 617 ReFault: 618 619 /* 620 * lookup and lock the maps 621 */ 622 623 if (uvmfault_lookup(&ufi, FALSE) == FALSE) { 624 UVMHIST_LOG(maphist, "<- no mapping @ 0x%lx", vaddr, 0,0,0); 625 return (KERN_INVALID_ADDRESS); 626 } 627 /* locked: maps(read) */ 628 629 /* 630 * check protection 631 */ 632 633 if ((ufi.entry->protection & access_type) != access_type) { 634 UVMHIST_LOG(maphist, 635 "<- protection failure (prot=0x%lx, access=0x%lx)", 636 ufi.entry->protection, access_type, 0, 0); 637 uvmfault_unlockmaps(&ufi, FALSE); 638 return (KERN_PROTECTION_FAILURE); 639 } 640 641 /* 642 * if the map is not a pageable map, a page fault always fails. 643 */ 644 645 if ((ufi.map->flags & VM_MAP_PAGEABLE) == 0) { 646 UVMHIST_LOG(maphist, 647 "<- map %p not pageable", ufi.map, 0, 0, 0); 648 uvmfault_unlockmaps(&ufi, FALSE); 649 return (KERN_FAILURE); 650 } 651 652 /* 653 * "enter_prot" is the protection we want to enter the page in at. 654 * for certain pages (e.g. copy-on-write pages) this protection can 655 * be more strict than ufi.entry->protection. "wired" means either 656 * the entry is wired or we are fault-wiring the pg. 657 */ 658 659 enter_prot = ufi.entry->protection; 660 wired = VM_MAPENT_ISWIRED(ufi.entry) || (fault_type == VM_FAULT_WIRE); 661 if (wired) 662 access_type = enter_prot; /* full access for wired */ 663 664 /* 665 * handle "needs_copy" case. if we need to copy the amap we will 666 * have to drop our readlock and relock it with a write lock. (we 667 * need a write lock to change anything in a map entry [e.g. 668 * needs_copy]). 669 */ 670 671 if (UVM_ET_ISNEEDSCOPY(ufi.entry)) { 672 if ((access_type & VM_PROT_WRITE) || 673 (ufi.entry->object.uvm_obj == NULL)) { 674 /* need to clear */ 675 UVMHIST_LOG(maphist, 676 " need to clear needs_copy and refault",0,0,0,0); 677 uvmfault_unlockmaps(&ufi, FALSE); 678 uvmfault_amapcopy(&ufi); 679 uvmexp.fltamcopy++; 680 goto ReFault; 681 682 } else { 683 684 /* 685 * ensure that we pmap_enter page R/O since 686 * needs_copy is still true 687 */ 688 enter_prot &= ~VM_PROT_WRITE; 689 690 } 691 } 692 693 /* 694 * identify the players 695 */ 696 697 amap = ufi.entry->aref.ar_amap; /* top layer */ 698 uobj = ufi.entry->object.uvm_obj; /* bottom layer */ 699 700 /* 701 * check for a case 0 fault. if nothing backing the entry then 702 * error now. 703 */ 704 705 if (amap == NULL && uobj == NULL) { 706 uvmfault_unlockmaps(&ufi, FALSE); 707 UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0); 708 return (KERN_INVALID_ADDRESS); 709 } 710 711 /* 712 * establish range of interest based on advice from mapper 713 * and then clip to fit map entry. note that we only want 714 * to do this the first time through the fault. if we 715 * ReFault we will disable this by setting "narrow" to true. 716 */ 717 718 if (narrow == FALSE) { 719 720 /* wide fault (!narrow) */ 721 KASSERT(uvmadvice[ufi.entry->advice].advice == 722 ufi.entry->advice); 723 nback = min(uvmadvice[ufi.entry->advice].nback, 724 (ufi.orig_rvaddr - ufi.entry->start) >> PAGE_SHIFT); 725 startva = ufi.orig_rvaddr - (nback << PAGE_SHIFT); 726 nforw = min(uvmadvice[ufi.entry->advice].nforw, 727 ((ufi.entry->end - ufi.orig_rvaddr) >> 728 PAGE_SHIFT) - 1); 729 /* 730 * note: "-1" because we don't want to count the 731 * faulting page as forw 732 */ 733 npages = nback + nforw + 1; 734 centeridx = nback; 735 736 narrow = TRUE; /* ensure only once per-fault */ 737 738 } else { 739 740 /* narrow fault! */ 741 nback = nforw = 0; 742 startva = ufi.orig_rvaddr; 743 npages = 1; 744 centeridx = 0; 745 746 } 747 748 /* locked: maps(read) */ 749 UVMHIST_LOG(maphist, " narrow=%ld, back=%ld, forw=%ld, startva=0x%lx", 750 narrow, nback, nforw, startva); 751 UVMHIST_LOG(maphist, " entry=%p, amap=%p, obj=%p", ufi.entry, 752 amap, uobj, 0); 753 754 /* 755 * if we've got an amap, lock it and extract current anons. 756 */ 757 758 if (amap) { 759 amap_lock(amap); 760 anons = anons_store; 761 amap_lookups(&ufi.entry->aref, startva - ufi.entry->start, 762 anons, npages); 763 } else { 764 anons = NULL; /* to be safe */ 765 } 766 767 /* locked: maps(read), amap(if there) */ 768 769 /* 770 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages 771 * now and then forget about them (for the rest of the fault). 772 */ 773 774 if (ufi.entry->advice == MADV_SEQUENTIAL && nback != 0) { 775 776 UVMHIST_LOG(maphist, " MADV_SEQUENTIAL: flushing backpages", 777 0,0,0,0); 778 /* flush back-page anons? */ 779 if (amap) 780 uvmfault_anonflush(anons, nback); 781 782 /* flush object? */ 783 if (uobj) { 784 uoff = (startva - ufi.entry->start) + ufi.entry->offset; 785 simple_lock(&uobj->vmobjlock); 786 (void) uobj->pgops->pgo_flush(uobj, uoff, uoff + 787 (nback << PAGE_SHIFT), PGO_DEACTIVATE); 788 simple_unlock(&uobj->vmobjlock); 789 } 790 791 /* now forget about the backpages */ 792 if (amap) 793 anons += nback; 794 startva += (nback << PAGE_SHIFT); 795 npages -= nback; 796 nback = centeridx = 0; 797 } 798 799 /* locked: maps(read), amap(if there) */ 800 801 /* 802 * map in the backpages and frontpages we found in the amap in hopes 803 * of preventing future faults. we also init the pages[] array as 804 * we go. 805 */ 806 807 currva = startva; 808 shadowed = FALSE; 809 for (lcv = 0 ; lcv < npages ; lcv++, currva += PAGE_SIZE) { 810 811 /* 812 * dont play with VAs that are already mapped 813 * except for center) 814 */ 815 if (lcv != centeridx && 816 pmap_extract(ufi.orig_map->pmap, currva, &pa)) { 817 pages[lcv] = PGO_DONTCARE; 818 continue; 819 } 820 821 /* 822 * unmapped or center page. check if any anon at this level. 823 */ 824 if (amap == NULL || anons[lcv] == NULL) { 825 pages[lcv] = NULL; 826 continue; 827 } 828 829 /* 830 * check for present page and map if possible. re-activate it. 831 */ 832 833 pages[lcv] = PGO_DONTCARE; 834 if (lcv == centeridx) { /* save center for later! */ 835 shadowed = TRUE; 836 continue; 837 } 838 anon = anons[lcv]; 839 simple_lock(&anon->an_lock); 840 /* ignore loaned pages */ 841 if (anon->u.an_page && anon->u.an_page->loan_count == 0 && 842 (anon->u.an_page->flags & (PG_RELEASED|PG_BUSY)) == 0) { 843 uvm_lock_pageq(); 844 uvm_pageactivate(anon->u.an_page); /* reactivate */ 845 uvm_unlock_pageq(); 846 UVMHIST_LOG(maphist, 847 " MAPPING: n anon: pm=%p, va=0x%lx, pg=%p", 848 ufi.orig_map->pmap, currva, anon->u.an_page, 0); 849 uvmexp.fltnamap++; 850 851 /* 852 * Since this isn't the page that's actually faulting, 853 * ignore pmap_enter() failures; it's not critical 854 * that we enter these right now. 855 */ 856 857 (void) pmap_enter(ufi.orig_map->pmap, currva, 858 VM_PAGE_TO_PHYS(anon->u.an_page), 859 (anon->an_ref > 1) ? (enter_prot & ~VM_PROT_WRITE) : 860 enter_prot, 861 PMAP_CANFAIL | 862 (VM_MAPENT_ISWIRED(ufi.entry) ? PMAP_WIRED : 0)); 863 } 864 simple_unlock(&anon->an_lock); 865 pmap_update(ufi.orig_map->pmap); 866 } 867 868 /* locked: maps(read), amap(if there) */ 869 /* (shadowed == TRUE) if there is an anon at the faulting address */ 870 UVMHIST_LOG(maphist, " shadowed=%ld, will_get=%ld", shadowed, 871 (uobj && shadowed == FALSE),0,0); 872 873 /* 874 * note that if we are really short of RAM we could sleep in the above 875 * call to pmap_enter with everything locked. bad? 876 * 877 * XXX Actually, that is bad; pmap_enter() should just fail in that 878 * XXX case. --thorpej 879 */ 880 881 /* 882 * if the desired page is not shadowed by the amap and we have a 883 * backing object, then we check to see if the backing object would 884 * prefer to handle the fault itself (rather than letting us do it 885 * with the usual pgo_get hook). the backing object signals this by 886 * providing a pgo_fault routine. 887 */ 888 889 if (uobj && shadowed == FALSE && uobj->pgops->pgo_fault != NULL) { 890 simple_lock(&uobj->vmobjlock); 891 892 /* locked: maps(read), amap (if there), uobj */ 893 result = uobj->pgops->pgo_fault(&ufi, startva, pages, npages, 894 centeridx, fault_type, access_type, 895 PGO_LOCKED); 896 897 /* locked: nothing, pgo_fault has unlocked everything */ 898 899 if (result == VM_PAGER_OK) 900 return (KERN_SUCCESS); /* pgo_fault did pmap enter */ 901 else if (result == VM_PAGER_REFAULT) 902 goto ReFault; /* try again! */ 903 else 904 return (KERN_PROTECTION_FAILURE); 905 } 906 907 /* 908 * now, if the desired page is not shadowed by the amap and we have 909 * a backing object that does not have a special fault routine, then 910 * we ask (with pgo_get) the object for resident pages that we care 911 * about and attempt to map them in. we do not let pgo_get block 912 * (PGO_LOCKED). 913 * 914 * ("get" has the option of doing a pmap_enter for us) 915 */ 916 917 if (uobj && shadowed == FALSE) { 918 simple_lock(&uobj->vmobjlock); 919 920 /* locked (!shadowed): maps(read), amap (if there), uobj */ 921 /* 922 * the following call to pgo_get does _not_ change locking state 923 */ 924 925 uvmexp.fltlget++; 926 gotpages = npages; 927 (void) uobj->pgops->pgo_get(uobj, ufi.entry->offset + 928 (startva - ufi.entry->start), 929 pages, &gotpages, centeridx, 930 access_type & MASK(ufi.entry), 931 ufi.entry->advice, PGO_LOCKED); 932 933 /* 934 * check for pages to map, if we got any 935 */ 936 937 uobjpage = NULL; 938 939 if (gotpages) { 940 currva = startva; 941 for (lcv = 0 ; lcv < npages ; 942 lcv++, currva += PAGE_SIZE) { 943 944 if (pages[lcv] == NULL || 945 pages[lcv] == PGO_DONTCARE) 946 continue; 947 948 KASSERT((pages[lcv]->flags & PG_RELEASED) == 0); 949 950 /* 951 * if center page is resident and not 952 * PG_BUSY|PG_RELEASED then pgo_get 953 * made it PG_BUSY for us and gave 954 * us a handle to it. remember this 955 * page as "uobjpage." (for later use). 956 */ 957 958 if (lcv == centeridx) { 959 uobjpage = pages[lcv]; 960 UVMHIST_LOG(maphist, " got uobjpage " 961 "(%p) with locked get", 962 uobjpage, 0,0,0); 963 continue; 964 } 965 966 /* 967 * note: calling pgo_get with locked data 968 * structures returns us pages which are 969 * neither busy nor released, so we don't 970 * need to check for this. we can just 971 * directly enter the page (after moving it 972 * to the head of the active queue [useful?]). 973 */ 974 975 uvm_lock_pageq(); 976 uvm_pageactivate(pages[lcv]); /* reactivate */ 977 uvm_unlock_pageq(); 978 UVMHIST_LOG(maphist, 979 " MAPPING: n obj: pm=%p, va=0x%lx, pg=%p", 980 ufi.orig_map->pmap, currva, pages[lcv], 0); 981 uvmexp.fltnomap++; 982 983 /* 984 * Since this page isn't the page that's 985 * actually fauling, ignore pmap_enter() 986 * failures; it's not critical that we 987 * enter these right now. 988 */ 989 990 (void) pmap_enter(ufi.orig_map->pmap, currva, 991 VM_PAGE_TO_PHYS(pages[lcv]), 992 enter_prot & MASK(ufi.entry), 993 PMAP_CANFAIL | 994 (wired ? PMAP_WIRED : 0)); 995 996 /* 997 * NOTE: page can't be PG_WANTED or PG_RELEASED 998 * because we've held the lock the whole time 999 * we've had the handle. 1000 */ 1001 1002 pages[lcv]->flags &= ~(PG_BUSY); /* un-busy! */ 1003 UVM_PAGE_OWN(pages[lcv], NULL); 1004 } /* for "lcv" loop */ 1005 pmap_update(ufi.orig_map->pmap); 1006 } /* "gotpages" != 0 */ 1007 /* note: object still _locked_ */ 1008 } else { 1009 uobjpage = NULL; 1010 } 1011 1012 /* locked (shadowed): maps(read), amap */ 1013 /* locked (!shadowed): maps(read), amap(if there), 1014 uobj(if !null), uobjpage(if !null) */ 1015 1016 /* 1017 * note that at this point we are done with any front or back pages. 1018 * we are now going to focus on the center page (i.e. the one we've 1019 * faulted on). if we have faulted on the top (anon) layer 1020 * [i.e. case 1], then the anon we want is anons[centeridx] (we have 1021 * not touched it yet). if we have faulted on the bottom (uobj) 1022 * layer [i.e. case 2] and the page was both present and available, 1023 * then we've got a pointer to it as "uobjpage" and we've already 1024 * made it BUSY. 1025 */ 1026 1027 /* 1028 * there are four possible cases we must address: 1A, 1B, 2A, and 2B 1029 */ 1030 1031 /* 1032 * redirect case 2: if we are not shadowed, go to case 2. 1033 */ 1034 1035 if (shadowed == FALSE) 1036 goto Case2; 1037 1038 /* locked: maps(read), amap */ 1039 1040 /* 1041 * handle case 1: fault on an anon in our amap 1042 */ 1043 1044 anon = anons[centeridx]; 1045 UVMHIST_LOG(maphist, " case 1 fault: anon=%p", anon, 0,0,0); 1046 simple_lock(&anon->an_lock); 1047 1048 /* locked: maps(read), amap, anon */ 1049 1050 /* 1051 * no matter if we have case 1A or case 1B we are going to need to 1052 * have the anon's memory resident. ensure that now. 1053 */ 1054 1055 /* 1056 * let uvmfault_anonget do the dirty work. 1057 * if it fails (!OK) it will unlock everything for us. 1058 * if it succeeds, locks are still valid and locked. 1059 * also, if it is OK, then the anon's page is on the queues. 1060 * if the page is on loan from a uvm_object, then anonget will 1061 * lock that object for us if it does not fail. 1062 */ 1063 1064 result = uvmfault_anonget(&ufi, amap, anon); 1065 switch (result) { 1066 case VM_PAGER_OK: 1067 break; 1068 1069 case VM_PAGER_REFAULT: 1070 goto ReFault; 1071 1072 case VM_PAGER_ERROR: 1073 /* 1074 * An error occured while trying to bring in the 1075 * page -- this is the only error we return right 1076 * now. 1077 */ 1078 return (KERN_PROTECTION_FAILURE); /* XXX */ 1079 1080 default: 1081 #ifdef DIAGNOSTIC 1082 panic("uvm_fault: uvmfault_anonget -> %d", result); 1083 #else 1084 return (KERN_PROTECTION_FAILURE); 1085 #endif 1086 } 1087 1088 /* 1089 * uobj is non null if the page is on loan from an object (i.e. uobj) 1090 */ 1091 1092 uobj = anon->u.an_page->uobject; /* locked by anonget if !NULL */ 1093 1094 /* locked: maps(read), amap, anon, uobj(if one) */ 1095 1096 /* 1097 * special handling for loaned pages 1098 */ 1099 1100 if (anon->u.an_page->loan_count) { 1101 1102 if ((access_type & VM_PROT_WRITE) == 0) { 1103 1104 /* 1105 * for read faults on loaned pages we just cap the 1106 * protection at read-only. 1107 */ 1108 1109 enter_prot = enter_prot & ~VM_PROT_WRITE; 1110 1111 } else { 1112 /* 1113 * note that we can't allow writes into a loaned page! 1114 * 1115 * if we have a write fault on a loaned page in an 1116 * anon then we need to look at the anon's ref count. 1117 * if it is greater than one then we are going to do 1118 * a normal copy-on-write fault into a new anon (this 1119 * is not a problem). however, if the reference count 1120 * is one (a case where we would normally allow a 1121 * write directly to the page) then we need to kill 1122 * the loan before we continue. 1123 */ 1124 1125 /* >1 case is already ok */ 1126 if (anon->an_ref == 1) { 1127 1128 /* get new un-owned replacement page */ 1129 pg = uvm_pagealloc(NULL, 0, NULL, 0); 1130 if (pg == NULL) { 1131 uvmfault_unlockall(&ufi, amap, uobj, 1132 anon); 1133 uvm_wait("flt_noram2"); 1134 goto ReFault; 1135 } 1136 1137 /* 1138 * copy data, kill loan, and drop uobj lock 1139 * (if any) 1140 */ 1141 /* copy old -> new */ 1142 uvm_pagecopy(anon->u.an_page, pg); 1143 1144 /* force reload */ 1145 pmap_page_protect(anon->u.an_page, 1146 VM_PROT_NONE); 1147 uvm_lock_pageq(); /* KILL loan */ 1148 if (uobj) 1149 /* if we were loaning */ 1150 anon->u.an_page->loan_count--; 1151 anon->u.an_page->uanon = NULL; 1152 /* in case we owned */ 1153 anon->u.an_page->pqflags &= ~PQ_ANON; 1154 uvm_pageactivate(pg); 1155 uvm_unlock_pageq(); 1156 if (uobj) { 1157 simple_unlock(&uobj->vmobjlock); 1158 uobj = NULL; 1159 } 1160 1161 /* install new page in anon */ 1162 anon->u.an_page = pg; 1163 pg->uanon = anon; 1164 pg->pqflags |= PQ_ANON; 1165 pg->flags &= ~(PG_BUSY|PG_FAKE); 1166 UVM_PAGE_OWN(pg, NULL); 1167 1168 /* done! */ 1169 } /* ref == 1 */ 1170 } /* write fault */ 1171 } /* loan count */ 1172 1173 /* 1174 * if we are case 1B then we will need to allocate a new blank 1175 * anon to transfer the data into. note that we have a lock 1176 * on anon, so no one can busy or release the page until we are done. 1177 * also note that the ref count can't drop to zero here because 1178 * it is > 1 and we are only dropping one ref. 1179 * 1180 * in the (hopefully very rare) case that we are out of RAM we 1181 * will unlock, wait for more RAM, and refault. 1182 * 1183 * if we are out of anon VM we kill the process (XXX: could wait?). 1184 */ 1185 1186 if ((access_type & VM_PROT_WRITE) != 0 && anon->an_ref > 1) { 1187 1188 UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0); 1189 uvmexp.flt_acow++; 1190 oanon = anon; /* oanon = old, locked anon */ 1191 anon = uvm_analloc(); 1192 if (anon) { 1193 pg = uvm_pagealloc(NULL, 0, anon, 0); 1194 } 1195 1196 /* check for out of RAM */ 1197 if (anon == NULL || pg == NULL) { 1198 if (anon) 1199 uvm_anfree(anon); 1200 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1201 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1202 if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) { 1203 UVMHIST_LOG(maphist, 1204 "<- failed. out of VM",0,0,0,0); 1205 uvmexp.fltnoanon++; 1206 return (KERN_RESOURCE_SHORTAGE); 1207 } 1208 1209 uvmexp.fltnoram++; 1210 uvm_wait("flt_noram3"); /* out of RAM, wait for more */ 1211 goto ReFault; 1212 } 1213 1214 /* got all resources, replace anon with nanon */ 1215 1216 uvm_pagecopy(oanon->u.an_page, pg); /* pg now !PG_CLEAN */ 1217 pg->flags &= ~(PG_BUSY|PG_FAKE); /* un-busy! new page */ 1218 UVM_PAGE_OWN(pg, NULL); 1219 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start, 1220 anon, 1); 1221 1222 /* deref: can not drop to zero here by defn! */ 1223 oanon->an_ref--; 1224 1225 /* 1226 * note: oanon still locked. anon is _not_ locked, but we 1227 * have the sole references to in from amap which _is_ locked. 1228 * thus, no one can get at it until we are done with it. 1229 */ 1230 1231 } else { 1232 1233 uvmexp.flt_anon++; 1234 oanon = anon; /* old, locked anon is same as anon */ 1235 pg = anon->u.an_page; 1236 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */ 1237 enter_prot = enter_prot & ~VM_PROT_WRITE; 1238 1239 } 1240 1241 /* locked: maps(read), amap, oanon */ 1242 1243 /* 1244 * now map the page in ... 1245 * XXX: old fault unlocks object before pmap_enter. this seems 1246 * suspect since some other thread could blast the page out from 1247 * under us between the unlock and the pmap_enter. 1248 */ 1249 1250 UVMHIST_LOG(maphist, " MAPPING: anon: pm=%p, va=0x%lx, pg=%p", 1251 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, 0); 1252 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1253 enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) 1254 != KERN_SUCCESS) { 1255 /* 1256 * No need to undo what we did; we can simply think of 1257 * this as the pmap throwing away the mapping information. 1258 * 1259 * We do, however, have to go through the ReFault path, 1260 * as the map may change while we're asleep. 1261 */ 1262 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1263 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1264 if (uvmexp.swpgonly == uvmexp.swpages) { 1265 UVMHIST_LOG(maphist, 1266 "<- failed. out of VM",0,0,0,0); 1267 /* XXX instrumentation */ 1268 return (KERN_RESOURCE_SHORTAGE); 1269 } 1270 /* XXX instrumentation */ 1271 uvm_wait("flt_pmfail1"); 1272 goto ReFault; 1273 } 1274 1275 /* 1276 * ... update the page queues. 1277 */ 1278 1279 uvm_lock_pageq(); 1280 1281 if (fault_type == VM_FAULT_WIRE) { 1282 uvm_pagewire(pg); 1283 1284 /* 1285 * since the now-wired page cannot be paged out, 1286 * release its swap resources for others to use. 1287 * since an anon with no swap cannot be PG_CLEAN, 1288 * clear its clean flag now. 1289 */ 1290 1291 pg->flags &= ~(PG_CLEAN); 1292 uvm_anon_dropswap(anon); 1293 } else { 1294 /* activate it */ 1295 uvm_pageactivate(pg); 1296 } 1297 1298 uvm_unlock_pageq(); 1299 1300 /* 1301 * done case 1! finish up by unlocking everything and returning success 1302 */ 1303 1304 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1305 pmap_update(ufi.orig_map->pmap); 1306 return (KERN_SUCCESS); 1307 1308 1309 Case2: 1310 /* 1311 * handle case 2: faulting on backing object or zero fill 1312 */ 1313 1314 /* 1315 * locked: 1316 * maps(read), amap(if there), uobj(if !null), uobjpage(if !null) 1317 */ 1318 1319 /* 1320 * note that uobjpage can not be PGO_DONTCARE at this point. we now 1321 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we 1322 * have a backing object, check and see if we are going to promote 1323 * the data up to an anon during the fault. 1324 */ 1325 1326 if (uobj == NULL) { 1327 uobjpage = PGO_DONTCARE; 1328 promote = TRUE; /* always need anon here */ 1329 } else { 1330 KASSERT(uobjpage != PGO_DONTCARE); 1331 promote = (access_type & VM_PROT_WRITE) && 1332 UVM_ET_ISCOPYONWRITE(ufi.entry); 1333 } 1334 UVMHIST_LOG(maphist, " case 2 fault: promote=%ld, zfill=%ld", 1335 promote, (uobj == NULL), 0,0); 1336 1337 /* 1338 * if uobjpage is not null then we do not need to do I/O to get the 1339 * uobjpage. 1340 * 1341 * if uobjpage is null, then we need to unlock and ask the pager to 1342 * get the data for us. once we have the data, we need to reverify 1343 * the state the world. we are currently not holding any resources. 1344 */ 1345 1346 if (uobjpage) { 1347 /* update rusage counters */ 1348 curproc->p_addr->u_stats.p_ru.ru_minflt++; 1349 } else { 1350 /* update rusage counters */ 1351 curproc->p_addr->u_stats.p_ru.ru_majflt++; 1352 1353 /* locked: maps(read), amap(if there), uobj */ 1354 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1355 /* locked: uobj */ 1356 1357 uvmexp.fltget++; 1358 gotpages = 1; 1359 uoff = (ufi.orig_rvaddr - ufi.entry->start) + ufi.entry->offset; 1360 result = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages, 1361 0, access_type & MASK(ufi.entry), ufi.entry->advice, 1362 PGO_SYNCIO); 1363 1364 /* locked: uobjpage(if result OK) */ 1365 1366 /* 1367 * recover from I/O 1368 */ 1369 1370 if (result != VM_PAGER_OK) { 1371 KASSERT(result != VM_PAGER_PEND); 1372 1373 if (result == VM_PAGER_AGAIN) { 1374 UVMHIST_LOG(maphist, 1375 " pgo_get says TRY AGAIN!",0,0,0,0); 1376 tsleep((caddr_t)&lbolt, PVM, "fltagain2", 0); 1377 goto ReFault; 1378 } 1379 1380 UVMHIST_LOG(maphist, "<- pgo_get failed (code %ld)", 1381 result, 0,0,0); 1382 return (KERN_PROTECTION_FAILURE); /* XXX i/o error */ 1383 } 1384 1385 /* locked: uobjpage */ 1386 1387 /* 1388 * re-verify the state of the world by first trying to relock 1389 * the maps. always relock the object. 1390 */ 1391 1392 locked = uvmfault_relock(&ufi); 1393 if (locked && amap) 1394 amap_lock(amap); 1395 simple_lock(&uobj->vmobjlock); 1396 1397 /* locked(locked): maps(read), amap(if !null), uobj, uobjpage */ 1398 /* locked(!locked): uobj, uobjpage */ 1399 1400 /* 1401 * verify that the page has not be released and re-verify 1402 * that amap slot is still free. if there is a problem, 1403 * we unlock and clean up. 1404 */ 1405 1406 if ((uobjpage->flags & PG_RELEASED) != 0 || 1407 (locked && amap && 1408 amap_lookup(&ufi.entry->aref, 1409 ufi.orig_rvaddr - ufi.entry->start))) { 1410 if (locked) 1411 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1412 locked = FALSE; 1413 } 1414 1415 /* 1416 * didn't get the lock? release the page and retry. 1417 */ 1418 1419 if (locked == FALSE) { 1420 1421 UVMHIST_LOG(maphist, 1422 " wasn't able to relock after fault: retry", 1423 0,0,0,0); 1424 if (uobjpage->flags & PG_WANTED) 1425 /* still holding object lock */ 1426 wakeup(uobjpage); 1427 1428 if (uobjpage->flags & PG_RELEASED) { 1429 uvmexp.fltpgrele++; 1430 KASSERT(uobj->pgops->pgo_releasepg != NULL); 1431 1432 /* frees page */ 1433 if (uobj->pgops->pgo_releasepg(uobjpage,NULL)) 1434 /* unlock if still alive */ 1435 simple_unlock(&uobj->vmobjlock); 1436 goto ReFault; 1437 } 1438 1439 uvm_lock_pageq(); 1440 /* make sure it is in queues */ 1441 uvm_pageactivate(uobjpage); 1442 1443 uvm_unlock_pageq(); 1444 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1445 UVM_PAGE_OWN(uobjpage, NULL); 1446 simple_unlock(&uobj->vmobjlock); 1447 goto ReFault; 1448 1449 } 1450 1451 /* 1452 * we have the data in uobjpage which is PG_BUSY and 1453 * !PG_RELEASED. we are holding object lock (so the page 1454 * can't be released on us). 1455 */ 1456 1457 /* locked: maps(read), amap(if !null), uobj, uobjpage */ 1458 } 1459 1460 /* 1461 * locked: 1462 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj) 1463 */ 1464 1465 /* 1466 * notes: 1467 * - at this point uobjpage can not be NULL 1468 * - at this point uobjpage can not be PG_RELEASED (since we checked 1469 * for it above) 1470 * - at this point uobjpage could be PG_WANTED (handle later) 1471 */ 1472 1473 if (promote == FALSE) { 1474 1475 /* 1476 * we are not promoting. if the mapping is COW ensure that we 1477 * don't give more access than we should (e.g. when doing a read 1478 * fault on a COPYONWRITE mapping we want to map the COW page in 1479 * R/O even though the entry protection could be R/W). 1480 * 1481 * set "pg" to the page we want to map in (uobjpage, usually) 1482 */ 1483 1484 uvmexp.flt_obj++; 1485 if (UVM_ET_ISCOPYONWRITE(ufi.entry)) 1486 enter_prot &= ~VM_PROT_WRITE; 1487 pg = uobjpage; /* map in the actual object */ 1488 1489 /* assert(uobjpage != PGO_DONTCARE) */ 1490 1491 /* 1492 * we are faulting directly on the page. be careful 1493 * about writing to loaned pages... 1494 */ 1495 if (uobjpage->loan_count) { 1496 1497 if ((access_type & VM_PROT_WRITE) == 0) { 1498 /* read fault: cap the protection at readonly */ 1499 /* cap! */ 1500 enter_prot = enter_prot & ~VM_PROT_WRITE; 1501 } else { 1502 /* write fault: must break the loan here */ 1503 1504 /* alloc new un-owned page */ 1505 pg = uvm_pagealloc(NULL, 0, NULL, 0); 1506 1507 if (pg == NULL) { 1508 /* 1509 * drop ownership of page, it can't 1510 * be released 1511 */ 1512 if (uobjpage->flags & PG_WANTED) 1513 wakeup(uobjpage); 1514 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1515 UVM_PAGE_OWN(uobjpage, NULL); 1516 1517 uvm_lock_pageq(); 1518 /* activate: we will need it later */ 1519 uvm_pageactivate(uobjpage); 1520 1521 uvm_unlock_pageq(); 1522 uvmfault_unlockall(&ufi, amap, uobj, 1523 NULL); 1524 UVMHIST_LOG(maphist, 1525 " out of RAM breaking loan, waiting", 1526 0,0,0,0); 1527 uvmexp.fltnoram++; 1528 uvm_wait("flt_noram4"); 1529 goto ReFault; 1530 } 1531 1532 /* 1533 * copy the data from the old page to the new 1534 * one and clear the fake/clean flags on the 1535 * new page (keep it busy). force a reload 1536 * of the old page by clearing it from all 1537 * pmaps. then lock the page queues to 1538 * rename the pages. 1539 */ 1540 uvm_pagecopy(uobjpage, pg); /* old -> new */ 1541 pg->flags &= ~(PG_FAKE|PG_CLEAN); 1542 pmap_page_protect(uobjpage, VM_PROT_NONE); 1543 if (uobjpage->flags & PG_WANTED) 1544 wakeup(uobjpage); 1545 /* uobj still locked */ 1546 uobjpage->flags &= ~(PG_WANTED|PG_BUSY); 1547 UVM_PAGE_OWN(uobjpage, NULL); 1548 1549 uvm_lock_pageq(); 1550 uoff = uobjpage->offset; 1551 /* remove old page */ 1552 uvm_pagerealloc(uobjpage, NULL, 0); 1553 1554 /* 1555 * at this point we have absolutely no 1556 * control over uobjpage 1557 */ 1558 /* install new page */ 1559 uvm_pagerealloc(pg, uobj, uoff); 1560 uvm_unlock_pageq(); 1561 1562 /* 1563 * done! loan is broken and "pg" is 1564 * PG_BUSY. it can now replace uobjpage. 1565 */ 1566 1567 uobjpage = pg; 1568 1569 } /* write fault case */ 1570 } /* if loan_count */ 1571 1572 } else { 1573 1574 /* 1575 * if we are going to promote the data to an anon we 1576 * allocate a blank anon here and plug it into our amap. 1577 */ 1578 #ifdef DIAGNOSTIC 1579 if (amap == NULL) 1580 panic("uvm_fault: want to promote data, but no anon"); 1581 #endif 1582 1583 anon = uvm_analloc(); 1584 if (anon) { 1585 /* 1586 * In `Fill in data...' below, if 1587 * uobjpage == PGO_DONTCARE, we want 1588 * a zero'd, dirty page, so have 1589 * uvm_pagealloc() do that for us. 1590 */ 1591 pg = uvm_pagealloc(NULL, 0, anon, 1592 (uobjpage == PGO_DONTCARE) ? UVM_PGA_ZERO : 0); 1593 } 1594 1595 /* 1596 * out of memory resources? 1597 */ 1598 if (anon == NULL || pg == NULL) { 1599 1600 /* 1601 * arg! must unbusy our page and fail or sleep. 1602 */ 1603 if (uobjpage != PGO_DONTCARE) { 1604 if (uobjpage->flags & PG_WANTED) 1605 /* still holding object lock */ 1606 wakeup(uobjpage); 1607 1608 uvm_lock_pageq(); 1609 uvm_pageactivate(uobjpage); 1610 uvm_unlock_pageq(); 1611 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1612 UVM_PAGE_OWN(uobjpage, NULL); 1613 } 1614 1615 /* unlock and fail ... */ 1616 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1617 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1618 if (anon == NULL || uvmexp.swpgonly == uvmexp.swpages) { 1619 UVMHIST_LOG(maphist, " promote: out of VM", 1620 0,0,0,0); 1621 uvmexp.fltnoanon++; 1622 return (KERN_RESOURCE_SHORTAGE); 1623 } 1624 1625 UVMHIST_LOG(maphist, " out of RAM, waiting for more", 1626 0,0,0,0); 1627 uvm_anfree(anon); 1628 uvmexp.fltnoram++; 1629 uvm_wait("flt_noram5"); 1630 goto ReFault; 1631 } 1632 1633 /* 1634 * fill in the data 1635 */ 1636 1637 if (uobjpage != PGO_DONTCARE) { 1638 uvmexp.flt_prcopy++; 1639 /* copy page [pg now dirty] */ 1640 uvm_pagecopy(uobjpage, pg); 1641 1642 /* 1643 * promote to shared amap? make sure all sharing 1644 * procs see it 1645 */ 1646 if ((amap_flags(amap) & AMAP_SHARED) != 0) { 1647 pmap_page_protect(uobjpage, VM_PROT_NONE); 1648 } 1649 1650 /* 1651 * dispose of uobjpage. it can't be PG_RELEASED 1652 * since we still hold the object lock. 1653 * drop handle to uobj as well. 1654 */ 1655 1656 if (uobjpage->flags & PG_WANTED) 1657 /* still have the obj lock */ 1658 wakeup(uobjpage); 1659 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1660 UVM_PAGE_OWN(uobjpage, NULL); 1661 uvm_lock_pageq(); 1662 uvm_pageactivate(uobjpage); 1663 uvm_unlock_pageq(); 1664 simple_unlock(&uobj->vmobjlock); 1665 uobj = NULL; 1666 1667 UVMHIST_LOG(maphist, 1668 " promote uobjpage %p to anon/page %p/%p", 1669 uobjpage, anon, pg, 0); 1670 1671 } else { 1672 uvmexp.flt_przero++; 1673 /* 1674 * Page is zero'd and marked dirty by uvm_pagealloc() 1675 * above. 1676 */ 1677 UVMHIST_LOG(maphist," zero fill anon/page %p/%p", 1678 anon, pg, 0, 0); 1679 } 1680 1681 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start, 1682 anon, 0); 1683 } 1684 1685 /* 1686 * locked: 1687 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj) 1688 * 1689 * note: pg is either the uobjpage or the new page in the new anon 1690 */ 1691 1692 /* 1693 * all resources are present. we can now map it in and free our 1694 * resources. 1695 */ 1696 1697 UVMHIST_LOG(maphist, 1698 " MAPPING: case2: pm=%p, va=0x%lx, pg=%p, promote=%ld", 1699 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, promote); 1700 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1701 enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) 1702 != KERN_SUCCESS) { 1703 1704 /* 1705 * No need to undo what we did; we can simply think of 1706 * this as the pmap throwing away the mapping information. 1707 * 1708 * We do, however, have to go through the ReFault path, 1709 * as the map may change while we're asleep. 1710 */ 1711 1712 if (pg->flags & PG_WANTED) 1713 wakeup(pg); /* lock still held */ 1714 1715 /* 1716 * note that pg can't be PG_RELEASED since we did not drop 1717 * the object lock since the last time we checked. 1718 */ 1719 1720 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); 1721 UVM_PAGE_OWN(pg, NULL); 1722 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1723 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1724 if (uvmexp.swpgonly == uvmexp.swpages) { 1725 UVMHIST_LOG(maphist, 1726 "<- failed. out of VM",0,0,0,0); 1727 /* XXX instrumentation */ 1728 return (KERN_RESOURCE_SHORTAGE); 1729 } 1730 /* XXX instrumentation */ 1731 uvm_wait("flt_pmfail2"); 1732 goto ReFault; 1733 } 1734 1735 uvm_lock_pageq(); 1736 1737 if (fault_type == VM_FAULT_WIRE) { 1738 uvm_pagewire(pg); 1739 if (pg->pqflags & PQ_AOBJ) { 1740 1741 /* 1742 * since the now-wired page cannot be paged out, 1743 * release its swap resources for others to use. 1744 * since an aobj page with no swap cannot be PG_CLEAN, 1745 * clear its clean flag now. 1746 */ 1747 1748 pg->flags &= ~(PG_CLEAN); 1749 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT); 1750 } 1751 } else { 1752 /* activate it */ 1753 uvm_pageactivate(pg); 1754 } 1755 uvm_unlock_pageq(); 1756 1757 if (pg->flags & PG_WANTED) 1758 wakeup(pg); /* lock still held */ 1759 1760 /* 1761 * note that pg can't be PG_RELEASED since we did not drop the object 1762 * lock since the last time we checked. 1763 */ 1764 1765 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); 1766 UVM_PAGE_OWN(pg, NULL); 1767 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1768 pmap_update(ufi.orig_map->pmap); 1769 1770 UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0); 1771 return (KERN_SUCCESS); 1772 } 1773 1774 1775 /* 1776 * uvm_fault_wire: wire down a range of virtual addresses in a map. 1777 * 1778 * => map may be read-locked by caller, but MUST NOT be write-locked. 1779 * => if map is read-locked, any operations which may cause map to 1780 * be write-locked in uvm_fault() must be taken care of by 1781 * the caller. See uvm_map_pageable(). 1782 */ 1783 1784 int 1785 uvm_fault_wire(map, start, end, access_type) 1786 vm_map_t map; 1787 vaddr_t start, end; 1788 vm_prot_t access_type; 1789 { 1790 vaddr_t va; 1791 pmap_t pmap; 1792 int rv; 1793 1794 pmap = vm_map_pmap(map); 1795 1796 /* 1797 * now fault it in a page at a time. if the fault fails then we have 1798 * to undo what we have done. note that in uvm_fault VM_PROT_NONE 1799 * is replaced with the max protection if fault_type is VM_FAULT_WIRE. 1800 */ 1801 1802 for (va = start ; va < end ; va += PAGE_SIZE) { 1803 rv = uvm_fault(map, va, VM_FAULT_WIRE, access_type); 1804 if (rv) { 1805 if (va != start) { 1806 uvm_fault_unwire(map, start, va); 1807 } 1808 return (rv); 1809 } 1810 } 1811 1812 return (KERN_SUCCESS); 1813 } 1814 1815 /* 1816 * uvm_fault_unwire(): unwire range of virtual space. 1817 */ 1818 1819 void 1820 uvm_fault_unwire(map, start, end) 1821 vm_map_t map; 1822 vaddr_t start, end; 1823 { 1824 1825 vm_map_lock_read(map); 1826 uvm_fault_unwire_locked(map, start, end); 1827 vm_map_unlock_read(map); 1828 } 1829 1830 /* 1831 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire(). 1832 * 1833 * => map must be at least read-locked. 1834 */ 1835 1836 void 1837 uvm_fault_unwire_locked(map, start, end) 1838 vm_map_t map; 1839 vaddr_t start, end; 1840 { 1841 vm_map_entry_t entry; 1842 pmap_t pmap = vm_map_pmap(map); 1843 vaddr_t va; 1844 paddr_t pa; 1845 struct vm_page *pg; 1846 1847 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0); 1848 1849 /* 1850 * we assume that the area we are unwiring has actually been wired 1851 * in the first place. this means that we should be able to extract 1852 * the PAs from the pmap. we also lock out the page daemon so that 1853 * we can call uvm_pageunwire. 1854 */ 1855 1856 uvm_lock_pageq(); 1857 1858 /* 1859 * find the beginning map entry for the region. 1860 */ 1861 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map)); 1862 if (uvm_map_lookup_entry(map, start, &entry) == FALSE) 1863 panic("uvm_fault_unwire_locked: address not in map"); 1864 1865 for (va = start; va < end ; va += PAGE_SIZE) { 1866 if (pmap_extract(pmap, va, &pa) == FALSE) 1867 continue; 1868 1869 /* 1870 * find the map entry for the current address. 1871 */ 1872 KASSERT(va >= entry->start); 1873 while (va >= entry->end) { 1874 KASSERT(entry->next != &map->header && 1875 entry->next->start <= entry->end); 1876 entry = entry->next; 1877 } 1878 1879 /* 1880 * if the entry is no longer wired, tell the pmap. 1881 */ 1882 if (VM_MAPENT_ISWIRED(entry) == 0) 1883 pmap_unwire(pmap, va); 1884 1885 pg = PHYS_TO_VM_PAGE(pa); 1886 if (pg) 1887 uvm_pageunwire(pg); 1888 } 1889 1890 uvm_unlock_pageq(); 1891 } 1892