1 /* $NetBSD: uvm_fault.c,v 1.100 2005/07/31 04:04:47 yamt Exp $ */ 2 3 /* 4 * 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by Charles D. Cranor and 19 * Washington University. 20 * 4. The name of the author may not be used to endorse or promote products 21 * derived from this software without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 24 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 25 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 26 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 27 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 28 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 32 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 * 34 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp 35 */ 36 37 /* 38 * uvm_fault.c: fault handler 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.100 2005/07/31 04:04:47 yamt Exp $"); 43 44 #include "opt_uvmhist.h" 45 46 #include <sys/param.h> 47 #include <sys/systm.h> 48 #include <sys/kernel.h> 49 #include <sys/proc.h> 50 #include <sys/malloc.h> 51 #include <sys/mman.h> 52 #include <sys/user.h> 53 #include <sys/vnode.h> 54 55 #include <uvm/uvm.h> 56 57 /* 58 * 59 * a word on page faults: 60 * 61 * types of page faults we handle: 62 * 63 * CASE 1: upper layer faults CASE 2: lower layer faults 64 * 65 * CASE 1A CASE 1B CASE 2A CASE 2B 66 * read/write1 write>1 read/write +-cow_write/zero 67 * | | | | 68 * +--|--+ +--|--+ +-----+ + | + | +-----+ 69 * amap | V | | ----------->new| | | | ^ | 70 * +-----+ +-----+ +-----+ + | + | +--|--+ 71 * | | | 72 * +-----+ +-----+ +--|--+ | +--|--+ 73 * uobj | d/c | | d/c | | V | +----| | 74 * +-----+ +-----+ +-----+ +-----+ 75 * 76 * d/c = don't care 77 * 78 * case [0]: layerless fault 79 * no amap or uobj is present. this is an error. 80 * 81 * case [1]: upper layer fault [anon active] 82 * 1A: [read] or [write with anon->an_ref == 1] 83 * I/O takes place in top level anon and uobj is not touched. 84 * 1B: [write with anon->an_ref > 1] 85 * new anon is alloc'd and data is copied off ["COW"] 86 * 87 * case [2]: lower layer fault [uobj] 88 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area] 89 * I/O takes place directly in object. 90 * 2B: [write to copy_on_write] or [read on NULL uobj] 91 * data is "promoted" from uobj to a new anon. 92 * if uobj is null, then we zero fill. 93 * 94 * we follow the standard UVM locking protocol ordering: 95 * 96 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ) 97 * we hold a PG_BUSY page if we unlock for I/O 98 * 99 * 100 * the code is structured as follows: 101 * 102 * - init the "IN" params in the ufi structure 103 * ReFault: 104 * - do lookups [locks maps], check protection, handle needs_copy 105 * - check for case 0 fault (error) 106 * - establish "range" of fault 107 * - if we have an amap lock it and extract the anons 108 * - if sequential advice deactivate pages behind us 109 * - at the same time check pmap for unmapped areas and anon for pages 110 * that we could map in (and do map it if found) 111 * - check object for resident pages that we could map in 112 * - if (case 2) goto Case2 113 * - >>> handle case 1 114 * - ensure source anon is resident in RAM 115 * - if case 1B alloc new anon and copy from source 116 * - map the correct page in 117 * Case2: 118 * - >>> handle case 2 119 * - ensure source page is resident (if uobj) 120 * - if case 2B alloc new anon and copy from source (could be zero 121 * fill if uobj == NULL) 122 * - map the correct page in 123 * - done! 124 * 125 * note on paging: 126 * if we have to do I/O we place a PG_BUSY page in the correct object, 127 * unlock everything, and do the I/O. when I/O is done we must reverify 128 * the state of the world before assuming that our data structures are 129 * valid. [because mappings could change while the map is unlocked] 130 * 131 * alternative 1: unbusy the page in question and restart the page fault 132 * from the top (ReFault). this is easy but does not take advantage 133 * of the information that we already have from our previous lookup, 134 * although it is possible that the "hints" in the vm_map will help here. 135 * 136 * alternative 2: the system already keeps track of a "version" number of 137 * a map. [i.e. every time you write-lock a map (e.g. to change a 138 * mapping) you bump the version number up by one...] so, we can save 139 * the version number of the map before we release the lock and start I/O. 140 * then when I/O is done we can relock and check the version numbers 141 * to see if anything changed. this might save us some over 1 because 142 * we don't have to unbusy the page and may be less compares(?). 143 * 144 * alternative 3: put in backpointers or a way to "hold" part of a map 145 * in place while I/O is in progress. this could be complex to 146 * implement (especially with structures like amap that can be referenced 147 * by multiple map entries, and figuring out what should wait could be 148 * complex as well...). 149 * 150 * given that we are not currently multiprocessor or multithreaded we might 151 * as well choose alternative 2 now. maybe alternative 3 would be useful 152 * in the future. XXX keep in mind for future consideration//rechecking. 153 */ 154 155 /* 156 * local data structures 157 */ 158 159 struct uvm_advice { 160 int advice; 161 int nback; 162 int nforw; 163 }; 164 165 /* 166 * page range array: 167 * note: index in array must match "advice" value 168 * XXX: borrowed numbers from freebsd. do they work well for us? 169 */ 170 171 static const struct uvm_advice uvmadvice[] = { 172 { MADV_NORMAL, 3, 4 }, 173 { MADV_RANDOM, 0, 0 }, 174 { MADV_SEQUENTIAL, 8, 7}, 175 }; 176 177 #define UVM_MAXRANGE 16 /* must be MAX() of nback+nforw+1 */ 178 179 /* 180 * private prototypes 181 */ 182 183 /* 184 * inline functions 185 */ 186 187 /* 188 * uvmfault_anonflush: try and deactivate pages in specified anons 189 * 190 * => does not have to deactivate page if it is busy 191 */ 192 193 static __inline void 194 uvmfault_anonflush(struct vm_anon **anons, int n) 195 { 196 int lcv; 197 struct vm_page *pg; 198 199 for (lcv = 0 ; lcv < n ; lcv++) { 200 if (anons[lcv] == NULL) 201 continue; 202 simple_lock(&anons[lcv]->an_lock); 203 pg = anons[lcv]->an_page; 204 if (pg && (pg->flags & PG_BUSY) == 0 && pg->loan_count == 0) { 205 uvm_lock_pageq(); 206 if (pg->wire_count == 0) { 207 pmap_clear_reference(pg); 208 uvm_pagedeactivate(pg); 209 } 210 uvm_unlock_pageq(); 211 } 212 simple_unlock(&anons[lcv]->an_lock); 213 } 214 } 215 216 /* 217 * normal functions 218 */ 219 220 /* 221 * uvmfault_amapcopy: clear "needs_copy" in a map. 222 * 223 * => called with VM data structures unlocked (usually, see below) 224 * => we get a write lock on the maps and clear needs_copy for a VA 225 * => if we are out of RAM we sleep (waiting for more) 226 */ 227 228 static void 229 uvmfault_amapcopy(struct uvm_faultinfo *ufi) 230 { 231 for (;;) { 232 233 /* 234 * no mapping? give up. 235 */ 236 237 if (uvmfault_lookup(ufi, TRUE) == FALSE) 238 return; 239 240 /* 241 * copy if needed. 242 */ 243 244 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) 245 amap_copy(ufi->map, ufi->entry, M_NOWAIT, TRUE, 246 ufi->orig_rvaddr, ufi->orig_rvaddr + 1); 247 248 /* 249 * didn't work? must be out of RAM. unlock and sleep. 250 */ 251 252 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 253 uvmfault_unlockmaps(ufi, TRUE); 254 uvm_wait("fltamapcopy"); 255 continue; 256 } 257 258 /* 259 * got it! unlock and return. 260 */ 261 262 uvmfault_unlockmaps(ufi, TRUE); 263 return; 264 } 265 /*NOTREACHED*/ 266 } 267 268 /* 269 * uvmfault_anonget: get data in an anon into a non-busy, non-released 270 * page in that anon. 271 * 272 * => maps, amap, and anon locked by caller. 273 * => if we fail (result != 0) we unlock everything. 274 * => if we are successful, we return with everything still locked. 275 * => we don't move the page on the queues [gets moved later] 276 * => if we allocate a new page [we_own], it gets put on the queues. 277 * either way, the result is that the page is on the queues at return time 278 * => for pages which are on loan from a uvm_object (and thus are not 279 * owned by the anon): if successful, we return with the owning object 280 * locked. the caller must unlock this object when it unlocks everything 281 * else. 282 */ 283 284 int 285 uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap, 286 struct vm_anon *anon) 287 { 288 boolean_t we_own; /* we own anon's page? */ 289 boolean_t locked; /* did we relock? */ 290 struct vm_page *pg; 291 int error; 292 UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist); 293 294 LOCK_ASSERT(simple_lock_held(&anon->an_lock)); 295 296 error = 0; 297 uvmexp.fltanget++; 298 /* bump rusage counters */ 299 if (anon->an_page) 300 curproc->p_stats->p_ru.ru_minflt++; 301 else 302 curproc->p_stats->p_ru.ru_majflt++; 303 304 /* 305 * loop until we get it, or fail. 306 */ 307 308 for (;;) { 309 we_own = FALSE; /* TRUE if we set PG_BUSY on a page */ 310 pg = anon->an_page; 311 312 /* 313 * if there is a resident page and it is loaned, then anon 314 * may not own it. call out to uvm_anon_lockpage() to ensure 315 * the real owner of the page has been identified and locked. 316 */ 317 318 if (pg && pg->loan_count) 319 pg = uvm_anon_lockloanpg(anon); 320 321 /* 322 * page there? make sure it is not busy/released. 323 */ 324 325 if (pg) { 326 327 /* 328 * at this point, if the page has a uobject [meaning 329 * we have it on loan], then that uobject is locked 330 * by us! if the page is busy, we drop all the 331 * locks (including uobject) and try again. 332 */ 333 334 if ((pg->flags & PG_BUSY) == 0) { 335 UVMHIST_LOG(maphist, "<- OK",0,0,0,0); 336 return (0); 337 } 338 pg->flags |= PG_WANTED; 339 uvmexp.fltpgwait++; 340 341 /* 342 * the last unlock must be an atomic unlock+wait on 343 * the owner of page 344 */ 345 346 if (pg->uobject) { /* owner is uobject ? */ 347 uvmfault_unlockall(ufi, amap, NULL, anon); 348 UVMHIST_LOG(maphist, " unlock+wait on uobj",0, 349 0,0,0); 350 UVM_UNLOCK_AND_WAIT(pg, 351 &pg->uobject->vmobjlock, 352 FALSE, "anonget1",0); 353 } else { 354 /* anon owns page */ 355 uvmfault_unlockall(ufi, amap, NULL, NULL); 356 UVMHIST_LOG(maphist, " unlock+wait on anon",0, 357 0,0,0); 358 UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0, 359 "anonget2",0); 360 } 361 } else { 362 363 /* 364 * no page, we must try and bring it in. 365 */ 366 367 pg = uvm_pagealloc(NULL, 0, anon, 0); 368 if (pg == NULL) { /* out of RAM. */ 369 uvmfault_unlockall(ufi, amap, NULL, anon); 370 uvmexp.fltnoram++; 371 UVMHIST_LOG(maphist, " noram -- UVM_WAIT",0, 372 0,0,0); 373 if (!uvm_reclaimable()) { 374 return ENOMEM; 375 } 376 uvm_wait("flt_noram1"); 377 } else { 378 /* we set the PG_BUSY bit */ 379 we_own = TRUE; 380 uvmfault_unlockall(ufi, amap, NULL, anon); 381 382 /* 383 * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN 384 * page into the uvm_swap_get function with 385 * all data structures unlocked. note that 386 * it is ok to read an_swslot here because 387 * we hold PG_BUSY on the page. 388 */ 389 uvmexp.pageins++; 390 error = uvm_swap_get(pg, anon->an_swslot, 391 PGO_SYNCIO); 392 393 /* 394 * we clean up after the i/o below in the 395 * "we_own" case 396 */ 397 } 398 } 399 400 /* 401 * now relock and try again 402 */ 403 404 locked = uvmfault_relock(ufi); 405 if (locked && amap != NULL) { 406 amap_lock(amap); 407 } 408 if (locked || we_own) 409 simple_lock(&anon->an_lock); 410 411 /* 412 * if we own the page (i.e. we set PG_BUSY), then we need 413 * to clean up after the I/O. there are three cases to 414 * consider: 415 * [1] page released during I/O: free anon and ReFault. 416 * [2] I/O not OK. free the page and cause the fault 417 * to fail. 418 * [3] I/O OK! activate the page and sync with the 419 * non-we_own case (i.e. drop anon lock if not locked). 420 */ 421 422 if (we_own) { 423 if (pg->flags & PG_WANTED) { 424 wakeup(pg); 425 } 426 if (error) { 427 428 /* 429 * remove the swap slot from the anon 430 * and mark the anon as having no real slot. 431 * don't free the swap slot, thus preventing 432 * it from being used again. 433 */ 434 435 if (anon->an_swslot > 0) 436 uvm_swap_markbad(anon->an_swslot, 1); 437 anon->an_swslot = SWSLOT_BAD; 438 439 if ((pg->flags & PG_RELEASED) != 0) 440 goto released; 441 442 /* 443 * note: page was never !PG_BUSY, so it 444 * can't be mapped and thus no need to 445 * pmap_page_protect it... 446 */ 447 448 uvm_lock_pageq(); 449 uvm_pagefree(pg); 450 uvm_unlock_pageq(); 451 452 if (locked) 453 uvmfault_unlockall(ufi, amap, NULL, 454 anon); 455 else 456 simple_unlock(&anon->an_lock); 457 UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0); 458 return error; 459 } 460 461 if ((pg->flags & PG_RELEASED) != 0) { 462 released: 463 KASSERT(anon->an_ref == 0); 464 465 /* 466 * released while we unlocked amap. 467 */ 468 469 if (locked) 470 uvmfault_unlockall(ufi, amap, NULL, 471 NULL); 472 473 uvm_anon_release(anon); 474 475 if (error) { 476 UVMHIST_LOG(maphist, 477 "<- ERROR/RELEASED", 0,0,0,0); 478 return error; 479 } 480 481 UVMHIST_LOG(maphist, "<- RELEASED", 0,0,0,0); 482 return ERESTART; 483 } 484 485 /* 486 * we've successfully read the page, activate it. 487 */ 488 489 uvm_lock_pageq(); 490 uvm_pageactivate(pg); 491 uvm_unlock_pageq(); 492 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE); 493 UVM_PAGE_OWN(pg, NULL); 494 if (!locked) 495 simple_unlock(&anon->an_lock); 496 } 497 498 /* 499 * we were not able to relock. restart fault. 500 */ 501 502 if (!locked) { 503 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 504 return (ERESTART); 505 } 506 507 /* 508 * verify no one has touched the amap and moved the anon on us. 509 */ 510 511 if (ufi != NULL && 512 amap_lookup(&ufi->entry->aref, 513 ufi->orig_rvaddr - ufi->entry->start) != anon) { 514 515 uvmfault_unlockall(ufi, amap, NULL, anon); 516 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 517 return (ERESTART); 518 } 519 520 /* 521 * try it again! 522 */ 523 524 uvmexp.fltanretry++; 525 continue; 526 } 527 /*NOTREACHED*/ 528 } 529 530 /* 531 * F A U L T - m a i n e n t r y p o i n t 532 */ 533 534 /* 535 * uvm_fault: page fault handler 536 * 537 * => called from MD code to resolve a page fault 538 * => VM data structures usually should be unlocked. however, it is 539 * possible to call here with the main map locked if the caller 540 * gets a write lock, sets it recusive, and then calls us (c.f. 541 * uvm_map_pageable). this should be avoided because it keeps 542 * the map locked off during I/O. 543 * => MUST NEVER BE CALLED IN INTERRUPT CONTEXT 544 */ 545 546 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 547 ~VM_PROT_WRITE : VM_PROT_ALL) 548 549 int 550 uvm_fault(struct vm_map *orig_map, vaddr_t vaddr, vm_fault_t fault_type, 551 vm_prot_t access_type) 552 { 553 struct uvm_faultinfo ufi; 554 vm_prot_t enter_prot, check_prot; 555 boolean_t wired, narrow, promote, locked, shadowed, wire_fault, cow_now; 556 int npages, nback, nforw, centeridx, error, lcv, gotpages; 557 vaddr_t startva, currva; 558 voff_t uoff; 559 struct vm_amap *amap; 560 struct uvm_object *uobj; 561 struct vm_anon *anons_store[UVM_MAXRANGE], **anons, *anon, *oanon; 562 struct vm_page *pages[UVM_MAXRANGE], *pg, *uobjpage; 563 UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist); 564 565 UVMHIST_LOG(maphist, "(map=0x%x, vaddr=0x%x, ft=%d, at=%d)", 566 orig_map, vaddr, fault_type, access_type); 567 568 anon = NULL; 569 pg = NULL; 570 571 uvmexp.faults++; /* XXX: locking? */ 572 573 /* 574 * init the IN parameters in the ufi 575 */ 576 577 ufi.orig_map = orig_map; 578 ufi.orig_rvaddr = trunc_page(vaddr); 579 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ 580 wire_fault = fault_type == VM_FAULT_WIRE || 581 fault_type == VM_FAULT_WIREMAX; 582 if (wire_fault) 583 narrow = TRUE; /* don't look for neighborhood 584 * pages on wire */ 585 else 586 narrow = FALSE; /* normal fault */ 587 588 /* 589 * "goto ReFault" means restart the page fault from ground zero. 590 */ 591 ReFault: 592 593 /* 594 * lookup and lock the maps 595 */ 596 597 if (uvmfault_lookup(&ufi, FALSE) == FALSE) { 598 UVMHIST_LOG(maphist, "<- no mapping @ 0x%x", vaddr, 0,0,0); 599 return (EFAULT); 600 } 601 /* locked: maps(read) */ 602 603 #ifdef DIAGNOSTIC 604 if ((ufi.map->flags & VM_MAP_PAGEABLE) == 0) { 605 printf("Page fault on non-pageable map:\n"); 606 printf("ufi.map = %p\n", ufi.map); 607 printf("ufi.orig_map = %p\n", ufi.orig_map); 608 printf("ufi.orig_rvaddr = 0x%lx\n", (u_long) ufi.orig_rvaddr); 609 panic("uvm_fault: (ufi.map->flags & VM_MAP_PAGEABLE) == 0"); 610 } 611 #endif 612 613 /* 614 * check protection 615 */ 616 617 check_prot = fault_type == VM_FAULT_WIREMAX ? 618 ufi.entry->max_protection : ufi.entry->protection; 619 if ((check_prot & access_type) != access_type) { 620 UVMHIST_LOG(maphist, 621 "<- protection failure (prot=0x%x, access=0x%x)", 622 ufi.entry->protection, access_type, 0, 0); 623 uvmfault_unlockmaps(&ufi, FALSE); 624 return EACCES; 625 } 626 627 /* 628 * "enter_prot" is the protection we want to enter the page in at. 629 * for certain pages (e.g. copy-on-write pages) this protection can 630 * be more strict than ufi.entry->protection. "wired" means either 631 * the entry is wired or we are fault-wiring the pg. 632 */ 633 634 enter_prot = ufi.entry->protection; 635 wired = VM_MAPENT_ISWIRED(ufi.entry) || wire_fault; 636 if (wired) { 637 access_type = enter_prot; /* full access for wired */ 638 cow_now = (check_prot & VM_PROT_WRITE) != 0; 639 } else { 640 cow_now = (access_type & VM_PROT_WRITE) != 0; 641 } 642 643 /* 644 * handle "needs_copy" case. if we need to copy the amap we will 645 * have to drop our readlock and relock it with a write lock. (we 646 * need a write lock to change anything in a map entry [e.g. 647 * needs_copy]). 648 */ 649 650 if (UVM_ET_ISNEEDSCOPY(ufi.entry)) { 651 KASSERT(fault_type != VM_FAULT_WIREMAX); 652 if (cow_now || (ufi.entry->object.uvm_obj == NULL)) { 653 /* need to clear */ 654 UVMHIST_LOG(maphist, 655 " need to clear needs_copy and refault",0,0,0,0); 656 uvmfault_unlockmaps(&ufi, FALSE); 657 uvmfault_amapcopy(&ufi); 658 uvmexp.fltamcopy++; 659 goto ReFault; 660 661 } else { 662 663 /* 664 * ensure that we pmap_enter page R/O since 665 * needs_copy is still true 666 */ 667 668 enter_prot &= ~VM_PROT_WRITE; 669 } 670 } 671 672 /* 673 * identify the players 674 */ 675 676 amap = ufi.entry->aref.ar_amap; /* top layer */ 677 uobj = ufi.entry->object.uvm_obj; /* bottom layer */ 678 679 /* 680 * check for a case 0 fault. if nothing backing the entry then 681 * error now. 682 */ 683 684 if (amap == NULL && uobj == NULL) { 685 uvmfault_unlockmaps(&ufi, FALSE); 686 UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0); 687 return (EFAULT); 688 } 689 690 /* 691 * establish range of interest based on advice from mapper 692 * and then clip to fit map entry. note that we only want 693 * to do this the first time through the fault. if we 694 * ReFault we will disable this by setting "narrow" to true. 695 */ 696 697 if (narrow == FALSE) { 698 699 /* wide fault (!narrow) */ 700 KASSERT(uvmadvice[ufi.entry->advice].advice == 701 ufi.entry->advice); 702 nback = MIN(uvmadvice[ufi.entry->advice].nback, 703 (ufi.orig_rvaddr - ufi.entry->start) >> PAGE_SHIFT); 704 startva = ufi.orig_rvaddr - (nback << PAGE_SHIFT); 705 nforw = MIN(uvmadvice[ufi.entry->advice].nforw, 706 ((ufi.entry->end - ufi.orig_rvaddr) >> 707 PAGE_SHIFT) - 1); 708 /* 709 * note: "-1" because we don't want to count the 710 * faulting page as forw 711 */ 712 npages = nback + nforw + 1; 713 centeridx = nback; 714 715 narrow = TRUE; /* ensure only once per-fault */ 716 717 } else { 718 719 /* narrow fault! */ 720 nback = nforw = 0; 721 startva = ufi.orig_rvaddr; 722 npages = 1; 723 centeridx = 0; 724 725 } 726 727 /* locked: maps(read) */ 728 UVMHIST_LOG(maphist, " narrow=%d, back=%d, forw=%d, startva=0x%x", 729 narrow, nback, nforw, startva); 730 UVMHIST_LOG(maphist, " entry=0x%x, amap=0x%x, obj=0x%x", ufi.entry, 731 amap, uobj, 0); 732 733 /* 734 * if we've got an amap, lock it and extract current anons. 735 */ 736 737 if (amap) { 738 amap_lock(amap); 739 anons = anons_store; 740 amap_lookups(&ufi.entry->aref, startva - ufi.entry->start, 741 anons, npages); 742 } else { 743 anons = NULL; /* to be safe */ 744 } 745 746 /* locked: maps(read), amap(if there) */ 747 748 /* 749 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages 750 * now and then forget about them (for the rest of the fault). 751 */ 752 753 if (ufi.entry->advice == MADV_SEQUENTIAL && nback != 0) { 754 755 UVMHIST_LOG(maphist, " MADV_SEQUENTIAL: flushing backpages", 756 0,0,0,0); 757 /* flush back-page anons? */ 758 if (amap) 759 uvmfault_anonflush(anons, nback); 760 761 /* flush object? */ 762 if (uobj) { 763 uoff = (startva - ufi.entry->start) + ufi.entry->offset; 764 simple_lock(&uobj->vmobjlock); 765 (void) (uobj->pgops->pgo_put)(uobj, uoff, uoff + 766 (nback << PAGE_SHIFT), PGO_DEACTIVATE); 767 } 768 769 /* now forget about the backpages */ 770 if (amap) 771 anons += nback; 772 startva += (nback << PAGE_SHIFT); 773 npages -= nback; 774 nback = centeridx = 0; 775 } 776 777 /* locked: maps(read), amap(if there) */ 778 779 /* 780 * map in the backpages and frontpages we found in the amap in hopes 781 * of preventing future faults. we also init the pages[] array as 782 * we go. 783 */ 784 785 currva = startva; 786 shadowed = FALSE; 787 for (lcv = 0 ; lcv < npages ; lcv++, currva += PAGE_SIZE) { 788 789 /* 790 * dont play with VAs that are already mapped 791 * except for center) 792 */ 793 if (lcv != centeridx && 794 pmap_extract(ufi.orig_map->pmap, currva, NULL)) { 795 pages[lcv] = PGO_DONTCARE; 796 continue; 797 } 798 799 /* 800 * unmapped or center page. check if any anon at this level. 801 */ 802 if (amap == NULL || anons[lcv] == NULL) { 803 pages[lcv] = NULL; 804 continue; 805 } 806 807 /* 808 * check for present page and map if possible. re-activate it. 809 */ 810 811 pages[lcv] = PGO_DONTCARE; 812 if (lcv == centeridx) { /* save center for later! */ 813 shadowed = TRUE; 814 continue; 815 } 816 anon = anons[lcv]; 817 simple_lock(&anon->an_lock); 818 /* ignore loaned pages */ 819 if (anon->an_page && anon->an_page->loan_count == 0 && 820 (anon->an_page->flags & PG_BUSY) == 0) { 821 uvm_lock_pageq(); 822 uvm_pageactivate(anon->an_page); 823 uvm_unlock_pageq(); 824 UVMHIST_LOG(maphist, 825 " MAPPING: n anon: pm=0x%x, va=0x%x, pg=0x%x", 826 ufi.orig_map->pmap, currva, anon->an_page, 0); 827 uvmexp.fltnamap++; 828 829 /* 830 * Since this isn't the page that's actually faulting, 831 * ignore pmap_enter() failures; it's not critical 832 * that we enter these right now. 833 */ 834 835 (void) pmap_enter(ufi.orig_map->pmap, currva, 836 VM_PAGE_TO_PHYS(anon->an_page), 837 (anon->an_ref > 1) ? (enter_prot & ~VM_PROT_WRITE) : 838 enter_prot, 839 PMAP_CANFAIL | 840 (VM_MAPENT_ISWIRED(ufi.entry) ? PMAP_WIRED : 0)); 841 } 842 simple_unlock(&anon->an_lock); 843 pmap_update(ufi.orig_map->pmap); 844 } 845 846 /* locked: maps(read), amap(if there) */ 847 /* (shadowed == TRUE) if there is an anon at the faulting address */ 848 UVMHIST_LOG(maphist, " shadowed=%d, will_get=%d", shadowed, 849 (uobj && shadowed == FALSE),0,0); 850 851 /* 852 * note that if we are really short of RAM we could sleep in the above 853 * call to pmap_enter with everything locked. bad? 854 * 855 * XXX Actually, that is bad; pmap_enter() should just fail in that 856 * XXX case. --thorpej 857 */ 858 859 /* 860 * if the desired page is not shadowed by the amap and we have a 861 * backing object, then we check to see if the backing object would 862 * prefer to handle the fault itself (rather than letting us do it 863 * with the usual pgo_get hook). the backing object signals this by 864 * providing a pgo_fault routine. 865 */ 866 867 if (uobj && shadowed == FALSE && uobj->pgops->pgo_fault != NULL) { 868 simple_lock(&uobj->vmobjlock); 869 870 /* locked: maps(read), amap (if there), uobj */ 871 error = uobj->pgops->pgo_fault(&ufi, startva, pages, npages, 872 centeridx, fault_type, access_type, PGO_LOCKED|PGO_SYNCIO); 873 874 /* locked: nothing, pgo_fault has unlocked everything */ 875 876 if (error == ERESTART) 877 goto ReFault; /* try again! */ 878 /* 879 * object fault routine responsible for pmap_update(). 880 */ 881 return error; 882 } 883 884 /* 885 * now, if the desired page is not shadowed by the amap and we have 886 * a backing object that does not have a special fault routine, then 887 * we ask (with pgo_get) the object for resident pages that we care 888 * about and attempt to map them in. we do not let pgo_get block 889 * (PGO_LOCKED). 890 */ 891 892 if (uobj && shadowed == FALSE) { 893 simple_lock(&uobj->vmobjlock); 894 895 /* locked (!shadowed): maps(read), amap (if there), uobj */ 896 /* 897 * the following call to pgo_get does _not_ change locking state 898 */ 899 900 uvmexp.fltlget++; 901 gotpages = npages; 902 (void) uobj->pgops->pgo_get(uobj, ufi.entry->offset + 903 (startva - ufi.entry->start), 904 pages, &gotpages, centeridx, 905 access_type & MASK(ufi.entry), 906 ufi.entry->advice, PGO_LOCKED); 907 908 /* 909 * check for pages to map, if we got any 910 */ 911 912 uobjpage = NULL; 913 914 if (gotpages) { 915 currva = startva; 916 for (lcv = 0; lcv < npages; 917 lcv++, currva += PAGE_SIZE) { 918 struct vm_page *curpg; 919 boolean_t readonly; 920 921 curpg = pages[lcv]; 922 if (curpg == NULL || curpg == PGO_DONTCARE) { 923 continue; 924 } 925 926 /* 927 * if center page is resident and not 928 * PG_BUSY|PG_RELEASED then pgo_get 929 * made it PG_BUSY for us and gave 930 * us a handle to it. remember this 931 * page as "uobjpage." (for later use). 932 */ 933 934 if (lcv == centeridx) { 935 uobjpage = curpg; 936 UVMHIST_LOG(maphist, " got uobjpage " 937 "(0x%x) with locked get", 938 uobjpage, 0,0,0); 939 continue; 940 } 941 942 /* 943 * calling pgo_get with PGO_LOCKED returns us 944 * pages which are neither busy nor released, 945 * so we don't need to check for this. 946 * we can just directly enter the pages. 947 */ 948 949 uvm_lock_pageq(); 950 uvm_pageactivate(curpg); 951 uvm_unlock_pageq(); 952 UVMHIST_LOG(maphist, 953 " MAPPING: n obj: pm=0x%x, va=0x%x, pg=0x%x", 954 ufi.orig_map->pmap, currva, curpg, 0); 955 uvmexp.fltnomap++; 956 957 /* 958 * Since this page isn't the page that's 959 * actually faulting, ignore pmap_enter() 960 * failures; it's not critical that we 961 * enter these right now. 962 */ 963 KASSERT((curpg->flags & PG_PAGEOUT) == 0); 964 KASSERT((curpg->flags & PG_RELEASED) == 0); 965 KASSERT(!UVM_OBJ_IS_CLEAN(curpg->uobject) || 966 (curpg->flags & PG_CLEAN) != 0); 967 readonly = (curpg->flags & PG_RDONLY) 968 || (curpg->loan_count > 0) 969 || UVM_OBJ_NEEDS_WRITEFAULT(curpg->uobject); 970 971 (void) pmap_enter(ufi.orig_map->pmap, currva, 972 VM_PAGE_TO_PHYS(curpg), 973 readonly ? 974 enter_prot & ~VM_PROT_WRITE : 975 enter_prot & MASK(ufi.entry), 976 PMAP_CANFAIL | 977 (wired ? PMAP_WIRED : 0)); 978 979 /* 980 * NOTE: page can't be PG_WANTED or PG_RELEASED 981 * because we've held the lock the whole time 982 * we've had the handle. 983 */ 984 985 curpg->flags &= ~(PG_BUSY); 986 UVM_PAGE_OWN(curpg, NULL); 987 } 988 pmap_update(ufi.orig_map->pmap); 989 } 990 } else { 991 uobjpage = NULL; 992 } 993 994 /* locked (shadowed): maps(read), amap */ 995 /* locked (!shadowed): maps(read), amap(if there), 996 uobj(if !null), uobjpage(if !null) */ 997 998 /* 999 * note that at this point we are done with any front or back pages. 1000 * we are now going to focus on the center page (i.e. the one we've 1001 * faulted on). if we have faulted on the top (anon) layer 1002 * [i.e. case 1], then the anon we want is anons[centeridx] (we have 1003 * not touched it yet). if we have faulted on the bottom (uobj) 1004 * layer [i.e. case 2] and the page was both present and available, 1005 * then we've got a pointer to it as "uobjpage" and we've already 1006 * made it BUSY. 1007 */ 1008 1009 /* 1010 * there are four possible cases we must address: 1A, 1B, 2A, and 2B 1011 */ 1012 1013 /* 1014 * redirect case 2: if we are not shadowed, go to case 2. 1015 */ 1016 1017 if (shadowed == FALSE) 1018 goto Case2; 1019 1020 /* locked: maps(read), amap */ 1021 1022 /* 1023 * handle case 1: fault on an anon in our amap 1024 */ 1025 1026 anon = anons[centeridx]; 1027 UVMHIST_LOG(maphist, " case 1 fault: anon=0x%x", anon, 0,0,0); 1028 simple_lock(&anon->an_lock); 1029 1030 /* locked: maps(read), amap, anon */ 1031 1032 /* 1033 * no matter if we have case 1A or case 1B we are going to need to 1034 * have the anon's memory resident. ensure that now. 1035 */ 1036 1037 /* 1038 * let uvmfault_anonget do the dirty work. 1039 * if it fails (!OK) it will unlock everything for us. 1040 * if it succeeds, locks are still valid and locked. 1041 * also, if it is OK, then the anon's page is on the queues. 1042 * if the page is on loan from a uvm_object, then anonget will 1043 * lock that object for us if it does not fail. 1044 */ 1045 1046 error = uvmfault_anonget(&ufi, amap, anon); 1047 switch (error) { 1048 case 0: 1049 break; 1050 1051 case ERESTART: 1052 goto ReFault; 1053 1054 case EAGAIN: 1055 tsleep(&lbolt, PVM, "fltagain1", 0); 1056 goto ReFault; 1057 1058 default: 1059 return error; 1060 } 1061 1062 /* 1063 * uobj is non null if the page is on loan from an object (i.e. uobj) 1064 */ 1065 1066 uobj = anon->an_page->uobject; /* locked by anonget if !NULL */ 1067 1068 /* locked: maps(read), amap, anon, uobj(if one) */ 1069 1070 /* 1071 * special handling for loaned pages 1072 */ 1073 1074 if (anon->an_page->loan_count) { 1075 1076 if (!cow_now) { 1077 1078 /* 1079 * for read faults on loaned pages we just cap the 1080 * protection at read-only. 1081 */ 1082 1083 enter_prot = enter_prot & ~VM_PROT_WRITE; 1084 1085 } else { 1086 /* 1087 * note that we can't allow writes into a loaned page! 1088 * 1089 * if we have a write fault on a loaned page in an 1090 * anon then we need to look at the anon's ref count. 1091 * if it is greater than one then we are going to do 1092 * a normal copy-on-write fault into a new anon (this 1093 * is not a problem). however, if the reference count 1094 * is one (a case where we would normally allow a 1095 * write directly to the page) then we need to kill 1096 * the loan before we continue. 1097 */ 1098 1099 /* >1 case is already ok */ 1100 if (anon->an_ref == 1) { 1101 1102 /* get new un-owned replacement page */ 1103 pg = uvm_pagealloc(NULL, 0, NULL, 0); 1104 if (pg == NULL) { 1105 uvmfault_unlockall(&ufi, amap, uobj, 1106 anon); 1107 uvm_wait("flt_noram2"); 1108 goto ReFault; 1109 } 1110 1111 /* 1112 * copy data, kill loan, and drop uobj lock 1113 * (if any) 1114 */ 1115 /* copy old -> new */ 1116 uvm_pagecopy(anon->an_page, pg); 1117 1118 /* force reload */ 1119 pmap_page_protect(anon->an_page, 1120 VM_PROT_NONE); 1121 uvm_lock_pageq(); /* KILL loan */ 1122 1123 anon->an_page->uanon = NULL; 1124 /* in case we owned */ 1125 anon->an_page->pqflags &= ~PQ_ANON; 1126 1127 if (uobj) { 1128 /* if we were receiver of loan */ 1129 anon->an_page->loan_count--; 1130 } else { 1131 /* 1132 * we were the lender (A->K); need 1133 * to remove the page from pageq's. 1134 */ 1135 uvm_pagedequeue(anon->an_page); 1136 } 1137 1138 uvm_pageactivate(pg); 1139 uvm_unlock_pageq(); 1140 if (uobj) { 1141 simple_unlock(&uobj->vmobjlock); 1142 uobj = NULL; 1143 } 1144 1145 /* install new page in anon */ 1146 anon->an_page = pg; 1147 pg->uanon = anon; 1148 pg->pqflags |= PQ_ANON; 1149 pg->flags &= ~(PG_BUSY|PG_FAKE); 1150 UVM_PAGE_OWN(pg, NULL); 1151 1152 /* done! */ 1153 } /* ref == 1 */ 1154 } /* write fault */ 1155 } /* loan count */ 1156 1157 /* 1158 * if we are case 1B then we will need to allocate a new blank 1159 * anon to transfer the data into. note that we have a lock 1160 * on anon, so no one can busy or release the page until we are done. 1161 * also note that the ref count can't drop to zero here because 1162 * it is > 1 and we are only dropping one ref. 1163 * 1164 * in the (hopefully very rare) case that we are out of RAM we 1165 * will unlock, wait for more RAM, and refault. 1166 * 1167 * if we are out of anon VM we kill the process (XXX: could wait?). 1168 */ 1169 1170 if (cow_now && anon->an_ref > 1) { 1171 1172 UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0); 1173 uvmexp.flt_acow++; 1174 oanon = anon; /* oanon = old, locked anon */ 1175 anon = uvm_analloc(); 1176 if (anon) { 1177 /* new anon is locked! */ 1178 pg = uvm_pagealloc(NULL, 0, anon, 0); 1179 } 1180 1181 /* check for out of RAM */ 1182 if (anon == NULL || pg == NULL) { 1183 if (anon) { 1184 anon->an_ref--; 1185 simple_unlock(&anon->an_lock); 1186 uvm_anfree(anon); 1187 } 1188 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1189 if (!uvm_reclaimable()) { 1190 UVMHIST_LOG(maphist, 1191 "<- failed. out of VM",0,0,0,0); 1192 uvmexp.fltnoanon++; 1193 return ENOMEM; 1194 } 1195 1196 uvmexp.fltnoram++; 1197 uvm_wait("flt_noram3"); /* out of RAM, wait for more */ 1198 goto ReFault; 1199 } 1200 1201 /* got all resources, replace anon with nanon */ 1202 uvm_pagecopy(oanon->an_page, pg); 1203 uvm_lock_pageq(); 1204 uvm_pageactivate(pg); 1205 pg->flags &= ~(PG_BUSY|PG_FAKE); 1206 uvm_unlock_pageq(); 1207 UVM_PAGE_OWN(pg, NULL); 1208 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start, 1209 anon, TRUE); 1210 1211 /* deref: can not drop to zero here by defn! */ 1212 oanon->an_ref--; 1213 1214 /* 1215 * note: oanon is still locked, as is the new anon. we 1216 * need to check for this later when we unlock oanon; if 1217 * oanon != anon, we'll have to unlock anon, too. 1218 */ 1219 1220 } else { 1221 1222 uvmexp.flt_anon++; 1223 oanon = anon; /* old, locked anon is same as anon */ 1224 pg = anon->an_page; 1225 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */ 1226 enter_prot = enter_prot & ~VM_PROT_WRITE; 1227 1228 } 1229 1230 /* locked: maps(read), amap, oanon, anon (if different from oanon) */ 1231 1232 /* 1233 * now map the page in. 1234 */ 1235 1236 UVMHIST_LOG(maphist, " MAPPING: anon: pm=0x%x, va=0x%x, pg=0x%x", 1237 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, 0); 1238 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1239 enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) 1240 != 0) { 1241 1242 /* 1243 * No need to undo what we did; we can simply think of 1244 * this as the pmap throwing away the mapping information. 1245 * 1246 * We do, however, have to go through the ReFault path, 1247 * as the map may change while we're asleep. 1248 */ 1249 1250 if (anon != oanon) 1251 simple_unlock(&anon->an_lock); 1252 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1253 if (!uvm_reclaimable()) { 1254 UVMHIST_LOG(maphist, 1255 "<- failed. out of VM",0,0,0,0); 1256 /* XXX instrumentation */ 1257 return ENOMEM; 1258 } 1259 /* XXX instrumentation */ 1260 uvm_wait("flt_pmfail1"); 1261 goto ReFault; 1262 } 1263 1264 /* 1265 * ... update the page queues. 1266 */ 1267 1268 uvm_lock_pageq(); 1269 if (wire_fault) { 1270 uvm_pagewire(pg); 1271 1272 /* 1273 * since the now-wired page cannot be paged out, 1274 * release its swap resources for others to use. 1275 * since an anon with no swap cannot be PG_CLEAN, 1276 * clear its clean flag now. 1277 */ 1278 1279 pg->flags &= ~(PG_CLEAN); 1280 uvm_anon_dropswap(anon); 1281 } else { 1282 uvm_pageactivate(pg); 1283 } 1284 uvm_unlock_pageq(); 1285 1286 /* 1287 * done case 1! finish up by unlocking everything and returning success 1288 */ 1289 1290 if (anon != oanon) 1291 simple_unlock(&anon->an_lock); 1292 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1293 pmap_update(ufi.orig_map->pmap); 1294 return 0; 1295 1296 Case2: 1297 /* 1298 * handle case 2: faulting on backing object or zero fill 1299 */ 1300 1301 /* 1302 * locked: 1303 * maps(read), amap(if there), uobj(if !null), uobjpage(if !null) 1304 */ 1305 1306 /* 1307 * note that uobjpage can not be PGO_DONTCARE at this point. we now 1308 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we 1309 * have a backing object, check and see if we are going to promote 1310 * the data up to an anon during the fault. 1311 */ 1312 1313 if (uobj == NULL) { 1314 uobjpage = PGO_DONTCARE; 1315 promote = TRUE; /* always need anon here */ 1316 } else { 1317 KASSERT(uobjpage != PGO_DONTCARE); 1318 promote = cow_now && UVM_ET_ISCOPYONWRITE(ufi.entry); 1319 } 1320 UVMHIST_LOG(maphist, " case 2 fault: promote=%d, zfill=%d", 1321 promote, (uobj == NULL), 0,0); 1322 1323 /* 1324 * if uobjpage is not null then we do not need to do I/O to get the 1325 * uobjpage. 1326 * 1327 * if uobjpage is null, then we need to unlock and ask the pager to 1328 * get the data for us. once we have the data, we need to reverify 1329 * the state the world. we are currently not holding any resources. 1330 */ 1331 1332 if (uobjpage) { 1333 /* update rusage counters */ 1334 curproc->p_stats->p_ru.ru_minflt++; 1335 } else { 1336 /* update rusage counters */ 1337 curproc->p_stats->p_ru.ru_majflt++; 1338 1339 /* locked: maps(read), amap(if there), uobj */ 1340 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1341 /* locked: uobj */ 1342 1343 uvmexp.fltget++; 1344 gotpages = 1; 1345 uoff = (ufi.orig_rvaddr - ufi.entry->start) + ufi.entry->offset; 1346 error = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages, 1347 0, access_type & MASK(ufi.entry), ufi.entry->advice, 1348 PGO_SYNCIO); 1349 /* locked: uobjpage(if no error) */ 1350 1351 /* 1352 * recover from I/O 1353 */ 1354 1355 if (error) { 1356 if (error == EAGAIN) { 1357 UVMHIST_LOG(maphist, 1358 " pgo_get says TRY AGAIN!",0,0,0,0); 1359 tsleep(&lbolt, PVM, "fltagain2", 0); 1360 goto ReFault; 1361 } 1362 1363 UVMHIST_LOG(maphist, "<- pgo_get failed (code %d)", 1364 error, 0,0,0); 1365 return error; 1366 } 1367 1368 /* locked: uobjpage */ 1369 1370 uvm_lock_pageq(); 1371 uvm_pageactivate(uobjpage); 1372 uvm_unlock_pageq(); 1373 1374 /* 1375 * re-verify the state of the world by first trying to relock 1376 * the maps. always relock the object. 1377 */ 1378 1379 locked = uvmfault_relock(&ufi); 1380 if (locked && amap) 1381 amap_lock(amap); 1382 simple_lock(&uobj->vmobjlock); 1383 1384 /* locked(locked): maps(read), amap(if !null), uobj, uobjpage */ 1385 /* locked(!locked): uobj, uobjpage */ 1386 1387 /* 1388 * verify that the page has not be released and re-verify 1389 * that amap slot is still free. if there is a problem, 1390 * we unlock and clean up. 1391 */ 1392 1393 if ((uobjpage->flags & PG_RELEASED) != 0 || 1394 (locked && amap && 1395 amap_lookup(&ufi.entry->aref, 1396 ufi.orig_rvaddr - ufi.entry->start))) { 1397 if (locked) 1398 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1399 locked = FALSE; 1400 } 1401 1402 /* 1403 * didn't get the lock? release the page and retry. 1404 */ 1405 1406 if (locked == FALSE) { 1407 UVMHIST_LOG(maphist, 1408 " wasn't able to relock after fault: retry", 1409 0,0,0,0); 1410 if (uobjpage->flags & PG_WANTED) 1411 wakeup(uobjpage); 1412 if (uobjpage->flags & PG_RELEASED) { 1413 uvmexp.fltpgrele++; 1414 uvm_pagefree(uobjpage); 1415 goto ReFault; 1416 } 1417 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1418 UVM_PAGE_OWN(uobjpage, NULL); 1419 simple_unlock(&uobj->vmobjlock); 1420 goto ReFault; 1421 } 1422 1423 /* 1424 * we have the data in uobjpage which is busy and 1425 * not released. we are holding object lock (so the page 1426 * can't be released on us). 1427 */ 1428 1429 /* locked: maps(read), amap(if !null), uobj, uobjpage */ 1430 } 1431 1432 /* 1433 * locked: 1434 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj) 1435 */ 1436 1437 /* 1438 * notes: 1439 * - at this point uobjpage can not be NULL 1440 * - at this point uobjpage can not be PG_RELEASED (since we checked 1441 * for it above) 1442 * - at this point uobjpage could be PG_WANTED (handle later) 1443 */ 1444 1445 KASSERT(uobj == NULL || !UVM_OBJ_IS_CLEAN(uobjpage->uobject) || 1446 (uobjpage->flags & PG_CLEAN) != 0); 1447 if (promote == FALSE) { 1448 1449 /* 1450 * we are not promoting. if the mapping is COW ensure that we 1451 * don't give more access than we should (e.g. when doing a read 1452 * fault on a COPYONWRITE mapping we want to map the COW page in 1453 * R/O even though the entry protection could be R/W). 1454 * 1455 * set "pg" to the page we want to map in (uobjpage, usually) 1456 */ 1457 1458 /* no anon in this case. */ 1459 anon = NULL; 1460 1461 uvmexp.flt_obj++; 1462 if (UVM_ET_ISCOPYONWRITE(ufi.entry) || 1463 UVM_OBJ_NEEDS_WRITEFAULT(uobjpage->uobject)) 1464 enter_prot &= ~VM_PROT_WRITE; 1465 pg = uobjpage; /* map in the actual object */ 1466 1467 /* assert(uobjpage != PGO_DONTCARE) */ 1468 1469 /* 1470 * we are faulting directly on the page. be careful 1471 * about writing to loaned pages... 1472 */ 1473 1474 if (uobjpage->loan_count) { 1475 if (!cow_now) { 1476 /* read fault: cap the protection at readonly */ 1477 /* cap! */ 1478 enter_prot = enter_prot & ~VM_PROT_WRITE; 1479 } else { 1480 /* write fault: must break the loan here */ 1481 1482 pg = uvm_loanbreak(uobjpage); 1483 if (pg == NULL) { 1484 1485 /* 1486 * drop ownership of page, it can't 1487 * be released 1488 */ 1489 1490 if (uobjpage->flags & PG_WANTED) 1491 wakeup(uobjpage); 1492 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1493 UVM_PAGE_OWN(uobjpage, NULL); 1494 1495 uvmfault_unlockall(&ufi, amap, uobj, 1496 NULL); 1497 UVMHIST_LOG(maphist, 1498 " out of RAM breaking loan, waiting", 1499 0,0,0,0); 1500 uvmexp.fltnoram++; 1501 uvm_wait("flt_noram4"); 1502 goto ReFault; 1503 } 1504 uobjpage = pg; 1505 } 1506 } 1507 } else { 1508 1509 /* 1510 * if we are going to promote the data to an anon we 1511 * allocate a blank anon here and plug it into our amap. 1512 */ 1513 #if DIAGNOSTIC 1514 if (amap == NULL) 1515 panic("uvm_fault: want to promote data, but no anon"); 1516 #endif 1517 1518 anon = uvm_analloc(); 1519 if (anon) { 1520 1521 /* 1522 * The new anon is locked. 1523 * 1524 * In `Fill in data...' below, if 1525 * uobjpage == PGO_DONTCARE, we want 1526 * a zero'd, dirty page, so have 1527 * uvm_pagealloc() do that for us. 1528 */ 1529 1530 pg = uvm_pagealloc(NULL, 0, anon, 1531 (uobjpage == PGO_DONTCARE) ? UVM_PGA_ZERO : 0); 1532 } 1533 1534 /* 1535 * out of memory resources? 1536 */ 1537 1538 if (anon == NULL || pg == NULL) { 1539 if (anon != NULL) { 1540 anon->an_ref--; 1541 simple_unlock(&anon->an_lock); 1542 uvm_anfree(anon); 1543 } 1544 1545 /* 1546 * arg! must unbusy our page and fail or sleep. 1547 */ 1548 1549 if (uobjpage != PGO_DONTCARE) { 1550 if (uobjpage->flags & PG_WANTED) 1551 /* still holding object lock */ 1552 wakeup(uobjpage); 1553 1554 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1555 UVM_PAGE_OWN(uobjpage, NULL); 1556 } 1557 1558 /* unlock and fail ... */ 1559 uvmfault_unlockall(&ufi, amap, uobj, NULL); 1560 if (!uvm_reclaimable()) { 1561 UVMHIST_LOG(maphist, " promote: out of VM", 1562 0,0,0,0); 1563 uvmexp.fltnoanon++; 1564 return ENOMEM; 1565 } 1566 1567 UVMHIST_LOG(maphist, " out of RAM, waiting for more", 1568 0,0,0,0); 1569 uvmexp.fltnoram++; 1570 uvm_wait("flt_noram5"); 1571 goto ReFault; 1572 } 1573 1574 /* 1575 * fill in the data 1576 */ 1577 1578 if (uobjpage != PGO_DONTCARE) { 1579 uvmexp.flt_prcopy++; 1580 /* copy page [pg now dirty] */ 1581 uvm_pagecopy(uobjpage, pg); 1582 1583 /* 1584 * promote to shared amap? make sure all sharing 1585 * procs see it 1586 */ 1587 1588 if ((amap_flags(amap) & AMAP_SHARED) != 0) { 1589 pmap_page_protect(uobjpage, VM_PROT_NONE); 1590 /* 1591 * XXX: PAGE MIGHT BE WIRED! 1592 */ 1593 } 1594 1595 /* 1596 * dispose of uobjpage. it can't be PG_RELEASED 1597 * since we still hold the object lock. 1598 * drop handle to uobj as well. 1599 */ 1600 1601 if (uobjpage->flags & PG_WANTED) 1602 /* still have the obj lock */ 1603 wakeup(uobjpage); 1604 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1605 UVM_PAGE_OWN(uobjpage, NULL); 1606 simple_unlock(&uobj->vmobjlock); 1607 uobj = NULL; 1608 1609 UVMHIST_LOG(maphist, 1610 " promote uobjpage 0x%x to anon/page 0x%x/0x%x", 1611 uobjpage, anon, pg, 0); 1612 1613 } else { 1614 uvmexp.flt_przero++; 1615 1616 /* 1617 * Page is zero'd and marked dirty by uvm_pagealloc() 1618 * above. 1619 */ 1620 1621 UVMHIST_LOG(maphist," zero fill anon/page 0x%x/0%x", 1622 anon, pg, 0, 0); 1623 } 1624 amap_add(&ufi.entry->aref, ufi.orig_rvaddr - ufi.entry->start, 1625 anon, FALSE); 1626 } 1627 1628 /* 1629 * locked: 1630 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj), 1631 * anon(if !null), pg(if anon) 1632 * 1633 * note: pg is either the uobjpage or the new page in the new anon 1634 */ 1635 1636 /* 1637 * all resources are present. we can now map it in and free our 1638 * resources. 1639 */ 1640 1641 UVMHIST_LOG(maphist, 1642 " MAPPING: case2: pm=0x%x, va=0x%x, pg=0x%x, promote=%d", 1643 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, promote); 1644 KASSERT((access_type & VM_PROT_WRITE) == 0 || 1645 (pg->flags & PG_RDONLY) == 0); 1646 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1647 pg->flags & PG_RDONLY ? enter_prot & ~VM_PROT_WRITE : enter_prot, 1648 access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) != 0) { 1649 1650 /* 1651 * No need to undo what we did; we can simply think of 1652 * this as the pmap throwing away the mapping information. 1653 * 1654 * We do, however, have to go through the ReFault path, 1655 * as the map may change while we're asleep. 1656 */ 1657 1658 if (pg->flags & PG_WANTED) 1659 wakeup(pg); 1660 1661 /* 1662 * note that pg can't be PG_RELEASED since we did not drop 1663 * the object lock since the last time we checked. 1664 */ 1665 1666 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); 1667 UVM_PAGE_OWN(pg, NULL); 1668 uvmfault_unlockall(&ufi, amap, uobj, anon); 1669 if (!uvm_reclaimable()) { 1670 UVMHIST_LOG(maphist, 1671 "<- failed. out of VM",0,0,0,0); 1672 /* XXX instrumentation */ 1673 return ENOMEM; 1674 } 1675 /* XXX instrumentation */ 1676 uvm_wait("flt_pmfail2"); 1677 goto ReFault; 1678 } 1679 1680 uvm_lock_pageq(); 1681 if (wire_fault) { 1682 uvm_pagewire(pg); 1683 if (pg->pqflags & PQ_AOBJ) { 1684 1685 /* 1686 * since the now-wired page cannot be paged out, 1687 * release its swap resources for others to use. 1688 * since an aobj page with no swap cannot be PG_CLEAN, 1689 * clear its clean flag now. 1690 */ 1691 1692 pg->flags &= ~(PG_CLEAN); 1693 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT); 1694 } 1695 } else { 1696 uvm_pageactivate(pg); 1697 } 1698 uvm_unlock_pageq(); 1699 if (pg->flags & PG_WANTED) 1700 wakeup(pg); 1701 1702 /* 1703 * note that pg can't be PG_RELEASED since we did not drop the object 1704 * lock since the last time we checked. 1705 */ 1706 1707 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); 1708 UVM_PAGE_OWN(pg, NULL); 1709 uvmfault_unlockall(&ufi, amap, uobj, anon); 1710 pmap_update(ufi.orig_map->pmap); 1711 UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0); 1712 return 0; 1713 } 1714 1715 /* 1716 * uvm_fault_wire: wire down a range of virtual addresses in a map. 1717 * 1718 * => map may be read-locked by caller, but MUST NOT be write-locked. 1719 * => if map is read-locked, any operations which may cause map to 1720 * be write-locked in uvm_fault() must be taken care of by 1721 * the caller. See uvm_map_pageable(). 1722 */ 1723 1724 int 1725 uvm_fault_wire(struct vm_map *map, vaddr_t start, vaddr_t end, 1726 vm_fault_t fault_type, vm_prot_t access_type) 1727 { 1728 vaddr_t va; 1729 int error; 1730 1731 /* 1732 * now fault it in a page at a time. if the fault fails then we have 1733 * to undo what we have done. note that in uvm_fault VM_PROT_NONE 1734 * is replaced with the max protection if fault_type is VM_FAULT_WIRE. 1735 */ 1736 1737 /* 1738 * XXX work around overflowing a vaddr_t. this prevents us from 1739 * wiring the last page in the address space, though. 1740 */ 1741 if (start > end) { 1742 return EFAULT; 1743 } 1744 1745 for (va = start ; va < end ; va += PAGE_SIZE) { 1746 error = uvm_fault(map, va, fault_type, access_type); 1747 if (error) { 1748 if (va != start) { 1749 uvm_fault_unwire(map, start, va); 1750 } 1751 return error; 1752 } 1753 } 1754 return 0; 1755 } 1756 1757 /* 1758 * uvm_fault_unwire(): unwire range of virtual space. 1759 */ 1760 1761 void 1762 uvm_fault_unwire(struct vm_map *map, vaddr_t start, vaddr_t end) 1763 { 1764 vm_map_lock_read(map); 1765 uvm_fault_unwire_locked(map, start, end); 1766 vm_map_unlock_read(map); 1767 } 1768 1769 /* 1770 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire(). 1771 * 1772 * => map must be at least read-locked. 1773 */ 1774 1775 void 1776 uvm_fault_unwire_locked(struct vm_map *map, vaddr_t start, vaddr_t end) 1777 { 1778 struct vm_map_entry *entry; 1779 pmap_t pmap = vm_map_pmap(map); 1780 vaddr_t va; 1781 paddr_t pa; 1782 struct vm_page *pg; 1783 1784 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0); 1785 1786 /* 1787 * we assume that the area we are unwiring has actually been wired 1788 * in the first place. this means that we should be able to extract 1789 * the PAs from the pmap. we also lock out the page daemon so that 1790 * we can call uvm_pageunwire. 1791 */ 1792 1793 uvm_lock_pageq(); 1794 1795 /* 1796 * find the beginning map entry for the region. 1797 */ 1798 1799 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map)); 1800 if (uvm_map_lookup_entry(map, start, &entry) == FALSE) 1801 panic("uvm_fault_unwire_locked: address not in map"); 1802 1803 for (va = start; va < end; va += PAGE_SIZE) { 1804 if (pmap_extract(pmap, va, &pa) == FALSE) 1805 continue; 1806 1807 /* 1808 * find the map entry for the current address. 1809 */ 1810 1811 KASSERT(va >= entry->start); 1812 while (va >= entry->end) { 1813 KASSERT(entry->next != &map->header && 1814 entry->next->start <= entry->end); 1815 entry = entry->next; 1816 } 1817 1818 /* 1819 * if the entry is no longer wired, tell the pmap. 1820 */ 1821 1822 if (VM_MAPENT_ISWIRED(entry) == 0) 1823 pmap_unwire(pmap, va); 1824 1825 pg = PHYS_TO_VM_PAGE(pa); 1826 if (pg) 1827 uvm_pageunwire(pg); 1828 } 1829 1830 uvm_unlock_pageq(); 1831 } 1832