1 /* $NetBSD: uvm_fault.c,v 1.111 2006/04/11 09:28:14 yamt Exp $ */ 2 3 /* 4 * 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by Charles D. Cranor and 19 * Washington University. 20 * 4. The name of the author may not be used to endorse or promote products 21 * derived from this software without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 24 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 25 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 26 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 27 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 28 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 32 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 * 34 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp 35 */ 36 37 /* 38 * uvm_fault.c: fault handler 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.111 2006/04/11 09:28:14 yamt Exp $"); 43 44 #include "opt_uvmhist.h" 45 46 #include <sys/param.h> 47 #include <sys/systm.h> 48 #include <sys/kernel.h> 49 #include <sys/proc.h> 50 #include <sys/malloc.h> 51 #include <sys/mman.h> 52 #include <sys/user.h> 53 #include <sys/vnode.h> 54 55 #include <uvm/uvm.h> 56 57 /* 58 * 59 * a word on page faults: 60 * 61 * types of page faults we handle: 62 * 63 * CASE 1: upper layer faults CASE 2: lower layer faults 64 * 65 * CASE 1A CASE 1B CASE 2A CASE 2B 66 * read/write1 write>1 read/write +-cow_write/zero 67 * | | | | 68 * +--|--+ +--|--+ +-----+ + | + | +-----+ 69 * amap | V | | ----------->new| | | | ^ | 70 * +-----+ +-----+ +-----+ + | + | +--|--+ 71 * | | | 72 * +-----+ +-----+ +--|--+ | +--|--+ 73 * uobj | d/c | | d/c | | V | +----| | 74 * +-----+ +-----+ +-----+ +-----+ 75 * 76 * d/c = don't care 77 * 78 * case [0]: layerless fault 79 * no amap or uobj is present. this is an error. 80 * 81 * case [1]: upper layer fault [anon active] 82 * 1A: [read] or [write with anon->an_ref == 1] 83 * I/O takes place in top level anon and uobj is not touched. 84 * 1B: [write with anon->an_ref > 1] 85 * new anon is alloc'd and data is copied off ["COW"] 86 * 87 * case [2]: lower layer fault [uobj] 88 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area] 89 * I/O takes place directly in object. 90 * 2B: [write to copy_on_write] or [read on NULL uobj] 91 * data is "promoted" from uobj to a new anon. 92 * if uobj is null, then we zero fill. 93 * 94 * we follow the standard UVM locking protocol ordering: 95 * 96 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ) 97 * we hold a PG_BUSY page if we unlock for I/O 98 * 99 * 100 * the code is structured as follows: 101 * 102 * - init the "IN" params in the ufi structure 103 * ReFault: 104 * - do lookups [locks maps], check protection, handle needs_copy 105 * - check for case 0 fault (error) 106 * - establish "range" of fault 107 * - if we have an amap lock it and extract the anons 108 * - if sequential advice deactivate pages behind us 109 * - at the same time check pmap for unmapped areas and anon for pages 110 * that we could map in (and do map it if found) 111 * - check object for resident pages that we could map in 112 * - if (case 2) goto Case2 113 * - >>> handle case 1 114 * - ensure source anon is resident in RAM 115 * - if case 1B alloc new anon and copy from source 116 * - map the correct page in 117 * Case2: 118 * - >>> handle case 2 119 * - ensure source page is resident (if uobj) 120 * - if case 2B alloc new anon and copy from source (could be zero 121 * fill if uobj == NULL) 122 * - map the correct page in 123 * - done! 124 * 125 * note on paging: 126 * if we have to do I/O we place a PG_BUSY page in the correct object, 127 * unlock everything, and do the I/O. when I/O is done we must reverify 128 * the state of the world before assuming that our data structures are 129 * valid. [because mappings could change while the map is unlocked] 130 * 131 * alternative 1: unbusy the page in question and restart the page fault 132 * from the top (ReFault). this is easy but does not take advantage 133 * of the information that we already have from our previous lookup, 134 * although it is possible that the "hints" in the vm_map will help here. 135 * 136 * alternative 2: the system already keeps track of a "version" number of 137 * a map. [i.e. every time you write-lock a map (e.g. to change a 138 * mapping) you bump the version number up by one...] so, we can save 139 * the version number of the map before we release the lock and start I/O. 140 * then when I/O is done we can relock and check the version numbers 141 * to see if anything changed. this might save us some over 1 because 142 * we don't have to unbusy the page and may be less compares(?). 143 * 144 * alternative 3: put in backpointers or a way to "hold" part of a map 145 * in place while I/O is in progress. this could be complex to 146 * implement (especially with structures like amap that can be referenced 147 * by multiple map entries, and figuring out what should wait could be 148 * complex as well...). 149 * 150 * given that we are not currently multiprocessor or multithreaded we might 151 * as well choose alternative 2 now. maybe alternative 3 would be useful 152 * in the future. XXX keep in mind for future consideration//rechecking. 153 */ 154 155 /* 156 * local data structures 157 */ 158 159 struct uvm_advice { 160 int advice; 161 int nback; 162 int nforw; 163 }; 164 165 /* 166 * page range array: 167 * note: index in array must match "advice" value 168 * XXX: borrowed numbers from freebsd. do they work well for us? 169 */ 170 171 static const struct uvm_advice uvmadvice[] = { 172 { MADV_NORMAL, 3, 4 }, 173 { MADV_RANDOM, 0, 0 }, 174 { MADV_SEQUENTIAL, 8, 7}, 175 }; 176 177 #define UVM_MAXRANGE 16 /* must be MAX() of nback+nforw+1 */ 178 179 /* 180 * private prototypes 181 */ 182 183 /* 184 * inline functions 185 */ 186 187 /* 188 * uvmfault_anonflush: try and deactivate pages in specified anons 189 * 190 * => does not have to deactivate page if it is busy 191 */ 192 193 static inline void 194 uvmfault_anonflush(struct vm_anon **anons, int n) 195 { 196 int lcv; 197 struct vm_page *pg; 198 199 for (lcv = 0 ; lcv < n ; lcv++) { 200 if (anons[lcv] == NULL) 201 continue; 202 simple_lock(&anons[lcv]->an_lock); 203 pg = anons[lcv]->an_page; 204 if (pg && (pg->flags & PG_BUSY) == 0 && pg->loan_count == 0) { 205 uvm_lock_pageq(); 206 if (pg->wire_count == 0) { 207 pmap_clear_reference(pg); 208 uvm_pagedeactivate(pg); 209 } 210 uvm_unlock_pageq(); 211 } 212 simple_unlock(&anons[lcv]->an_lock); 213 } 214 } 215 216 /* 217 * normal functions 218 */ 219 220 /* 221 * uvmfault_amapcopy: clear "needs_copy" in a map. 222 * 223 * => called with VM data structures unlocked (usually, see below) 224 * => we get a write lock on the maps and clear needs_copy for a VA 225 * => if we are out of RAM we sleep (waiting for more) 226 */ 227 228 static void 229 uvmfault_amapcopy(struct uvm_faultinfo *ufi) 230 { 231 for (;;) { 232 233 /* 234 * no mapping? give up. 235 */ 236 237 if (uvmfault_lookup(ufi, TRUE) == FALSE) 238 return; 239 240 /* 241 * copy if needed. 242 */ 243 244 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) 245 amap_copy(ufi->map, ufi->entry, AMAP_COPY_NOWAIT, 246 ufi->orig_rvaddr, ufi->orig_rvaddr + 1); 247 248 /* 249 * didn't work? must be out of RAM. unlock and sleep. 250 */ 251 252 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 253 uvmfault_unlockmaps(ufi, TRUE); 254 uvm_wait("fltamapcopy"); 255 continue; 256 } 257 258 /* 259 * got it! unlock and return. 260 */ 261 262 uvmfault_unlockmaps(ufi, TRUE); 263 return; 264 } 265 /*NOTREACHED*/ 266 } 267 268 /* 269 * uvmfault_anonget: get data in an anon into a non-busy, non-released 270 * page in that anon. 271 * 272 * => maps, amap, and anon locked by caller. 273 * => if we fail (result != 0) we unlock everything. 274 * => if we are successful, we return with everything still locked. 275 * => we don't move the page on the queues [gets moved later] 276 * => if we allocate a new page [we_own], it gets put on the queues. 277 * either way, the result is that the page is on the queues at return time 278 * => for pages which are on loan from a uvm_object (and thus are not 279 * owned by the anon): if successful, we return with the owning object 280 * locked. the caller must unlock this object when it unlocks everything 281 * else. 282 */ 283 284 int 285 uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap, 286 struct vm_anon *anon) 287 { 288 boolean_t we_own; /* we own anon's page? */ 289 boolean_t locked; /* did we relock? */ 290 struct vm_page *pg; 291 int error; 292 UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist); 293 294 LOCK_ASSERT(simple_lock_held(&anon->an_lock)); 295 296 error = 0; 297 uvmexp.fltanget++; 298 /* bump rusage counters */ 299 if (anon->an_page) 300 curproc->p_stats->p_ru.ru_minflt++; 301 else 302 curproc->p_stats->p_ru.ru_majflt++; 303 304 /* 305 * loop until we get it, or fail. 306 */ 307 308 for (;;) { 309 we_own = FALSE; /* TRUE if we set PG_BUSY on a page */ 310 pg = anon->an_page; 311 312 /* 313 * if there is a resident page and it is loaned, then anon 314 * may not own it. call out to uvm_anon_lockpage() to ensure 315 * the real owner of the page has been identified and locked. 316 */ 317 318 if (pg && pg->loan_count) 319 pg = uvm_anon_lockloanpg(anon); 320 321 /* 322 * page there? make sure it is not busy/released. 323 */ 324 325 if (pg) { 326 327 /* 328 * at this point, if the page has a uobject [meaning 329 * we have it on loan], then that uobject is locked 330 * by us! if the page is busy, we drop all the 331 * locks (including uobject) and try again. 332 */ 333 334 if ((pg->flags & PG_BUSY) == 0) { 335 UVMHIST_LOG(maphist, "<- OK",0,0,0,0); 336 return (0); 337 } 338 pg->flags |= PG_WANTED; 339 uvmexp.fltpgwait++; 340 341 /* 342 * the last unlock must be an atomic unlock+wait on 343 * the owner of page 344 */ 345 346 if (pg->uobject) { /* owner is uobject ? */ 347 uvmfault_unlockall(ufi, amap, NULL, anon); 348 UVMHIST_LOG(maphist, " unlock+wait on uobj",0, 349 0,0,0); 350 UVM_UNLOCK_AND_WAIT(pg, 351 &pg->uobject->vmobjlock, 352 FALSE, "anonget1",0); 353 } else { 354 /* anon owns page */ 355 uvmfault_unlockall(ufi, amap, NULL, NULL); 356 UVMHIST_LOG(maphist, " unlock+wait on anon",0, 357 0,0,0); 358 UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0, 359 "anonget2",0); 360 } 361 } else { 362 #if defined(VMSWAP) 363 364 /* 365 * no page, we must try and bring it in. 366 */ 367 368 pg = uvm_pagealloc(NULL, 0, anon, 0); 369 if (pg == NULL) { /* out of RAM. */ 370 uvmfault_unlockall(ufi, amap, NULL, anon); 371 uvmexp.fltnoram++; 372 UVMHIST_LOG(maphist, " noram -- UVM_WAIT",0, 373 0,0,0); 374 if (!uvm_reclaimable()) { 375 return ENOMEM; 376 } 377 uvm_wait("flt_noram1"); 378 } else { 379 /* we set the PG_BUSY bit */ 380 we_own = TRUE; 381 uvmfault_unlockall(ufi, amap, NULL, anon); 382 383 /* 384 * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN 385 * page into the uvm_swap_get function with 386 * all data structures unlocked. note that 387 * it is ok to read an_swslot here because 388 * we hold PG_BUSY on the page. 389 */ 390 uvmexp.pageins++; 391 error = uvm_swap_get(pg, anon->an_swslot, 392 PGO_SYNCIO); 393 394 /* 395 * we clean up after the i/o below in the 396 * "we_own" case 397 */ 398 } 399 #else /* defined(VMSWAP) */ 400 panic("%s: no page", __func__); 401 #endif /* defined(VMSWAP) */ 402 } 403 404 /* 405 * now relock and try again 406 */ 407 408 locked = uvmfault_relock(ufi); 409 if (locked && amap != NULL) { 410 amap_lock(amap); 411 } 412 if (locked || we_own) 413 simple_lock(&anon->an_lock); 414 415 /* 416 * if we own the page (i.e. we set PG_BUSY), then we need 417 * to clean up after the I/O. there are three cases to 418 * consider: 419 * [1] page released during I/O: free anon and ReFault. 420 * [2] I/O not OK. free the page and cause the fault 421 * to fail. 422 * [3] I/O OK! activate the page and sync with the 423 * non-we_own case (i.e. drop anon lock if not locked). 424 */ 425 426 if (we_own) { 427 #if defined(VMSWAP) 428 if (pg->flags & PG_WANTED) { 429 wakeup(pg); 430 } 431 if (error) { 432 433 /* 434 * remove the swap slot from the anon 435 * and mark the anon as having no real slot. 436 * don't free the swap slot, thus preventing 437 * it from being used again. 438 */ 439 440 if (anon->an_swslot > 0) 441 uvm_swap_markbad(anon->an_swslot, 1); 442 anon->an_swslot = SWSLOT_BAD; 443 444 if ((pg->flags & PG_RELEASED) != 0) 445 goto released; 446 447 /* 448 * note: page was never !PG_BUSY, so it 449 * can't be mapped and thus no need to 450 * pmap_page_protect it... 451 */ 452 453 uvm_lock_pageq(); 454 uvm_pagefree(pg); 455 uvm_unlock_pageq(); 456 457 if (locked) 458 uvmfault_unlockall(ufi, amap, NULL, 459 anon); 460 else 461 simple_unlock(&anon->an_lock); 462 UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0); 463 return error; 464 } 465 466 if ((pg->flags & PG_RELEASED) != 0) { 467 released: 468 KASSERT(anon->an_ref == 0); 469 470 /* 471 * released while we unlocked amap. 472 */ 473 474 if (locked) 475 uvmfault_unlockall(ufi, amap, NULL, 476 NULL); 477 478 uvm_anon_release(anon); 479 480 if (error) { 481 UVMHIST_LOG(maphist, 482 "<- ERROR/RELEASED", 0,0,0,0); 483 return error; 484 } 485 486 UVMHIST_LOG(maphist, "<- RELEASED", 0,0,0,0); 487 return ERESTART; 488 } 489 490 /* 491 * we've successfully read the page, activate it. 492 */ 493 494 uvm_lock_pageq(); 495 uvm_pageactivate(pg); 496 uvm_unlock_pageq(); 497 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE); 498 UVM_PAGE_OWN(pg, NULL); 499 if (!locked) 500 simple_unlock(&anon->an_lock); 501 #else /* defined(VMSWAP) */ 502 panic("%s: we_own", __func__); 503 #endif /* defined(VMSWAP) */ 504 } 505 506 /* 507 * we were not able to relock. restart fault. 508 */ 509 510 if (!locked) { 511 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 512 return (ERESTART); 513 } 514 515 /* 516 * verify no one has touched the amap and moved the anon on us. 517 */ 518 519 if (ufi != NULL && 520 amap_lookup(&ufi->entry->aref, 521 ufi->orig_rvaddr - ufi->entry->start) != anon) { 522 523 uvmfault_unlockall(ufi, amap, NULL, anon); 524 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 525 return (ERESTART); 526 } 527 528 /* 529 * try it again! 530 */ 531 532 uvmexp.fltanretry++; 533 continue; 534 } 535 /*NOTREACHED*/ 536 } 537 538 /* 539 * uvmfault_promote: promote data to a new anon. used for 1B and 2B. 540 * 541 * 1. allocate an anon and a page. 542 * 2. fill its contents. 543 * 3. put it into amap. 544 * 545 * => if we fail (result != 0) we unlock everything. 546 * => on success, return a new locked anon via 'nanon'. 547 * (*nanon)->an_page will be a resident, locked, dirty page. 548 */ 549 550 static int 551 uvmfault_promote(struct uvm_faultinfo *ufi, 552 struct vm_anon *oanon, 553 struct vm_page *uobjpage, 554 struct vm_anon **nanon, /* OUT: allocated anon */ 555 struct vm_anon **spare) 556 { 557 struct vm_amap *amap = ufi->entry->aref.ar_amap; 558 struct uvm_object *uobj; 559 struct vm_anon *anon; 560 struct vm_page *pg; 561 struct vm_page *opg; 562 int error; 563 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 564 565 if (oanon) { 566 /* anon COW */ 567 opg = oanon->an_page; 568 KASSERT(opg != NULL); 569 KASSERT(opg->uobject == NULL || opg->loan_count > 0); 570 } else if (uobjpage != PGO_DONTCARE) { 571 /* object-backed COW */ 572 opg = uobjpage; 573 } else { 574 /* ZFOD */ 575 opg = NULL; 576 } 577 if (opg != NULL) { 578 uobj = opg->uobject; 579 } else { 580 uobj = NULL; 581 } 582 583 KASSERT(amap != NULL); 584 KASSERT(uobjpage != NULL); 585 KASSERT(uobjpage == PGO_DONTCARE || (uobjpage->flags & PG_BUSY) != 0); 586 LOCK_ASSERT(simple_lock_held(&amap->am_l)); 587 LOCK_ASSERT(oanon == NULL || simple_lock_held(&oanon->an_lock)); 588 LOCK_ASSERT(uobj == NULL || simple_lock_held(&uobj->vmobjlock)); 589 LOCK_ASSERT(*spare == NULL || !simple_lock_held(&(*spare)->an_lock)); 590 591 if (*spare != NULL) { 592 anon = *spare; 593 *spare = NULL; 594 simple_lock(&anon->an_lock); 595 } else if (ufi->map != kernel_map) { 596 anon = uvm_analloc(); 597 } else { 598 UVMHIST_LOG(maphist, "kernel_map, unlock and retry", 0,0,0,0); 599 600 /* 601 * we can't allocate anons with kernel_map locked. 602 */ 603 604 uvm_page_unbusy(&uobjpage, 1); 605 uvmfault_unlockall(ufi, amap, uobj, oanon); 606 607 *spare = uvm_analloc(); 608 if (*spare == NULL) { 609 goto nomem; 610 } 611 simple_unlock(&(*spare)->an_lock); 612 error = ERESTART; 613 goto done; 614 } 615 if (anon) { 616 617 /* 618 * The new anon is locked. 619 * 620 * if opg == NULL, we want a zero'd, dirty page, 621 * so have uvm_pagealloc() do that for us. 622 */ 623 624 pg = uvm_pagealloc(NULL, 0, anon, 625 (opg == NULL) ? UVM_PGA_ZERO : 0); 626 } else { 627 pg = NULL; 628 } 629 630 /* 631 * out of memory resources? 632 */ 633 634 if (pg == NULL) { 635 /* save anon for the next try. */ 636 if (anon != NULL) { 637 simple_unlock(&anon->an_lock); 638 *spare = anon; 639 } 640 641 /* unlock and fail ... */ 642 uvm_page_unbusy(&uobjpage, 1); 643 uvmfault_unlockall(ufi, amap, uobj, oanon); 644 nomem: 645 if (!uvm_reclaimable()) { 646 UVMHIST_LOG(maphist, "out of VM", 0,0,0,0); 647 uvmexp.fltnoanon++; 648 error = ENOMEM; 649 goto done; 650 } 651 652 UVMHIST_LOG(maphist, "out of RAM, waiting for more", 0,0,0,0); 653 uvmexp.fltnoram++; 654 uvm_wait("flt_noram5"); 655 error = ERESTART; 656 goto done; 657 } 658 659 /* copy page [pg now dirty] */ 660 if (opg) { 661 uvm_pagecopy(opg, pg); 662 } 663 664 amap_add(&ufi->entry->aref, ufi->orig_rvaddr - ufi->entry->start, anon, 665 oanon != NULL); 666 667 *nanon = anon; 668 error = 0; 669 done: 670 return error; 671 } 672 673 674 /* 675 * F A U L T - m a i n e n t r y p o i n t 676 */ 677 678 /* 679 * uvm_fault: page fault handler 680 * 681 * => called from MD code to resolve a page fault 682 * => VM data structures usually should be unlocked. however, it is 683 * possible to call here with the main map locked if the caller 684 * gets a write lock, sets it recusive, and then calls us (c.f. 685 * uvm_map_pageable). this should be avoided because it keeps 686 * the map locked off during I/O. 687 * => MUST NEVER BE CALLED IN INTERRUPT CONTEXT 688 */ 689 690 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 691 ~VM_PROT_WRITE : VM_PROT_ALL) 692 693 /* fault_flag values passed from uvm_fault_wire to uvm_fault_internal */ 694 #define UVM_FAULT_WIRE 1 695 #define UVM_FAULT_WIREMAX 2 696 697 int 698 uvm_fault_internal(struct vm_map *orig_map, vaddr_t vaddr, 699 vm_prot_t access_type, int fault_flag) 700 { 701 struct uvm_faultinfo ufi; 702 vm_prot_t enter_prot, check_prot; 703 boolean_t wired, narrow, promote, locked, shadowed, wire_fault, cow_now; 704 int npages, nback, nforw, centeridx, error, lcv, gotpages; 705 vaddr_t startva, currva; 706 voff_t uoff; 707 struct vm_amap *amap; 708 struct uvm_object *uobj; 709 struct vm_anon *anons_store[UVM_MAXRANGE], **anons, *anon, *oanon; 710 struct vm_anon *anon_spare; 711 struct vm_page *pages[UVM_MAXRANGE], *pg, *uobjpage; 712 UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist); 713 714 UVMHIST_LOG(maphist, "(map=0x%x, vaddr=0x%x, at=%d, ff=%d)", 715 orig_map, vaddr, access_type, fault_flag); 716 717 anon = anon_spare = NULL; 718 pg = NULL; 719 720 uvmexp.faults++; /* XXX: locking? */ 721 722 /* 723 * init the IN parameters in the ufi 724 */ 725 726 ufi.orig_map = orig_map; 727 ufi.orig_rvaddr = trunc_page(vaddr); 728 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ 729 wire_fault = (fault_flag > 0); 730 if (wire_fault) 731 narrow = TRUE; /* don't look for neighborhood 732 * pages on wire */ 733 else 734 narrow = FALSE; /* normal fault */ 735 736 /* 737 * "goto ReFault" means restart the page fault from ground zero. 738 */ 739 ReFault: 740 741 /* 742 * lookup and lock the maps 743 */ 744 745 if (uvmfault_lookup(&ufi, FALSE) == FALSE) { 746 UVMHIST_LOG(maphist, "<- no mapping @ 0x%x", vaddr, 0,0,0); 747 error = EFAULT; 748 goto done; 749 } 750 /* locked: maps(read) */ 751 752 #ifdef DIAGNOSTIC 753 if ((ufi.map->flags & VM_MAP_PAGEABLE) == 0) { 754 printf("Page fault on non-pageable map:\n"); 755 printf("ufi.map = %p\n", ufi.map); 756 printf("ufi.orig_map = %p\n", ufi.orig_map); 757 printf("ufi.orig_rvaddr = 0x%lx\n", (u_long) ufi.orig_rvaddr); 758 panic("uvm_fault: (ufi.map->flags & VM_MAP_PAGEABLE) == 0"); 759 } 760 #endif 761 762 /* 763 * check protection 764 */ 765 766 check_prot = fault_flag == UVM_FAULT_WIREMAX ? 767 ufi.entry->max_protection : ufi.entry->protection; 768 if ((check_prot & access_type) != access_type) { 769 UVMHIST_LOG(maphist, 770 "<- protection failure (prot=0x%x, access=0x%x)", 771 ufi.entry->protection, access_type, 0, 0); 772 uvmfault_unlockmaps(&ufi, FALSE); 773 error = EACCES; 774 goto done; 775 } 776 777 /* 778 * "enter_prot" is the protection we want to enter the page in at. 779 * for certain pages (e.g. copy-on-write pages) this protection can 780 * be more strict than ufi.entry->protection. "wired" means either 781 * the entry is wired or we are fault-wiring the pg. 782 */ 783 784 enter_prot = ufi.entry->protection; 785 wired = VM_MAPENT_ISWIRED(ufi.entry) || wire_fault; 786 if (wired) { 787 access_type = enter_prot; /* full access for wired */ 788 cow_now = (check_prot & VM_PROT_WRITE) != 0; 789 } else { 790 cow_now = (access_type & VM_PROT_WRITE) != 0; 791 } 792 793 /* 794 * handle "needs_copy" case. if we need to copy the amap we will 795 * have to drop our readlock and relock it with a write lock. (we 796 * need a write lock to change anything in a map entry [e.g. 797 * needs_copy]). 798 */ 799 800 if (UVM_ET_ISNEEDSCOPY(ufi.entry)) { 801 KASSERT(fault_flag != UVM_FAULT_WIREMAX); 802 if (cow_now || (ufi.entry->object.uvm_obj == NULL)) { 803 /* need to clear */ 804 UVMHIST_LOG(maphist, 805 " need to clear needs_copy and refault",0,0,0,0); 806 uvmfault_unlockmaps(&ufi, FALSE); 807 uvmfault_amapcopy(&ufi); 808 uvmexp.fltamcopy++; 809 goto ReFault; 810 811 } else { 812 813 /* 814 * ensure that we pmap_enter page R/O since 815 * needs_copy is still true 816 */ 817 818 enter_prot &= ~VM_PROT_WRITE; 819 } 820 } 821 822 /* 823 * identify the players 824 */ 825 826 amap = ufi.entry->aref.ar_amap; /* top layer */ 827 uobj = ufi.entry->object.uvm_obj; /* bottom layer */ 828 829 /* 830 * check for a case 0 fault. if nothing backing the entry then 831 * error now. 832 */ 833 834 if (amap == NULL && uobj == NULL) { 835 uvmfault_unlockmaps(&ufi, FALSE); 836 UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0); 837 error = EFAULT; 838 goto done; 839 } 840 841 /* 842 * establish range of interest based on advice from mapper 843 * and then clip to fit map entry. note that we only want 844 * to do this the first time through the fault. if we 845 * ReFault we will disable this by setting "narrow" to true. 846 */ 847 848 if (narrow == FALSE) { 849 850 /* wide fault (!narrow) */ 851 KASSERT(uvmadvice[ufi.entry->advice].advice == 852 ufi.entry->advice); 853 nback = MIN(uvmadvice[ufi.entry->advice].nback, 854 (ufi.orig_rvaddr - ufi.entry->start) >> PAGE_SHIFT); 855 startva = ufi.orig_rvaddr - (nback << PAGE_SHIFT); 856 nforw = MIN(uvmadvice[ufi.entry->advice].nforw, 857 ((ufi.entry->end - ufi.orig_rvaddr) >> 858 PAGE_SHIFT) - 1); 859 /* 860 * note: "-1" because we don't want to count the 861 * faulting page as forw 862 */ 863 npages = nback + nforw + 1; 864 centeridx = nback; 865 866 narrow = TRUE; /* ensure only once per-fault */ 867 868 } else { 869 870 /* narrow fault! */ 871 nback = nforw = 0; 872 startva = ufi.orig_rvaddr; 873 npages = 1; 874 centeridx = 0; 875 876 } 877 878 /* locked: maps(read) */ 879 UVMHIST_LOG(maphist, " narrow=%d, back=%d, forw=%d, startva=0x%x", 880 narrow, nback, nforw, startva); 881 UVMHIST_LOG(maphist, " entry=0x%x, amap=0x%x, obj=0x%x", ufi.entry, 882 amap, uobj, 0); 883 884 /* 885 * if we've got an amap, lock it and extract current anons. 886 */ 887 888 if (amap) { 889 amap_lock(amap); 890 anons = anons_store; 891 amap_lookups(&ufi.entry->aref, startva - ufi.entry->start, 892 anons, npages); 893 } else { 894 anons = NULL; /* to be safe */ 895 } 896 897 /* locked: maps(read), amap(if there) */ 898 LOCK_ASSERT(amap == NULL || simple_lock_held(&amap->am_l)); 899 900 /* 901 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages 902 * now and then forget about them (for the rest of the fault). 903 */ 904 905 if (ufi.entry->advice == MADV_SEQUENTIAL && nback != 0) { 906 907 UVMHIST_LOG(maphist, " MADV_SEQUENTIAL: flushing backpages", 908 0,0,0,0); 909 /* flush back-page anons? */ 910 if (amap) 911 uvmfault_anonflush(anons, nback); 912 913 /* flush object? */ 914 if (uobj) { 915 uoff = (startva - ufi.entry->start) + ufi.entry->offset; 916 simple_lock(&uobj->vmobjlock); 917 (void) (uobj->pgops->pgo_put)(uobj, uoff, uoff + 918 (nback << PAGE_SHIFT), PGO_DEACTIVATE); 919 } 920 921 /* now forget about the backpages */ 922 if (amap) 923 anons += nback; 924 startva += (nback << PAGE_SHIFT); 925 npages -= nback; 926 nback = centeridx = 0; 927 } 928 929 /* locked: maps(read), amap(if there) */ 930 LOCK_ASSERT(amap == NULL || simple_lock_held(&amap->am_l)); 931 932 /* 933 * map in the backpages and frontpages we found in the amap in hopes 934 * of preventing future faults. we also init the pages[] array as 935 * we go. 936 */ 937 938 currva = startva; 939 shadowed = FALSE; 940 for (lcv = 0 ; lcv < npages ; lcv++, currva += PAGE_SIZE) { 941 942 /* 943 * dont play with VAs that are already mapped 944 * except for center) 945 */ 946 if (lcv != centeridx && 947 pmap_extract(ufi.orig_map->pmap, currva, NULL)) { 948 pages[lcv] = PGO_DONTCARE; 949 continue; 950 } 951 952 /* 953 * unmapped or center page. check if any anon at this level. 954 */ 955 if (amap == NULL || anons[lcv] == NULL) { 956 pages[lcv] = NULL; 957 continue; 958 } 959 960 /* 961 * check for present page and map if possible. re-activate it. 962 */ 963 964 pages[lcv] = PGO_DONTCARE; 965 if (lcv == centeridx) { /* save center for later! */ 966 shadowed = TRUE; 967 continue; 968 } 969 anon = anons[lcv]; 970 simple_lock(&anon->an_lock); 971 /* ignore loaned pages */ 972 if (anon->an_page && anon->an_page->loan_count == 0 && 973 (anon->an_page->flags & PG_BUSY) == 0) { 974 uvm_lock_pageq(); 975 uvm_pageactivate(anon->an_page); 976 uvm_unlock_pageq(); 977 UVMHIST_LOG(maphist, 978 " MAPPING: n anon: pm=0x%x, va=0x%x, pg=0x%x", 979 ufi.orig_map->pmap, currva, anon->an_page, 0); 980 uvmexp.fltnamap++; 981 982 /* 983 * Since this isn't the page that's actually faulting, 984 * ignore pmap_enter() failures; it's not critical 985 * that we enter these right now. 986 */ 987 988 (void) pmap_enter(ufi.orig_map->pmap, currva, 989 VM_PAGE_TO_PHYS(anon->an_page), 990 (anon->an_ref > 1) ? (enter_prot & ~VM_PROT_WRITE) : 991 enter_prot, 992 PMAP_CANFAIL | 993 (VM_MAPENT_ISWIRED(ufi.entry) ? PMAP_WIRED : 0)); 994 } 995 simple_unlock(&anon->an_lock); 996 pmap_update(ufi.orig_map->pmap); 997 } 998 999 /* locked: maps(read), amap(if there) */ 1000 LOCK_ASSERT(amap == NULL || simple_lock_held(&amap->am_l)); 1001 /* (shadowed == TRUE) if there is an anon at the faulting address */ 1002 UVMHIST_LOG(maphist, " shadowed=%d, will_get=%d", shadowed, 1003 (uobj && shadowed == FALSE),0,0); 1004 1005 /* 1006 * note that if we are really short of RAM we could sleep in the above 1007 * call to pmap_enter with everything locked. bad? 1008 * 1009 * XXX Actually, that is bad; pmap_enter() should just fail in that 1010 * XXX case. --thorpej 1011 */ 1012 1013 /* 1014 * if the desired page is not shadowed by the amap and we have a 1015 * backing object, then we check to see if the backing object would 1016 * prefer to handle the fault itself (rather than letting us do it 1017 * with the usual pgo_get hook). the backing object signals this by 1018 * providing a pgo_fault routine. 1019 */ 1020 1021 if (uobj && shadowed == FALSE && uobj->pgops->pgo_fault != NULL) { 1022 simple_lock(&uobj->vmobjlock); 1023 1024 /* locked: maps(read), amap (if there), uobj */ 1025 error = uobj->pgops->pgo_fault(&ufi, startva, pages, npages, 1026 centeridx, access_type, PGO_LOCKED|PGO_SYNCIO); 1027 1028 /* locked: nothing, pgo_fault has unlocked everything */ 1029 1030 if (error == ERESTART) 1031 goto ReFault; /* try again! */ 1032 /* 1033 * object fault routine responsible for pmap_update(). 1034 */ 1035 goto done; 1036 } 1037 1038 /* 1039 * now, if the desired page is not shadowed by the amap and we have 1040 * a backing object that does not have a special fault routine, then 1041 * we ask (with pgo_get) the object for resident pages that we care 1042 * about and attempt to map them in. we do not let pgo_get block 1043 * (PGO_LOCKED). 1044 */ 1045 1046 if (uobj && shadowed == FALSE) { 1047 simple_lock(&uobj->vmobjlock); 1048 1049 /* locked (!shadowed): maps(read), amap (if there), uobj */ 1050 /* 1051 * the following call to pgo_get does _not_ change locking state 1052 */ 1053 1054 uvmexp.fltlget++; 1055 gotpages = npages; 1056 (void) uobj->pgops->pgo_get(uobj, ufi.entry->offset + 1057 (startva - ufi.entry->start), 1058 pages, &gotpages, centeridx, 1059 access_type & MASK(ufi.entry), 1060 ufi.entry->advice, PGO_LOCKED); 1061 1062 /* 1063 * check for pages to map, if we got any 1064 */ 1065 1066 uobjpage = NULL; 1067 1068 if (gotpages) { 1069 currva = startva; 1070 for (lcv = 0; lcv < npages; 1071 lcv++, currva += PAGE_SIZE) { 1072 struct vm_page *curpg; 1073 boolean_t readonly; 1074 1075 curpg = pages[lcv]; 1076 if (curpg == NULL || curpg == PGO_DONTCARE) { 1077 continue; 1078 } 1079 KASSERT(curpg->uobject == uobj); 1080 1081 /* 1082 * if center page is resident and not 1083 * PG_BUSY|PG_RELEASED then pgo_get 1084 * made it PG_BUSY for us and gave 1085 * us a handle to it. remember this 1086 * page as "uobjpage." (for later use). 1087 */ 1088 1089 if (lcv == centeridx) { 1090 uobjpage = curpg; 1091 UVMHIST_LOG(maphist, " got uobjpage " 1092 "(0x%x) with locked get", 1093 uobjpage, 0,0,0); 1094 continue; 1095 } 1096 1097 /* 1098 * calling pgo_get with PGO_LOCKED returns us 1099 * pages which are neither busy nor released, 1100 * so we don't need to check for this. 1101 * we can just directly enter the pages. 1102 */ 1103 1104 uvm_lock_pageq(); 1105 uvm_pageactivate(curpg); 1106 uvm_unlock_pageq(); 1107 UVMHIST_LOG(maphist, 1108 " MAPPING: n obj: pm=0x%x, va=0x%x, pg=0x%x", 1109 ufi.orig_map->pmap, currva, curpg, 0); 1110 uvmexp.fltnomap++; 1111 1112 /* 1113 * Since this page isn't the page that's 1114 * actually faulting, ignore pmap_enter() 1115 * failures; it's not critical that we 1116 * enter these right now. 1117 */ 1118 KASSERT((curpg->flags & PG_PAGEOUT) == 0); 1119 KASSERT((curpg->flags & PG_RELEASED) == 0); 1120 KASSERT(!UVM_OBJ_IS_CLEAN(curpg->uobject) || 1121 (curpg->flags & PG_CLEAN) != 0); 1122 readonly = (curpg->flags & PG_RDONLY) 1123 || (curpg->loan_count > 0) 1124 || UVM_OBJ_NEEDS_WRITEFAULT(curpg->uobject); 1125 1126 (void) pmap_enter(ufi.orig_map->pmap, currva, 1127 VM_PAGE_TO_PHYS(curpg), 1128 readonly ? 1129 enter_prot & ~VM_PROT_WRITE : 1130 enter_prot & MASK(ufi.entry), 1131 PMAP_CANFAIL | 1132 (wired ? PMAP_WIRED : 0)); 1133 1134 /* 1135 * NOTE: page can't be PG_WANTED or PG_RELEASED 1136 * because we've held the lock the whole time 1137 * we've had the handle. 1138 */ 1139 KASSERT((curpg->flags & PG_WANTED) == 0); 1140 KASSERT((curpg->flags & PG_RELEASED) == 0); 1141 1142 curpg->flags &= ~(PG_BUSY); 1143 UVM_PAGE_OWN(curpg, NULL); 1144 } 1145 pmap_update(ufi.orig_map->pmap); 1146 } 1147 } else { 1148 uobjpage = NULL; 1149 } 1150 1151 /* locked (shadowed): maps(read), amap */ 1152 /* locked (!shadowed): maps(read), amap(if there), 1153 uobj(if !null), uobjpage(if !null) */ 1154 if (shadowed) { 1155 LOCK_ASSERT(simple_lock_held(&amap->am_l)); 1156 } else { 1157 LOCK_ASSERT(amap == NULL || simple_lock_held(&amap->am_l)); 1158 LOCK_ASSERT(uobj == NULL || simple_lock_held(&uobj->vmobjlock)); 1159 KASSERT(uobjpage == NULL || (uobjpage->flags & PG_BUSY) != 0); 1160 } 1161 1162 /* 1163 * note that at this point we are done with any front or back pages. 1164 * we are now going to focus on the center page (i.e. the one we've 1165 * faulted on). if we have faulted on the top (anon) layer 1166 * [i.e. case 1], then the anon we want is anons[centeridx] (we have 1167 * not touched it yet). if we have faulted on the bottom (uobj) 1168 * layer [i.e. case 2] and the page was both present and available, 1169 * then we've got a pointer to it as "uobjpage" and we've already 1170 * made it BUSY. 1171 */ 1172 1173 /* 1174 * there are four possible cases we must address: 1A, 1B, 2A, and 2B 1175 */ 1176 1177 /* 1178 * redirect case 2: if we are not shadowed, go to case 2. 1179 */ 1180 1181 if (shadowed == FALSE) 1182 goto Case2; 1183 1184 /* locked: maps(read), amap */ 1185 1186 /* 1187 * handle case 1: fault on an anon in our amap 1188 */ 1189 1190 anon = anons[centeridx]; 1191 UVMHIST_LOG(maphist, " case 1 fault: anon=0x%x", anon, 0,0,0); 1192 simple_lock(&anon->an_lock); 1193 1194 /* locked: maps(read), amap, anon */ 1195 LOCK_ASSERT(simple_lock_held(&amap->am_l)); 1196 LOCK_ASSERT(simple_lock_held(&anon->an_lock)); 1197 1198 /* 1199 * no matter if we have case 1A or case 1B we are going to need to 1200 * have the anon's memory resident. ensure that now. 1201 */ 1202 1203 /* 1204 * let uvmfault_anonget do the dirty work. 1205 * if it fails (!OK) it will unlock everything for us. 1206 * if it succeeds, locks are still valid and locked. 1207 * also, if it is OK, then the anon's page is on the queues. 1208 * if the page is on loan from a uvm_object, then anonget will 1209 * lock that object for us if it does not fail. 1210 */ 1211 1212 error = uvmfault_anonget(&ufi, amap, anon); 1213 switch (error) { 1214 case 0: 1215 break; 1216 1217 case ERESTART: 1218 goto ReFault; 1219 1220 case EAGAIN: 1221 tsleep(&lbolt, PVM, "fltagain1", 0); 1222 goto ReFault; 1223 1224 default: 1225 goto done; 1226 } 1227 1228 /* 1229 * uobj is non null if the page is on loan from an object (i.e. uobj) 1230 */ 1231 1232 uobj = anon->an_page->uobject; /* locked by anonget if !NULL */ 1233 1234 /* locked: maps(read), amap, anon, uobj(if one) */ 1235 LOCK_ASSERT(simple_lock_held(&amap->am_l)); 1236 LOCK_ASSERT(simple_lock_held(&anon->an_lock)); 1237 LOCK_ASSERT(uobj == NULL || simple_lock_held(&uobj->vmobjlock)); 1238 1239 /* 1240 * special handling for loaned pages 1241 */ 1242 1243 if (anon->an_page->loan_count) { 1244 1245 if (!cow_now) { 1246 1247 /* 1248 * for read faults on loaned pages we just cap the 1249 * protection at read-only. 1250 */ 1251 1252 enter_prot = enter_prot & ~VM_PROT_WRITE; 1253 1254 } else { 1255 /* 1256 * note that we can't allow writes into a loaned page! 1257 * 1258 * if we have a write fault on a loaned page in an 1259 * anon then we need to look at the anon's ref count. 1260 * if it is greater than one then we are going to do 1261 * a normal copy-on-write fault into a new anon (this 1262 * is not a problem). however, if the reference count 1263 * is one (a case where we would normally allow a 1264 * write directly to the page) then we need to kill 1265 * the loan before we continue. 1266 */ 1267 1268 /* >1 case is already ok */ 1269 if (anon->an_ref == 1) { 1270 1271 /* get new un-owned replacement page */ 1272 pg = uvm_pagealloc(NULL, 0, NULL, 0); 1273 if (pg == NULL) { 1274 uvmfault_unlockall(&ufi, amap, uobj, 1275 anon); 1276 uvm_wait("flt_noram2"); 1277 goto ReFault; 1278 } 1279 1280 /* 1281 * copy data, kill loan, and drop uobj lock 1282 * (if any) 1283 */ 1284 /* copy old -> new */ 1285 uvm_pagecopy(anon->an_page, pg); 1286 1287 /* force reload */ 1288 pmap_page_protect(anon->an_page, 1289 VM_PROT_NONE); 1290 uvm_lock_pageq(); /* KILL loan */ 1291 1292 anon->an_page->uanon = NULL; 1293 /* in case we owned */ 1294 anon->an_page->pqflags &= ~PQ_ANON; 1295 1296 if (uobj) { 1297 /* if we were receiver of loan */ 1298 anon->an_page->loan_count--; 1299 } else { 1300 /* 1301 * we were the lender (A->K); need 1302 * to remove the page from pageq's. 1303 */ 1304 uvm_pagedequeue(anon->an_page); 1305 } 1306 1307 uvm_pageactivate(pg); 1308 uvm_unlock_pageq(); 1309 if (uobj) { 1310 simple_unlock(&uobj->vmobjlock); 1311 uobj = NULL; 1312 } 1313 1314 /* install new page in anon */ 1315 anon->an_page = pg; 1316 pg->uanon = anon; 1317 pg->pqflags |= PQ_ANON; 1318 pg->flags &= ~(PG_BUSY|PG_FAKE); 1319 UVM_PAGE_OWN(pg, NULL); 1320 1321 /* done! */ 1322 } /* ref == 1 */ 1323 } /* write fault */ 1324 } /* loan count */ 1325 1326 /* 1327 * if we are case 1B then we will need to allocate a new blank 1328 * anon to transfer the data into. note that we have a lock 1329 * on anon, so no one can busy or release the page until we are done. 1330 * also note that the ref count can't drop to zero here because 1331 * it is > 1 and we are only dropping one ref. 1332 * 1333 * in the (hopefully very rare) case that we are out of RAM we 1334 * will unlock, wait for more RAM, and refault. 1335 * 1336 * if we are out of anon VM we kill the process (XXX: could wait?). 1337 */ 1338 1339 if (cow_now && anon->an_ref > 1) { 1340 1341 UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0); 1342 uvmexp.flt_acow++; 1343 oanon = anon; /* oanon = old, locked anon */ 1344 1345 error = uvmfault_promote(&ufi, oanon, PGO_DONTCARE, 1346 &anon, &anon_spare); 1347 switch (error) { 1348 case 0: 1349 break; 1350 case ERESTART: 1351 goto ReFault; 1352 default: 1353 goto done; 1354 } 1355 1356 pg = anon->an_page; 1357 uvm_lock_pageq(); 1358 uvm_pageactivate(pg); 1359 uvm_unlock_pageq(); 1360 pg->flags &= ~(PG_BUSY|PG_FAKE); 1361 UVM_PAGE_OWN(pg, NULL); 1362 1363 /* deref: can not drop to zero here by defn! */ 1364 oanon->an_ref--; 1365 1366 /* 1367 * note: oanon is still locked, as is the new anon. we 1368 * need to check for this later when we unlock oanon; if 1369 * oanon != anon, we'll have to unlock anon, too. 1370 */ 1371 1372 } else { 1373 1374 uvmexp.flt_anon++; 1375 oanon = anon; /* old, locked anon is same as anon */ 1376 pg = anon->an_page; 1377 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */ 1378 enter_prot = enter_prot & ~VM_PROT_WRITE; 1379 1380 } 1381 1382 /* locked: maps(read), amap, oanon, anon (if different from oanon) */ 1383 LOCK_ASSERT(simple_lock_held(&amap->am_l)); 1384 LOCK_ASSERT(simple_lock_held(&anon->an_lock)); 1385 LOCK_ASSERT(simple_lock_held(&oanon->an_lock)); 1386 1387 /* 1388 * now map the page in. 1389 */ 1390 1391 UVMHIST_LOG(maphist, " MAPPING: anon: pm=0x%x, va=0x%x, pg=0x%x", 1392 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, 0); 1393 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1394 enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) 1395 != 0) { 1396 1397 /* 1398 * No need to undo what we did; we can simply think of 1399 * this as the pmap throwing away the mapping information. 1400 * 1401 * We do, however, have to go through the ReFault path, 1402 * as the map may change while we're asleep. 1403 */ 1404 1405 if (anon != oanon) 1406 simple_unlock(&anon->an_lock); 1407 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1408 if (!uvm_reclaimable()) { 1409 UVMHIST_LOG(maphist, 1410 "<- failed. out of VM",0,0,0,0); 1411 /* XXX instrumentation */ 1412 error = ENOMEM; 1413 goto done; 1414 } 1415 /* XXX instrumentation */ 1416 uvm_wait("flt_pmfail1"); 1417 goto ReFault; 1418 } 1419 1420 /* 1421 * ... update the page queues. 1422 */ 1423 1424 uvm_lock_pageq(); 1425 if (wire_fault) { 1426 uvm_pagewire(pg); 1427 1428 /* 1429 * since the now-wired page cannot be paged out, 1430 * release its swap resources for others to use. 1431 * since an anon with no swap cannot be PG_CLEAN, 1432 * clear its clean flag now. 1433 */ 1434 1435 pg->flags &= ~(PG_CLEAN); 1436 uvm_anon_dropswap(anon); 1437 } else { 1438 uvm_pageactivate(pg); 1439 } 1440 uvm_unlock_pageq(); 1441 1442 /* 1443 * done case 1! finish up by unlocking everything and returning success 1444 */ 1445 1446 if (anon != oanon) 1447 simple_unlock(&anon->an_lock); 1448 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1449 pmap_update(ufi.orig_map->pmap); 1450 error = 0; 1451 goto done; 1452 1453 Case2: 1454 /* 1455 * handle case 2: faulting on backing object or zero fill 1456 */ 1457 1458 /* 1459 * locked: 1460 * maps(read), amap(if there), uobj(if !null), uobjpage(if !null) 1461 */ 1462 LOCK_ASSERT(amap == NULL || simple_lock_held(&amap->am_l)); 1463 LOCK_ASSERT(uobj == NULL || simple_lock_held(&uobj->vmobjlock)); 1464 LOCK_ASSERT(uobjpage == NULL || (uobjpage->flags & PG_BUSY) != 0); 1465 1466 /* 1467 * note that uobjpage can not be PGO_DONTCARE at this point. we now 1468 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we 1469 * have a backing object, check and see if we are going to promote 1470 * the data up to an anon during the fault. 1471 */ 1472 1473 if (uobj == NULL) { 1474 uobjpage = PGO_DONTCARE; 1475 promote = TRUE; /* always need anon here */ 1476 } else { 1477 KASSERT(uobjpage != PGO_DONTCARE); 1478 promote = cow_now && UVM_ET_ISCOPYONWRITE(ufi.entry); 1479 } 1480 UVMHIST_LOG(maphist, " case 2 fault: promote=%d, zfill=%d", 1481 promote, (uobj == NULL), 0,0); 1482 1483 /* 1484 * if uobjpage is not null then we do not need to do I/O to get the 1485 * uobjpage. 1486 * 1487 * if uobjpage is null, then we need to unlock and ask the pager to 1488 * get the data for us. once we have the data, we need to reverify 1489 * the state the world. we are currently not holding any resources. 1490 */ 1491 1492 if (uobjpage) { 1493 /* update rusage counters */ 1494 curproc->p_stats->p_ru.ru_minflt++; 1495 } else { 1496 /* update rusage counters */ 1497 curproc->p_stats->p_ru.ru_majflt++; 1498 1499 /* locked: maps(read), amap(if there), uobj */ 1500 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1501 /* locked: uobj */ 1502 1503 uvmexp.fltget++; 1504 gotpages = 1; 1505 uoff = (ufi.orig_rvaddr - ufi.entry->start) + ufi.entry->offset; 1506 error = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages, 1507 0, access_type & MASK(ufi.entry), ufi.entry->advice, 1508 PGO_SYNCIO); 1509 /* locked: uobjpage(if no error) */ 1510 LOCK_ASSERT(error != 0 || (uobjpage->flags & PG_BUSY) != 0); 1511 1512 /* 1513 * recover from I/O 1514 */ 1515 1516 if (error) { 1517 if (error == EAGAIN) { 1518 UVMHIST_LOG(maphist, 1519 " pgo_get says TRY AGAIN!",0,0,0,0); 1520 tsleep(&lbolt, PVM, "fltagain2", 0); 1521 goto ReFault; 1522 } 1523 1524 UVMHIST_LOG(maphist, "<- pgo_get failed (code %d)", 1525 error, 0,0,0); 1526 goto done; 1527 } 1528 1529 /* locked: uobjpage */ 1530 1531 uvm_lock_pageq(); 1532 uvm_pageactivate(uobjpage); 1533 uvm_unlock_pageq(); 1534 1535 /* 1536 * re-verify the state of the world by first trying to relock 1537 * the maps. always relock the object. 1538 */ 1539 1540 locked = uvmfault_relock(&ufi); 1541 if (locked && amap) 1542 amap_lock(amap); 1543 uobj = uobjpage->uobject; 1544 simple_lock(&uobj->vmobjlock); 1545 1546 /* locked(locked): maps(read), amap(if !null), uobj, uobjpage */ 1547 /* locked(!locked): uobj, uobjpage */ 1548 1549 /* 1550 * verify that the page has not be released and re-verify 1551 * that amap slot is still free. if there is a problem, 1552 * we unlock and clean up. 1553 */ 1554 1555 if ((uobjpage->flags & PG_RELEASED) != 0 || 1556 (locked && amap && 1557 amap_lookup(&ufi.entry->aref, 1558 ufi.orig_rvaddr - ufi.entry->start))) { 1559 if (locked) 1560 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1561 locked = FALSE; 1562 } 1563 1564 /* 1565 * didn't get the lock? release the page and retry. 1566 */ 1567 1568 if (locked == FALSE) { 1569 UVMHIST_LOG(maphist, 1570 " wasn't able to relock after fault: retry", 1571 0,0,0,0); 1572 if (uobjpage->flags & PG_WANTED) 1573 wakeup(uobjpage); 1574 if (uobjpage->flags & PG_RELEASED) { 1575 uvmexp.fltpgrele++; 1576 uvm_pagefree(uobjpage); 1577 goto ReFault; 1578 } 1579 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1580 UVM_PAGE_OWN(uobjpage, NULL); 1581 simple_unlock(&uobj->vmobjlock); 1582 goto ReFault; 1583 } 1584 1585 /* 1586 * we have the data in uobjpage which is busy and 1587 * not released. we are holding object lock (so the page 1588 * can't be released on us). 1589 */ 1590 1591 /* locked: maps(read), amap(if !null), uobj, uobjpage */ 1592 } 1593 1594 /* 1595 * locked: 1596 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj) 1597 */ 1598 LOCK_ASSERT(amap == NULL || simple_lock_held(&amap->am_l)); 1599 LOCK_ASSERT(uobj == NULL || simple_lock_held(&uobj->vmobjlock)); 1600 LOCK_ASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0); 1601 1602 /* 1603 * notes: 1604 * - at this point uobjpage can not be NULL 1605 * - at this point uobjpage can not be PG_RELEASED (since we checked 1606 * for it above) 1607 * - at this point uobjpage could be PG_WANTED (handle later) 1608 */ 1609 1610 KASSERT(uobj == NULL || uobj == uobjpage->uobject); 1611 KASSERT(uobj == NULL || !UVM_OBJ_IS_CLEAN(uobjpage->uobject) || 1612 (uobjpage->flags & PG_CLEAN) != 0); 1613 if (promote == FALSE) { 1614 1615 /* 1616 * we are not promoting. if the mapping is COW ensure that we 1617 * don't give more access than we should (e.g. when doing a read 1618 * fault on a COPYONWRITE mapping we want to map the COW page in 1619 * R/O even though the entry protection could be R/W). 1620 * 1621 * set "pg" to the page we want to map in (uobjpage, usually) 1622 */ 1623 1624 /* no anon in this case. */ 1625 anon = NULL; 1626 1627 uvmexp.flt_obj++; 1628 if (UVM_ET_ISCOPYONWRITE(ufi.entry) || 1629 UVM_OBJ_NEEDS_WRITEFAULT(uobjpage->uobject)) 1630 enter_prot &= ~VM_PROT_WRITE; 1631 pg = uobjpage; /* map in the actual object */ 1632 1633 KASSERT(uobjpage != PGO_DONTCARE); 1634 1635 /* 1636 * we are faulting directly on the page. be careful 1637 * about writing to loaned pages... 1638 */ 1639 1640 if (uobjpage->loan_count) { 1641 if (!cow_now) { 1642 /* read fault: cap the protection at readonly */ 1643 /* cap! */ 1644 enter_prot = enter_prot & ~VM_PROT_WRITE; 1645 } else { 1646 /* write fault: must break the loan here */ 1647 1648 pg = uvm_loanbreak(uobjpage); 1649 if (pg == NULL) { 1650 1651 /* 1652 * drop ownership of page, it can't 1653 * be released 1654 */ 1655 1656 if (uobjpage->flags & PG_WANTED) 1657 wakeup(uobjpage); 1658 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1659 UVM_PAGE_OWN(uobjpage, NULL); 1660 1661 uvmfault_unlockall(&ufi, amap, uobj, 1662 NULL); 1663 UVMHIST_LOG(maphist, 1664 " out of RAM breaking loan, waiting", 1665 0,0,0,0); 1666 uvmexp.fltnoram++; 1667 uvm_wait("flt_noram4"); 1668 goto ReFault; 1669 } 1670 uobjpage = pg; 1671 } 1672 } 1673 } else { 1674 1675 /* 1676 * if we are going to promote the data to an anon we 1677 * allocate a blank anon here and plug it into our amap. 1678 */ 1679 #if DIAGNOSTIC 1680 if (amap == NULL) 1681 panic("uvm_fault: want to promote data, but no anon"); 1682 #endif 1683 error = uvmfault_promote(&ufi, NULL, uobjpage, 1684 &anon, &anon_spare); 1685 switch (error) { 1686 case 0: 1687 break; 1688 case ERESTART: 1689 goto ReFault; 1690 default: 1691 goto done; 1692 } 1693 1694 pg = anon->an_page; 1695 1696 /* 1697 * fill in the data 1698 */ 1699 1700 if (uobjpage != PGO_DONTCARE) { 1701 uvmexp.flt_prcopy++; 1702 1703 /* 1704 * promote to shared amap? make sure all sharing 1705 * procs see it 1706 */ 1707 1708 if ((amap_flags(amap) & AMAP_SHARED) != 0) { 1709 pmap_page_protect(uobjpage, VM_PROT_NONE); 1710 /* 1711 * XXX: PAGE MIGHT BE WIRED! 1712 */ 1713 } 1714 1715 /* 1716 * dispose of uobjpage. it can't be PG_RELEASED 1717 * since we still hold the object lock. 1718 * drop handle to uobj as well. 1719 */ 1720 1721 if (uobjpage->flags & PG_WANTED) 1722 /* still have the obj lock */ 1723 wakeup(uobjpage); 1724 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1725 UVM_PAGE_OWN(uobjpage, NULL); 1726 simple_unlock(&uobj->vmobjlock); 1727 uobj = NULL; 1728 1729 UVMHIST_LOG(maphist, 1730 " promote uobjpage 0x%x to anon/page 0x%x/0x%x", 1731 uobjpage, anon, pg, 0); 1732 1733 } else { 1734 uvmexp.flt_przero++; 1735 1736 /* 1737 * Page is zero'd and marked dirty by 1738 * uvmfault_promote(). 1739 */ 1740 1741 UVMHIST_LOG(maphist," zero fill anon/page 0x%x/0%x", 1742 anon, pg, 0, 0); 1743 } 1744 } 1745 1746 /* 1747 * locked: 1748 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj), 1749 * anon(if !null), pg(if anon) 1750 * 1751 * note: pg is either the uobjpage or the new page in the new anon 1752 */ 1753 LOCK_ASSERT(amap == NULL || simple_lock_held(&amap->am_l)); 1754 LOCK_ASSERT(uobj == NULL || simple_lock_held(&uobj->vmobjlock)); 1755 LOCK_ASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0); 1756 LOCK_ASSERT(anon == NULL || simple_lock_held(&anon->an_lock)); 1757 LOCK_ASSERT((pg->flags & PG_BUSY) != 0); 1758 1759 /* 1760 * all resources are present. we can now map it in and free our 1761 * resources. 1762 */ 1763 1764 UVMHIST_LOG(maphist, 1765 " MAPPING: case2: pm=0x%x, va=0x%x, pg=0x%x, promote=%d", 1766 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, promote); 1767 KASSERT((access_type & VM_PROT_WRITE) == 0 || 1768 (pg->flags & PG_RDONLY) == 0); 1769 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1770 pg->flags & PG_RDONLY ? enter_prot & ~VM_PROT_WRITE : enter_prot, 1771 access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) != 0) { 1772 1773 /* 1774 * No need to undo what we did; we can simply think of 1775 * this as the pmap throwing away the mapping information. 1776 * 1777 * We do, however, have to go through the ReFault path, 1778 * as the map may change while we're asleep. 1779 */ 1780 1781 if (pg->flags & PG_WANTED) 1782 wakeup(pg); 1783 1784 /* 1785 * note that pg can't be PG_RELEASED since we did not drop 1786 * the object lock since the last time we checked. 1787 */ 1788 KASSERT((pg->flags & PG_RELEASED) == 0); 1789 1790 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); 1791 UVM_PAGE_OWN(pg, NULL); 1792 uvmfault_unlockall(&ufi, amap, uobj, anon); 1793 if (!uvm_reclaimable()) { 1794 UVMHIST_LOG(maphist, 1795 "<- failed. out of VM",0,0,0,0); 1796 /* XXX instrumentation */ 1797 error = ENOMEM; 1798 goto done; 1799 } 1800 /* XXX instrumentation */ 1801 uvm_wait("flt_pmfail2"); 1802 goto ReFault; 1803 } 1804 1805 uvm_lock_pageq(); 1806 if (wire_fault) { 1807 uvm_pagewire(pg); 1808 if (pg->pqflags & PQ_AOBJ) { 1809 1810 /* 1811 * since the now-wired page cannot be paged out, 1812 * release its swap resources for others to use. 1813 * since an aobj page with no swap cannot be PG_CLEAN, 1814 * clear its clean flag now. 1815 */ 1816 1817 pg->flags &= ~(PG_CLEAN); 1818 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT); 1819 } 1820 } else { 1821 uvm_pageactivate(pg); 1822 } 1823 uvm_unlock_pageq(); 1824 if (pg->flags & PG_WANTED) 1825 wakeup(pg); 1826 1827 /* 1828 * note that pg can't be PG_RELEASED since we did not drop the object 1829 * lock since the last time we checked. 1830 */ 1831 KASSERT((pg->flags & PG_RELEASED) == 0); 1832 1833 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); 1834 UVM_PAGE_OWN(pg, NULL); 1835 uvmfault_unlockall(&ufi, amap, uobj, anon); 1836 pmap_update(ufi.orig_map->pmap); 1837 UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0); 1838 error = 0; 1839 done: 1840 if (anon_spare != NULL) { 1841 anon_spare->an_ref--; 1842 uvm_anfree(anon_spare); 1843 } 1844 return error; 1845 } 1846 1847 1848 /* 1849 * uvm_fault_wire: wire down a range of virtual addresses in a map. 1850 * 1851 * => map may be read-locked by caller, but MUST NOT be write-locked. 1852 * => if map is read-locked, any operations which may cause map to 1853 * be write-locked in uvm_fault() must be taken care of by 1854 * the caller. See uvm_map_pageable(). 1855 */ 1856 1857 int 1858 uvm_fault_wire(struct vm_map *map, vaddr_t start, vaddr_t end, 1859 vm_prot_t access_type, int wiremax) 1860 { 1861 vaddr_t va; 1862 int error; 1863 1864 /* 1865 * now fault it in a page at a time. if the fault fails then we have 1866 * to undo what we have done. note that in uvm_fault VM_PROT_NONE 1867 * is replaced with the max protection if fault_type is VM_FAULT_WIRE. 1868 */ 1869 1870 /* 1871 * XXX work around overflowing a vaddr_t. this prevents us from 1872 * wiring the last page in the address space, though. 1873 */ 1874 if (start > end) { 1875 return EFAULT; 1876 } 1877 1878 for (va = start ; va < end ; va += PAGE_SIZE) { 1879 error = uvm_fault_internal(map, va, access_type, 1880 wiremax ? UVM_FAULT_WIREMAX : UVM_FAULT_WIRE); 1881 if (error) { 1882 if (va != start) { 1883 uvm_fault_unwire(map, start, va); 1884 } 1885 return error; 1886 } 1887 } 1888 return 0; 1889 } 1890 1891 /* 1892 * uvm_fault_unwire(): unwire range of virtual space. 1893 */ 1894 1895 void 1896 uvm_fault_unwire(struct vm_map *map, vaddr_t start, vaddr_t end) 1897 { 1898 vm_map_lock_read(map); 1899 uvm_fault_unwire_locked(map, start, end); 1900 vm_map_unlock_read(map); 1901 } 1902 1903 /* 1904 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire(). 1905 * 1906 * => map must be at least read-locked. 1907 */ 1908 1909 void 1910 uvm_fault_unwire_locked(struct vm_map *map, vaddr_t start, vaddr_t end) 1911 { 1912 struct vm_map_entry *entry; 1913 pmap_t pmap = vm_map_pmap(map); 1914 vaddr_t va; 1915 paddr_t pa; 1916 struct vm_page *pg; 1917 1918 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0); 1919 1920 /* 1921 * we assume that the area we are unwiring has actually been wired 1922 * in the first place. this means that we should be able to extract 1923 * the PAs from the pmap. we also lock out the page daemon so that 1924 * we can call uvm_pageunwire. 1925 */ 1926 1927 uvm_lock_pageq(); 1928 1929 /* 1930 * find the beginning map entry for the region. 1931 */ 1932 1933 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map)); 1934 if (uvm_map_lookup_entry(map, start, &entry) == FALSE) 1935 panic("uvm_fault_unwire_locked: address not in map"); 1936 1937 for (va = start; va < end; va += PAGE_SIZE) { 1938 if (pmap_extract(pmap, va, &pa) == FALSE) 1939 continue; 1940 1941 /* 1942 * find the map entry for the current address. 1943 */ 1944 1945 KASSERT(va >= entry->start); 1946 while (va >= entry->end) { 1947 KASSERT(entry->next != &map->header && 1948 entry->next->start <= entry->end); 1949 entry = entry->next; 1950 } 1951 1952 /* 1953 * if the entry is no longer wired, tell the pmap. 1954 */ 1955 1956 if (VM_MAPENT_ISWIRED(entry) == 0) 1957 pmap_unwire(pmap, va); 1958 1959 pg = PHYS_TO_VM_PAGE(pa); 1960 if (pg) 1961 uvm_pageunwire(pg); 1962 } 1963 1964 uvm_unlock_pageq(); 1965 } 1966