1 /* $NetBSD: uvm_fault.c,v 1.129 2009/12/17 01:25:10 rmind Exp $ */ 2 3 /* 4 * 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by Charles D. Cranor and 19 * Washington University. 20 * 4. The name of the author may not be used to endorse or promote products 21 * derived from this software without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 24 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 25 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 26 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 27 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 28 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 32 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 * 34 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp 35 */ 36 37 /* 38 * uvm_fault.c: fault handler 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.129 2009/12/17 01:25:10 rmind Exp $"); 43 44 #include "opt_uvmhist.h" 45 46 #include <sys/param.h> 47 #include <sys/systm.h> 48 #include <sys/kernel.h> 49 #include <sys/proc.h> 50 #include <sys/malloc.h> 51 #include <sys/mman.h> 52 53 #include <uvm/uvm.h> 54 55 /* 56 * 57 * a word on page faults: 58 * 59 * types of page faults we handle: 60 * 61 * CASE 1: upper layer faults CASE 2: lower layer faults 62 * 63 * CASE 1A CASE 1B CASE 2A CASE 2B 64 * read/write1 write>1 read/write +-cow_write/zero 65 * | | | | 66 * +--|--+ +--|--+ +-----+ + | + | +-----+ 67 * amap | V | | ---------> new | | | | ^ | 68 * +-----+ +-----+ +-----+ + | + | +--|--+ 69 * | | | 70 * +-----+ +-----+ +--|--+ | +--|--+ 71 * uobj | d/c | | d/c | | V | +----+ | 72 * +-----+ +-----+ +-----+ +-----+ 73 * 74 * d/c = don't care 75 * 76 * case [0]: layerless fault 77 * no amap or uobj is present. this is an error. 78 * 79 * case [1]: upper layer fault [anon active] 80 * 1A: [read] or [write with anon->an_ref == 1] 81 * I/O takes place in upper level anon and uobj is not touched. 82 * 1B: [write with anon->an_ref > 1] 83 * new anon is alloc'd and data is copied off ["COW"] 84 * 85 * case [2]: lower layer fault [uobj] 86 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area] 87 * I/O takes place directly in object. 88 * 2B: [write to copy_on_write] or [read on NULL uobj] 89 * data is "promoted" from uobj to a new anon. 90 * if uobj is null, then we zero fill. 91 * 92 * we follow the standard UVM locking protocol ordering: 93 * 94 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ) 95 * we hold a PG_BUSY page if we unlock for I/O 96 * 97 * 98 * the code is structured as follows: 99 * 100 * - init the "IN" params in the ufi structure 101 * ReFault: 102 * - do lookups [locks maps], check protection, handle needs_copy 103 * - check for case 0 fault (error) 104 * - establish "range" of fault 105 * - if we have an amap lock it and extract the anons 106 * - if sequential advice deactivate pages behind us 107 * - at the same time check pmap for unmapped areas and anon for pages 108 * that we could map in (and do map it if found) 109 * - check object for resident pages that we could map in 110 * - if (case 2) goto Case2 111 * - >>> handle case 1 112 * - ensure source anon is resident in RAM 113 * - if case 1B alloc new anon and copy from source 114 * - map the correct page in 115 * Case2: 116 * - >>> handle case 2 117 * - ensure source page is resident (if uobj) 118 * - if case 2B alloc new anon and copy from source (could be zero 119 * fill if uobj == NULL) 120 * - map the correct page in 121 * - done! 122 * 123 * note on paging: 124 * if we have to do I/O we place a PG_BUSY page in the correct object, 125 * unlock everything, and do the I/O. when I/O is done we must reverify 126 * the state of the world before assuming that our data structures are 127 * valid. [because mappings could change while the map is unlocked] 128 * 129 * alternative 1: unbusy the page in question and restart the page fault 130 * from the top (ReFault). this is easy but does not take advantage 131 * of the information that we already have from our previous lookup, 132 * although it is possible that the "hints" in the vm_map will help here. 133 * 134 * alternative 2: the system already keeps track of a "version" number of 135 * a map. [i.e. every time you write-lock a map (e.g. to change a 136 * mapping) you bump the version number up by one...] so, we can save 137 * the version number of the map before we release the lock and start I/O. 138 * then when I/O is done we can relock and check the version numbers 139 * to see if anything changed. this might save us some over 1 because 140 * we don't have to unbusy the page and may be less compares(?). 141 * 142 * alternative 3: put in backpointers or a way to "hold" part of a map 143 * in place while I/O is in progress. this could be complex to 144 * implement (especially with structures like amap that can be referenced 145 * by multiple map entries, and figuring out what should wait could be 146 * complex as well...). 147 * 148 * we use alternative 2. given that we are multi-threaded now we may want 149 * to reconsider the choice. 150 */ 151 152 /* 153 * local data structures 154 */ 155 156 struct uvm_advice { 157 int advice; 158 int nback; 159 int nforw; 160 }; 161 162 /* 163 * page range array: 164 * note: index in array must match "advice" value 165 * XXX: borrowed numbers from freebsd. do they work well for us? 166 */ 167 168 static const struct uvm_advice uvmadvice[] = { 169 { MADV_NORMAL, 3, 4 }, 170 { MADV_RANDOM, 0, 0 }, 171 { MADV_SEQUENTIAL, 8, 7}, 172 }; 173 174 #define UVM_MAXRANGE 16 /* must be MAX() of nback+nforw+1 */ 175 176 /* 177 * private prototypes 178 */ 179 180 /* 181 * inline functions 182 */ 183 184 /* 185 * uvmfault_anonflush: try and deactivate pages in specified anons 186 * 187 * => does not have to deactivate page if it is busy 188 */ 189 190 static inline void 191 uvmfault_anonflush(struct vm_anon **anons, int n) 192 { 193 int lcv; 194 struct vm_page *pg; 195 196 for (lcv = 0 ; lcv < n ; lcv++) { 197 if (anons[lcv] == NULL) 198 continue; 199 mutex_enter(&anons[lcv]->an_lock); 200 pg = anons[lcv]->an_page; 201 if (pg && (pg->flags & PG_BUSY) == 0) { 202 mutex_enter(&uvm_pageqlock); 203 if (pg->wire_count == 0) { 204 uvm_pagedeactivate(pg); 205 } 206 mutex_exit(&uvm_pageqlock); 207 } 208 mutex_exit(&anons[lcv]->an_lock); 209 } 210 } 211 212 /* 213 * normal functions 214 */ 215 216 /* 217 * uvmfault_amapcopy: clear "needs_copy" in a map. 218 * 219 * => called with VM data structures unlocked (usually, see below) 220 * => we get a write lock on the maps and clear needs_copy for a VA 221 * => if we are out of RAM we sleep (waiting for more) 222 */ 223 224 static void 225 uvmfault_amapcopy(struct uvm_faultinfo *ufi) 226 { 227 for (;;) { 228 229 /* 230 * no mapping? give up. 231 */ 232 233 if (uvmfault_lookup(ufi, true) == false) 234 return; 235 236 /* 237 * copy if needed. 238 */ 239 240 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) 241 amap_copy(ufi->map, ufi->entry, AMAP_COPY_NOWAIT, 242 ufi->orig_rvaddr, ufi->orig_rvaddr + 1); 243 244 /* 245 * didn't work? must be out of RAM. unlock and sleep. 246 */ 247 248 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 249 uvmfault_unlockmaps(ufi, true); 250 uvm_wait("fltamapcopy"); 251 continue; 252 } 253 254 /* 255 * got it! unlock and return. 256 */ 257 258 uvmfault_unlockmaps(ufi, true); 259 return; 260 } 261 /*NOTREACHED*/ 262 } 263 264 /* 265 * uvmfault_anonget: get data in an anon into a non-busy, non-released 266 * page in that anon. 267 * 268 * => maps, amap, and anon locked by caller. 269 * => if we fail (result != 0) we unlock everything. 270 * => if we are successful, we return with everything still locked. 271 * => we don't move the page on the queues [gets moved later] 272 * => if we allocate a new page [we_own], it gets put on the queues. 273 * either way, the result is that the page is on the queues at return time 274 * => for pages which are on loan from a uvm_object (and thus are not 275 * owned by the anon): if successful, we return with the owning object 276 * locked. the caller must unlock this object when it unlocks everything 277 * else. 278 */ 279 280 int 281 uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap, 282 struct vm_anon *anon) 283 { 284 bool we_own; /* we own anon's page? */ 285 bool locked; /* did we relock? */ 286 struct vm_page *pg; 287 int error; 288 UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist); 289 290 KASSERT(mutex_owned(&anon->an_lock)); 291 292 error = 0; 293 uvmexp.fltanget++; 294 /* bump rusage counters */ 295 if (anon->an_page) 296 curlwp->l_ru.ru_minflt++; 297 else 298 curlwp->l_ru.ru_majflt++; 299 300 /* 301 * loop until we get it, or fail. 302 */ 303 304 for (;;) { 305 we_own = false; /* true if we set PG_BUSY on a page */ 306 pg = anon->an_page; 307 308 /* 309 * if there is a resident page and it is loaned, then anon 310 * may not own it. call out to uvm_anon_lockpage() to ensure 311 * the real owner of the page has been identified and locked. 312 */ 313 314 if (pg && pg->loan_count) 315 pg = uvm_anon_lockloanpg(anon); 316 317 /* 318 * page there? make sure it is not busy/released. 319 */ 320 321 if (pg) { 322 323 /* 324 * at this point, if the page has a uobject [meaning 325 * we have it on loan], then that uobject is locked 326 * by us! if the page is busy, we drop all the 327 * locks (including uobject) and try again. 328 */ 329 330 if ((pg->flags & PG_BUSY) == 0) { 331 UVMHIST_LOG(maphist, "<- OK",0,0,0,0); 332 return (0); 333 } 334 pg->flags |= PG_WANTED; 335 uvmexp.fltpgwait++; 336 337 /* 338 * the last unlock must be an atomic unlock+wait on 339 * the owner of page 340 */ 341 342 if (pg->uobject) { /* owner is uobject ? */ 343 uvmfault_unlockall(ufi, amap, NULL, anon); 344 UVMHIST_LOG(maphist, " unlock+wait on uobj",0, 345 0,0,0); 346 UVM_UNLOCK_AND_WAIT(pg, 347 &pg->uobject->vmobjlock, 348 false, "anonget1",0); 349 } else { 350 /* anon owns page */ 351 uvmfault_unlockall(ufi, amap, NULL, NULL); 352 UVMHIST_LOG(maphist, " unlock+wait on anon",0, 353 0,0,0); 354 UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0, 355 "anonget2",0); 356 } 357 } else { 358 #if defined(VMSWAP) 359 360 /* 361 * no page, we must try and bring it in. 362 */ 363 364 pg = uvm_pagealloc(NULL, 0, anon, 0); 365 if (pg == NULL) { /* out of RAM. */ 366 uvmfault_unlockall(ufi, amap, NULL, anon); 367 uvmexp.fltnoram++; 368 UVMHIST_LOG(maphist, " noram -- UVM_WAIT",0, 369 0,0,0); 370 if (!uvm_reclaimable()) { 371 return ENOMEM; 372 } 373 uvm_wait("flt_noram1"); 374 } else { 375 /* we set the PG_BUSY bit */ 376 we_own = true; 377 uvmfault_unlockall(ufi, amap, NULL, anon); 378 379 /* 380 * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN 381 * page into the uvm_swap_get function with 382 * all data structures unlocked. note that 383 * it is ok to read an_swslot here because 384 * we hold PG_BUSY on the page. 385 */ 386 uvmexp.pageins++; 387 error = uvm_swap_get(pg, anon->an_swslot, 388 PGO_SYNCIO); 389 390 /* 391 * we clean up after the i/o below in the 392 * "we_own" case 393 */ 394 } 395 #else /* defined(VMSWAP) */ 396 panic("%s: no page", __func__); 397 #endif /* defined(VMSWAP) */ 398 } 399 400 /* 401 * now relock and try again 402 */ 403 404 locked = uvmfault_relock(ufi); 405 if (locked && amap != NULL) { 406 amap_lock(amap); 407 } 408 if (locked || we_own) 409 mutex_enter(&anon->an_lock); 410 411 /* 412 * if we own the page (i.e. we set PG_BUSY), then we need 413 * to clean up after the I/O. there are three cases to 414 * consider: 415 * [1] page released during I/O: free anon and ReFault. 416 * [2] I/O not OK. free the page and cause the fault 417 * to fail. 418 * [3] I/O OK! activate the page and sync with the 419 * non-we_own case (i.e. drop anon lock if not locked). 420 */ 421 422 if (we_own) { 423 #if defined(VMSWAP) 424 if (pg->flags & PG_WANTED) { 425 wakeup(pg); 426 } 427 if (error) { 428 429 /* 430 * remove the swap slot from the anon 431 * and mark the anon as having no real slot. 432 * don't free the swap slot, thus preventing 433 * it from being used again. 434 */ 435 436 if (anon->an_swslot > 0) 437 uvm_swap_markbad(anon->an_swslot, 1); 438 anon->an_swslot = SWSLOT_BAD; 439 440 if ((pg->flags & PG_RELEASED) != 0) 441 goto released; 442 443 /* 444 * note: page was never !PG_BUSY, so it 445 * can't be mapped and thus no need to 446 * pmap_page_protect it... 447 */ 448 449 mutex_enter(&uvm_pageqlock); 450 uvm_pagefree(pg); 451 mutex_exit(&uvm_pageqlock); 452 453 if (locked) 454 uvmfault_unlockall(ufi, amap, NULL, 455 anon); 456 else 457 mutex_exit(&anon->an_lock); 458 UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0); 459 return error; 460 } 461 462 if ((pg->flags & PG_RELEASED) != 0) { 463 released: 464 KASSERT(anon->an_ref == 0); 465 466 /* 467 * released while we unlocked amap. 468 */ 469 470 if (locked) 471 uvmfault_unlockall(ufi, amap, NULL, 472 NULL); 473 474 uvm_anon_release(anon); 475 476 if (error) { 477 UVMHIST_LOG(maphist, 478 "<- ERROR/RELEASED", 0,0,0,0); 479 return error; 480 } 481 482 UVMHIST_LOG(maphist, "<- RELEASED", 0,0,0,0); 483 return ERESTART; 484 } 485 486 /* 487 * we've successfully read the page, activate it. 488 */ 489 490 mutex_enter(&uvm_pageqlock); 491 uvm_pageactivate(pg); 492 mutex_exit(&uvm_pageqlock); 493 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE); 494 UVM_PAGE_OWN(pg, NULL); 495 if (!locked) 496 mutex_exit(&anon->an_lock); 497 #else /* defined(VMSWAP) */ 498 panic("%s: we_own", __func__); 499 #endif /* defined(VMSWAP) */ 500 } 501 502 /* 503 * we were not able to relock. restart fault. 504 */ 505 506 if (!locked) { 507 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 508 return (ERESTART); 509 } 510 511 /* 512 * verify no one has touched the amap and moved the anon on us. 513 */ 514 515 if (ufi != NULL && 516 amap_lookup(&ufi->entry->aref, 517 ufi->orig_rvaddr - ufi->entry->start) != anon) { 518 519 uvmfault_unlockall(ufi, amap, NULL, anon); 520 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 521 return (ERESTART); 522 } 523 524 /* 525 * try it again! 526 */ 527 528 uvmexp.fltanretry++; 529 continue; 530 } 531 /*NOTREACHED*/ 532 } 533 534 /* 535 * uvmfault_promote: promote data to a new anon. used for 1B and 2B. 536 * 537 * 1. allocate an anon and a page. 538 * 2. fill its contents. 539 * 3. put it into amap. 540 * 541 * => if we fail (result != 0) we unlock everything. 542 * => on success, return a new locked anon via 'nanon'. 543 * (*nanon)->an_page will be a resident, locked, dirty page. 544 */ 545 546 static int 547 uvmfault_promote(struct uvm_faultinfo *ufi, 548 struct vm_anon *oanon, 549 struct vm_page *uobjpage, 550 struct vm_anon **nanon, /* OUT: allocated anon */ 551 struct vm_anon **spare) 552 { 553 struct vm_amap *amap = ufi->entry->aref.ar_amap; 554 struct uvm_object *uobj; 555 struct vm_anon *anon; 556 struct vm_page *pg; 557 struct vm_page *opg; 558 int error; 559 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 560 561 if (oanon) { 562 /* anon COW */ 563 opg = oanon->an_page; 564 KASSERT(opg != NULL); 565 KASSERT(opg->uobject == NULL || opg->loan_count > 0); 566 } else if (uobjpage != PGO_DONTCARE) { 567 /* object-backed COW */ 568 opg = uobjpage; 569 } else { 570 /* ZFOD */ 571 opg = NULL; 572 } 573 if (opg != NULL) { 574 uobj = opg->uobject; 575 } else { 576 uobj = NULL; 577 } 578 579 KASSERT(amap != NULL); 580 KASSERT(uobjpage != NULL); 581 KASSERT(uobjpage == PGO_DONTCARE || (uobjpage->flags & PG_BUSY) != 0); 582 KASSERT(mutex_owned(&amap->am_l)); 583 KASSERT(oanon == NULL || mutex_owned(&oanon->an_lock)); 584 KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock)); 585 #if 0 586 KASSERT(*spare == NULL || !mutex_owned(&(*spare)->an_lock)); 587 #endif 588 589 if (*spare != NULL) { 590 anon = *spare; 591 *spare = NULL; 592 mutex_enter(&anon->an_lock); 593 } else if (ufi->map != kernel_map) { 594 anon = uvm_analloc(); 595 } else { 596 UVMHIST_LOG(maphist, "kernel_map, unlock and retry", 0,0,0,0); 597 598 /* 599 * we can't allocate anons with kernel_map locked. 600 */ 601 602 uvm_page_unbusy(&uobjpage, 1); 603 uvmfault_unlockall(ufi, amap, uobj, oanon); 604 605 *spare = uvm_analloc(); 606 if (*spare == NULL) { 607 goto nomem; 608 } 609 mutex_exit(&(*spare)->an_lock); 610 error = ERESTART; 611 goto done; 612 } 613 if (anon) { 614 615 /* 616 * The new anon is locked. 617 * 618 * if opg == NULL, we want a zero'd, dirty page, 619 * so have uvm_pagealloc() do that for us. 620 */ 621 622 pg = uvm_pagealloc(NULL, 0, anon, 623 (opg == NULL) ? UVM_PGA_ZERO : 0); 624 } else { 625 pg = NULL; 626 } 627 628 /* 629 * out of memory resources? 630 */ 631 632 if (pg == NULL) { 633 /* save anon for the next try. */ 634 if (anon != NULL) { 635 mutex_exit(&anon->an_lock); 636 *spare = anon; 637 } 638 639 /* unlock and fail ... */ 640 uvm_page_unbusy(&uobjpage, 1); 641 uvmfault_unlockall(ufi, amap, uobj, oanon); 642 nomem: 643 if (!uvm_reclaimable()) { 644 UVMHIST_LOG(maphist, "out of VM", 0,0,0,0); 645 uvmexp.fltnoanon++; 646 error = ENOMEM; 647 goto done; 648 } 649 650 UVMHIST_LOG(maphist, "out of RAM, waiting for more", 0,0,0,0); 651 uvmexp.fltnoram++; 652 uvm_wait("flt_noram5"); 653 error = ERESTART; 654 goto done; 655 } 656 657 /* copy page [pg now dirty] */ 658 if (opg) { 659 uvm_pagecopy(opg, pg); 660 } 661 662 amap_add(&ufi->entry->aref, ufi->orig_rvaddr - ufi->entry->start, anon, 663 oanon != NULL); 664 665 *nanon = anon; 666 error = 0; 667 done: 668 return error; 669 } 670 671 672 /* 673 * F A U L T - m a i n e n t r y p o i n t 674 */ 675 676 /* 677 * uvm_fault: page fault handler 678 * 679 * => called from MD code to resolve a page fault 680 * => VM data structures usually should be unlocked. however, it is 681 * possible to call here with the main map locked if the caller 682 * gets a write lock, sets it recusive, and then calls us (c.f. 683 * uvm_map_pageable). this should be avoided because it keeps 684 * the map locked off during I/O. 685 * => MUST NEVER BE CALLED IN INTERRUPT CONTEXT 686 */ 687 688 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 689 ~VM_PROT_WRITE : VM_PROT_ALL) 690 691 /* fault_flag values passed from uvm_fault_wire to uvm_fault_internal */ 692 #define UVM_FAULT_WIRE 1 693 #define UVM_FAULT_WIREMAX 2 694 695 int 696 uvm_fault_internal(struct vm_map *orig_map, vaddr_t vaddr, 697 vm_prot_t access_type, int fault_flag) 698 { 699 struct uvm_faultinfo ufi; 700 vm_prot_t enter_prot, check_prot; 701 bool wired, narrow, promote, locked, shadowed, wire_fault, cow_now; 702 int npages, nback, nforw, centeridx, error, lcv, gotpages; 703 vaddr_t startva, currva; 704 voff_t uoff; 705 struct vm_amap *amap; 706 struct uvm_object *uobj; 707 struct vm_anon *anons_store[UVM_MAXRANGE], **anons, *anon, *oanon; 708 struct vm_anon *anon_spare; 709 struct vm_page *pages[UVM_MAXRANGE], *pg, *uobjpage; 710 UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist); 711 712 UVMHIST_LOG(maphist, "(map=0x%x, vaddr=0x%x, at=%d, ff=%d)", 713 orig_map, vaddr, access_type, fault_flag); 714 715 anon = anon_spare = NULL; 716 pg = NULL; 717 718 uvmexp.faults++; /* XXX: locking? */ 719 720 /* 721 * init the IN parameters in the ufi 722 */ 723 724 ufi.orig_map = orig_map; 725 ufi.orig_rvaddr = trunc_page(vaddr); 726 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ 727 wire_fault = (fault_flag > 0); 728 if (wire_fault) 729 narrow = true; /* don't look for neighborhood 730 * pages on wire */ 731 else 732 narrow = false; /* normal fault */ 733 734 /* 735 * "goto ReFault" means restart the page fault from ground zero. 736 */ 737 ReFault: 738 739 /* 740 * lookup and lock the maps 741 */ 742 743 if (uvmfault_lookup(&ufi, false) == false) { 744 UVMHIST_LOG(maphist, "<- no mapping @ 0x%x", vaddr, 0,0,0); 745 error = EFAULT; 746 goto done; 747 } 748 /* locked: maps(read) */ 749 750 #ifdef DIAGNOSTIC 751 if ((ufi.map->flags & VM_MAP_PAGEABLE) == 0) { 752 printf("Page fault on non-pageable map:\n"); 753 printf("ufi.map = %p\n", ufi.map); 754 printf("ufi.orig_map = %p\n", ufi.orig_map); 755 printf("ufi.orig_rvaddr = 0x%lx\n", (u_long) ufi.orig_rvaddr); 756 panic("uvm_fault: (ufi.map->flags & VM_MAP_PAGEABLE) == 0"); 757 } 758 #endif 759 760 /* 761 * check protection 762 */ 763 764 check_prot = fault_flag == UVM_FAULT_WIREMAX ? 765 ufi.entry->max_protection : ufi.entry->protection; 766 if ((check_prot & access_type) != access_type) { 767 UVMHIST_LOG(maphist, 768 "<- protection failure (prot=0x%x, access=0x%x)", 769 ufi.entry->protection, access_type, 0, 0); 770 uvmfault_unlockmaps(&ufi, false); 771 error = EACCES; 772 goto done; 773 } 774 775 /* 776 * "enter_prot" is the protection we want to enter the page in at. 777 * for certain pages (e.g. copy-on-write pages) this protection can 778 * be more strict than ufi.entry->protection. "wired" means either 779 * the entry is wired or we are fault-wiring the pg. 780 */ 781 782 enter_prot = ufi.entry->protection; 783 wired = VM_MAPENT_ISWIRED(ufi.entry) || wire_fault; 784 if (wired) { 785 access_type = enter_prot; /* full access for wired */ 786 cow_now = (check_prot & VM_PROT_WRITE) != 0; 787 } else { 788 cow_now = (access_type & VM_PROT_WRITE) != 0; 789 } 790 791 /* 792 * handle "needs_copy" case. if we need to copy the amap we will 793 * have to drop our readlock and relock it with a write lock. (we 794 * need a write lock to change anything in a map entry [e.g. 795 * needs_copy]). 796 */ 797 798 if (UVM_ET_ISNEEDSCOPY(ufi.entry)) { 799 if (cow_now || (ufi.entry->object.uvm_obj == NULL)) { 800 KASSERT(fault_flag != UVM_FAULT_WIREMAX); 801 /* need to clear */ 802 UVMHIST_LOG(maphist, 803 " need to clear needs_copy and refault",0,0,0,0); 804 uvmfault_unlockmaps(&ufi, false); 805 uvmfault_amapcopy(&ufi); 806 uvmexp.fltamcopy++; 807 goto ReFault; 808 809 } else { 810 811 /* 812 * ensure that we pmap_enter page R/O since 813 * needs_copy is still true 814 */ 815 816 enter_prot &= ~VM_PROT_WRITE; 817 } 818 } 819 820 /* 821 * identify the players 822 */ 823 824 amap = ufi.entry->aref.ar_amap; /* upper layer */ 825 uobj = ufi.entry->object.uvm_obj; /* lower layer */ 826 827 /* 828 * check for a case 0 fault. if nothing backing the entry then 829 * error now. 830 */ 831 832 if (amap == NULL && uobj == NULL) { 833 uvmfault_unlockmaps(&ufi, false); 834 UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0); 835 error = EFAULT; 836 goto done; 837 } 838 839 /* 840 * establish range of interest based on advice from mapper 841 * and then clip to fit map entry. note that we only want 842 * to do this the first time through the fault. if we 843 * ReFault we will disable this by setting "narrow" to true. 844 */ 845 846 if (narrow == false) { 847 848 /* wide fault (!narrow) */ 849 KASSERT(uvmadvice[ufi.entry->advice].advice == 850 ufi.entry->advice); 851 nback = MIN(uvmadvice[ufi.entry->advice].nback, 852 (ufi.orig_rvaddr - ufi.entry->start) >> PAGE_SHIFT); 853 startva = ufi.orig_rvaddr - (nback << PAGE_SHIFT); 854 nforw = MIN(uvmadvice[ufi.entry->advice].nforw, 855 ((ufi.entry->end - ufi.orig_rvaddr) >> 856 PAGE_SHIFT) - 1); 857 /* 858 * note: "-1" because we don't want to count the 859 * faulting page as forw 860 */ 861 npages = nback + nforw + 1; 862 centeridx = nback; 863 864 narrow = true; /* ensure only once per-fault */ 865 866 } else { 867 868 /* narrow fault! */ 869 nback = nforw = 0; 870 startva = ufi.orig_rvaddr; 871 npages = 1; 872 centeridx = 0; 873 874 } 875 876 /* locked: maps(read) */ 877 UVMHIST_LOG(maphist, " narrow=%d, back=%d, forw=%d, startva=0x%x", 878 narrow, nback, nforw, startva); 879 UVMHIST_LOG(maphist, " entry=0x%x, amap=0x%x, obj=0x%x", ufi.entry, 880 amap, uobj, 0); 881 882 /* 883 * if we've got an amap, lock it and extract current anons. 884 */ 885 886 if (amap) { 887 amap_lock(amap); 888 anons = anons_store; 889 amap_lookups(&ufi.entry->aref, startva - ufi.entry->start, 890 anons, npages); 891 } else { 892 anons = NULL; /* to be safe */ 893 } 894 895 /* locked: maps(read), amap(if there) */ 896 KASSERT(amap == NULL || mutex_owned(&amap->am_l)); 897 898 /* 899 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages 900 * now and then forget about them (for the rest of the fault). 901 */ 902 903 if (ufi.entry->advice == MADV_SEQUENTIAL && nback != 0) { 904 905 UVMHIST_LOG(maphist, " MADV_SEQUENTIAL: flushing backpages", 906 0,0,0,0); 907 /* flush back-page anons? */ 908 if (amap) 909 uvmfault_anonflush(anons, nback); 910 911 /* flush object? */ 912 if (uobj) { 913 uoff = (startva - ufi.entry->start) + ufi.entry->offset; 914 mutex_enter(&uobj->vmobjlock); 915 (void) (uobj->pgops->pgo_put)(uobj, uoff, uoff + 916 (nback << PAGE_SHIFT), PGO_DEACTIVATE); 917 } 918 919 /* now forget about the backpages */ 920 if (amap) 921 anons += nback; 922 startva += (nback << PAGE_SHIFT); 923 npages -= nback; 924 nback = centeridx = 0; 925 } 926 927 /* locked: maps(read), amap(if there) */ 928 KASSERT(amap == NULL || mutex_owned(&amap->am_l)); 929 930 /* 931 * map in the backpages and frontpages we found in the amap in hopes 932 * of preventing future faults. we also init the pages[] array as 933 * we go. 934 */ 935 936 currva = startva; 937 shadowed = false; 938 for (lcv = 0 ; lcv < npages ; lcv++, currva += PAGE_SIZE) { 939 940 /* 941 * dont play with VAs that are already mapped 942 * except for center) 943 */ 944 if (lcv != centeridx && 945 pmap_extract(ufi.orig_map->pmap, currva, NULL)) { 946 pages[lcv] = PGO_DONTCARE; 947 continue; 948 } 949 950 /* 951 * unmapped or center page. check if any anon at this level. 952 */ 953 if (amap == NULL || anons[lcv] == NULL) { 954 pages[lcv] = NULL; 955 continue; 956 } 957 958 /* 959 * check for present page and map if possible. re-activate it. 960 */ 961 962 pages[lcv] = PGO_DONTCARE; 963 if (lcv == centeridx) { /* save center for later! */ 964 shadowed = true; 965 continue; 966 } 967 anon = anons[lcv]; 968 mutex_enter(&anon->an_lock); 969 /* ignore loaned pages */ 970 if (anon->an_page && anon->an_page->loan_count == 0 && 971 (anon->an_page->flags & PG_BUSY) == 0) { 972 mutex_enter(&uvm_pageqlock); 973 uvm_pageenqueue(anon->an_page); 974 mutex_exit(&uvm_pageqlock); 975 UVMHIST_LOG(maphist, 976 " MAPPING: n anon: pm=0x%x, va=0x%x, pg=0x%x", 977 ufi.orig_map->pmap, currva, anon->an_page, 0); 978 uvmexp.fltnamap++; 979 980 /* 981 * Since this isn't the page that's actually faulting, 982 * ignore pmap_enter() failures; it's not critical 983 * that we enter these right now. 984 */ 985 986 (void) pmap_enter(ufi.orig_map->pmap, currva, 987 VM_PAGE_TO_PHYS(anon->an_page), 988 (anon->an_ref > 1) ? (enter_prot & ~VM_PROT_WRITE) : 989 enter_prot, 990 PMAP_CANFAIL | 991 (VM_MAPENT_ISWIRED(ufi.entry) ? PMAP_WIRED : 0)); 992 } 993 pmap_update(ufi.orig_map->pmap); 994 mutex_exit(&anon->an_lock); 995 } 996 997 /* locked: maps(read), amap(if there) */ 998 KASSERT(amap == NULL || mutex_owned(&amap->am_l)); 999 /* (shadowed == true) if there is an anon at the faulting address */ 1000 UVMHIST_LOG(maphist, " shadowed=%d, will_get=%d", shadowed, 1001 (uobj && shadowed == false),0,0); 1002 1003 /* 1004 * note that if we are really short of RAM we could sleep in the above 1005 * call to pmap_enter with everything locked. bad? 1006 * 1007 * XXX Actually, that is bad; pmap_enter() should just fail in that 1008 * XXX case. --thorpej 1009 */ 1010 1011 /* 1012 * if the desired page is not shadowed by the amap and we have a 1013 * backing object, then we check to see if the backing object would 1014 * prefer to handle the fault itself (rather than letting us do it 1015 * with the usual pgo_get hook). the backing object signals this by 1016 * providing a pgo_fault routine. 1017 */ 1018 1019 if (uobj && shadowed == false && uobj->pgops->pgo_fault != NULL) { 1020 mutex_enter(&uobj->vmobjlock); 1021 /* locked: maps(read), amap (if there), uobj */ 1022 error = uobj->pgops->pgo_fault(&ufi, startva, pages, npages, 1023 centeridx, access_type, PGO_LOCKED|PGO_SYNCIO); 1024 1025 /* locked: nothing, pgo_fault has unlocked everything */ 1026 1027 if (error == ERESTART) 1028 goto ReFault; /* try again! */ 1029 /* 1030 * object fault routine responsible for pmap_update(). 1031 */ 1032 goto done; 1033 } 1034 1035 /* 1036 * now, if the desired page is not shadowed by the amap and we have 1037 * a backing object that does not have a special fault routine, then 1038 * we ask (with pgo_get) the object for resident pages that we care 1039 * about and attempt to map them in. we do not let pgo_get block 1040 * (PGO_LOCKED). 1041 */ 1042 1043 if (uobj && shadowed == false) { 1044 mutex_enter(&uobj->vmobjlock); 1045 /* locked (!shadowed): maps(read), amap (if there), uobj */ 1046 /* 1047 * the following call to pgo_get does _not_ change locking state 1048 */ 1049 1050 uvmexp.fltlget++; 1051 gotpages = npages; 1052 (void) uobj->pgops->pgo_get(uobj, ufi.entry->offset + 1053 (startva - ufi.entry->start), 1054 pages, &gotpages, centeridx, 1055 access_type & MASK(ufi.entry), 1056 ufi.entry->advice, PGO_LOCKED); 1057 1058 /* 1059 * check for pages to map, if we got any 1060 */ 1061 1062 uobjpage = NULL; 1063 1064 if (gotpages) { 1065 currva = startva; 1066 for (lcv = 0; lcv < npages; 1067 lcv++, currva += PAGE_SIZE) { 1068 struct vm_page *curpg; 1069 bool readonly; 1070 1071 curpg = pages[lcv]; 1072 if (curpg == NULL || curpg == PGO_DONTCARE) { 1073 continue; 1074 } 1075 KASSERT(curpg->uobject == uobj); 1076 1077 /* 1078 * if center page is resident and not 1079 * PG_BUSY|PG_RELEASED then pgo_get 1080 * made it PG_BUSY for us and gave 1081 * us a handle to it. remember this 1082 * page as "uobjpage." (for later use). 1083 */ 1084 1085 if (lcv == centeridx) { 1086 uobjpage = curpg; 1087 UVMHIST_LOG(maphist, " got uobjpage " 1088 "(0x%x) with locked get", 1089 uobjpage, 0,0,0); 1090 continue; 1091 } 1092 1093 /* 1094 * calling pgo_get with PGO_LOCKED returns us 1095 * pages which are neither busy nor released, 1096 * so we don't need to check for this. 1097 * we can just directly enter the pages. 1098 */ 1099 1100 mutex_enter(&uvm_pageqlock); 1101 uvm_pageenqueue(curpg); 1102 mutex_exit(&uvm_pageqlock); 1103 UVMHIST_LOG(maphist, 1104 " MAPPING: n obj: pm=0x%x, va=0x%x, pg=0x%x", 1105 ufi.orig_map->pmap, currva, curpg, 0); 1106 uvmexp.fltnomap++; 1107 1108 /* 1109 * Since this page isn't the page that's 1110 * actually faulting, ignore pmap_enter() 1111 * failures; it's not critical that we 1112 * enter these right now. 1113 */ 1114 KASSERT((curpg->flags & PG_PAGEOUT) == 0); 1115 KASSERT((curpg->flags & PG_RELEASED) == 0); 1116 KASSERT(!UVM_OBJ_IS_CLEAN(curpg->uobject) || 1117 (curpg->flags & PG_CLEAN) != 0); 1118 readonly = (curpg->flags & PG_RDONLY) 1119 || (curpg->loan_count > 0) 1120 || UVM_OBJ_NEEDS_WRITEFAULT(curpg->uobject); 1121 1122 (void) pmap_enter(ufi.orig_map->pmap, currva, 1123 VM_PAGE_TO_PHYS(curpg), 1124 readonly ? 1125 enter_prot & ~VM_PROT_WRITE : 1126 enter_prot & MASK(ufi.entry), 1127 PMAP_CANFAIL | 1128 (wired ? PMAP_WIRED : 0)); 1129 1130 /* 1131 * NOTE: page can't be PG_WANTED or PG_RELEASED 1132 * because we've held the lock the whole time 1133 * we've had the handle. 1134 */ 1135 KASSERT((curpg->flags & PG_WANTED) == 0); 1136 KASSERT((curpg->flags & PG_RELEASED) == 0); 1137 1138 curpg->flags &= ~(PG_BUSY); 1139 UVM_PAGE_OWN(curpg, NULL); 1140 } 1141 pmap_update(ufi.orig_map->pmap); 1142 } 1143 } else { 1144 uobjpage = NULL; 1145 } 1146 1147 /* locked (shadowed): maps(read), amap */ 1148 /* locked (!shadowed): maps(read), amap(if there), 1149 uobj(if !null), uobjpage(if !null) */ 1150 if (shadowed) { 1151 KASSERT(mutex_owned(&amap->am_l)); 1152 } else { 1153 KASSERT(amap == NULL || mutex_owned(&amap->am_l)); 1154 KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock)); 1155 KASSERT(uobjpage == NULL || (uobjpage->flags & PG_BUSY) != 0); 1156 } 1157 1158 /* 1159 * note that at this point we are done with any front or back pages. 1160 * we are now going to focus on the center page (i.e. the one we've 1161 * faulted on). if we have faulted on the upper (anon) layer 1162 * [i.e. case 1], then the anon we want is anons[centeridx] (we have 1163 * not touched it yet). if we have faulted on the bottom (uobj) 1164 * layer [i.e. case 2] and the page was both present and available, 1165 * then we've got a pointer to it as "uobjpage" and we've already 1166 * made it BUSY. 1167 */ 1168 1169 /* 1170 * there are four possible cases we must address: 1A, 1B, 2A, and 2B 1171 */ 1172 1173 /* 1174 * redirect case 2: if we are not shadowed, go to case 2. 1175 */ 1176 1177 if (shadowed == false) 1178 goto Case2; 1179 1180 /* locked: maps(read), amap */ 1181 1182 /* 1183 * handle case 1: fault on an anon in our amap 1184 */ 1185 1186 anon = anons[centeridx]; 1187 UVMHIST_LOG(maphist, " case 1 fault: anon=0x%x", anon, 0,0,0); 1188 mutex_enter(&anon->an_lock); 1189 1190 /* locked: maps(read), amap, anon */ 1191 KASSERT(mutex_owned(&amap->am_l)); 1192 KASSERT(mutex_owned(&anon->an_lock)); 1193 1194 /* 1195 * no matter if we have case 1A or case 1B we are going to need to 1196 * have the anon's memory resident. ensure that now. 1197 */ 1198 1199 /* 1200 * let uvmfault_anonget do the dirty work. 1201 * if it fails (!OK) it will unlock everything for us. 1202 * if it succeeds, locks are still valid and locked. 1203 * also, if it is OK, then the anon's page is on the queues. 1204 * if the page is on loan from a uvm_object, then anonget will 1205 * lock that object for us if it does not fail. 1206 */ 1207 1208 error = uvmfault_anonget(&ufi, amap, anon); 1209 switch (error) { 1210 case 0: 1211 break; 1212 1213 case ERESTART: 1214 goto ReFault; 1215 1216 case EAGAIN: 1217 kpause("fltagain1", false, hz/2, NULL); 1218 goto ReFault; 1219 1220 default: 1221 goto done; 1222 } 1223 1224 /* 1225 * uobj is non null if the page is on loan from an object (i.e. uobj) 1226 */ 1227 1228 uobj = anon->an_page->uobject; /* locked by anonget if !NULL */ 1229 1230 /* locked: maps(read), amap, anon, uobj(if one) */ 1231 KASSERT(mutex_owned(&amap->am_l)); 1232 KASSERT(mutex_owned(&anon->an_lock)); 1233 KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock)); 1234 1235 /* 1236 * special handling for loaned pages 1237 */ 1238 1239 if (anon->an_page->loan_count) { 1240 1241 if (!cow_now) { 1242 1243 /* 1244 * for read faults on loaned pages we just cap the 1245 * protection at read-only. 1246 */ 1247 1248 enter_prot = enter_prot & ~VM_PROT_WRITE; 1249 1250 } else { 1251 /* 1252 * note that we can't allow writes into a loaned page! 1253 * 1254 * if we have a write fault on a loaned page in an 1255 * anon then we need to look at the anon's ref count. 1256 * if it is greater than one then we are going to do 1257 * a normal copy-on-write fault into a new anon (this 1258 * is not a problem). however, if the reference count 1259 * is one (a case where we would normally allow a 1260 * write directly to the page) then we need to kill 1261 * the loan before we continue. 1262 */ 1263 1264 /* >1 case is already ok */ 1265 if (anon->an_ref == 1) { 1266 1267 /* get new un-owned replacement page */ 1268 pg = uvm_pagealloc(NULL, 0, NULL, 0); 1269 if (pg == NULL) { 1270 uvmfault_unlockall(&ufi, amap, uobj, 1271 anon); 1272 uvm_wait("flt_noram2"); 1273 goto ReFault; 1274 } 1275 1276 /* 1277 * copy data, kill loan, and drop uobj lock 1278 * (if any) 1279 */ 1280 /* copy old -> new */ 1281 uvm_pagecopy(anon->an_page, pg); 1282 1283 /* force reload */ 1284 pmap_page_protect(anon->an_page, VM_PROT_NONE); 1285 mutex_enter(&uvm_pageqlock); /* KILL loan */ 1286 1287 anon->an_page->uanon = NULL; 1288 /* in case we owned */ 1289 anon->an_page->pqflags &= ~PQ_ANON; 1290 1291 if (uobj) { 1292 /* if we were receiver of loan */ 1293 anon->an_page->loan_count--; 1294 } else { 1295 /* 1296 * we were the lender (A->K); need 1297 * to remove the page from pageq's. 1298 */ 1299 uvm_pagedequeue(anon->an_page); 1300 } 1301 1302 if (uobj) { 1303 mutex_exit(&uobj->vmobjlock); 1304 uobj = NULL; 1305 } 1306 1307 /* install new page in anon */ 1308 anon->an_page = pg; 1309 pg->uanon = anon; 1310 pg->pqflags |= PQ_ANON; 1311 1312 uvm_pageactivate(pg); 1313 mutex_exit(&uvm_pageqlock); 1314 1315 pg->flags &= ~(PG_BUSY|PG_FAKE); 1316 UVM_PAGE_OWN(pg, NULL); 1317 1318 /* done! */ 1319 } /* ref == 1 */ 1320 } /* write fault */ 1321 } /* loan count */ 1322 1323 /* 1324 * if we are case 1B then we will need to allocate a new blank 1325 * anon to transfer the data into. note that we have a lock 1326 * on anon, so no one can busy or release the page until we are done. 1327 * also note that the ref count can't drop to zero here because 1328 * it is > 1 and we are only dropping one ref. 1329 * 1330 * in the (hopefully very rare) case that we are out of RAM we 1331 * will unlock, wait for more RAM, and refault. 1332 * 1333 * if we are out of anon VM we kill the process (XXX: could wait?). 1334 */ 1335 1336 if (cow_now && anon->an_ref > 1) { 1337 1338 UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0); 1339 uvmexp.flt_acow++; 1340 oanon = anon; /* oanon = old, locked anon */ 1341 1342 error = uvmfault_promote(&ufi, oanon, PGO_DONTCARE, 1343 &anon, &anon_spare); 1344 switch (error) { 1345 case 0: 1346 break; 1347 case ERESTART: 1348 goto ReFault; 1349 default: 1350 goto done; 1351 } 1352 1353 pg = anon->an_page; 1354 mutex_enter(&uvm_pageqlock); 1355 uvm_pageactivate(pg); 1356 mutex_exit(&uvm_pageqlock); 1357 pg->flags &= ~(PG_BUSY|PG_FAKE); 1358 UVM_PAGE_OWN(pg, NULL); 1359 1360 /* deref: can not drop to zero here by defn! */ 1361 oanon->an_ref--; 1362 1363 /* 1364 * note: oanon is still locked, as is the new anon. we 1365 * need to check for this later when we unlock oanon; if 1366 * oanon != anon, we'll have to unlock anon, too. 1367 */ 1368 1369 } else { 1370 1371 uvmexp.flt_anon++; 1372 oanon = anon; /* old, locked anon is same as anon */ 1373 pg = anon->an_page; 1374 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */ 1375 enter_prot = enter_prot & ~VM_PROT_WRITE; 1376 1377 } 1378 1379 /* locked: maps(read), amap, oanon, anon (if different from oanon) */ 1380 KASSERT(mutex_owned(&amap->am_l)); 1381 KASSERT(mutex_owned(&anon->an_lock)); 1382 KASSERT(mutex_owned(&oanon->an_lock)); 1383 1384 /* 1385 * now map the page in. 1386 */ 1387 1388 UVMHIST_LOG(maphist, " MAPPING: anon: pm=0x%x, va=0x%x, pg=0x%x", 1389 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, 0); 1390 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1391 enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) 1392 != 0) { 1393 1394 /* 1395 * No need to undo what we did; we can simply think of 1396 * this as the pmap throwing away the mapping information. 1397 * 1398 * We do, however, have to go through the ReFault path, 1399 * as the map may change while we're asleep. 1400 */ 1401 1402 if (anon != oanon) 1403 mutex_exit(&anon->an_lock); 1404 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1405 if (!uvm_reclaimable()) { 1406 UVMHIST_LOG(maphist, 1407 "<- failed. out of VM",0,0,0,0); 1408 /* XXX instrumentation */ 1409 error = ENOMEM; 1410 goto done; 1411 } 1412 /* XXX instrumentation */ 1413 uvm_wait("flt_pmfail1"); 1414 goto ReFault; 1415 } 1416 1417 /* 1418 * ... update the page queues. 1419 */ 1420 1421 mutex_enter(&uvm_pageqlock); 1422 if (wire_fault) { 1423 uvm_pagewire(pg); 1424 1425 /* 1426 * since the now-wired page cannot be paged out, 1427 * release its swap resources for others to use. 1428 * since an anon with no swap cannot be PG_CLEAN, 1429 * clear its clean flag now. 1430 */ 1431 1432 pg->flags &= ~(PG_CLEAN); 1433 uvm_anon_dropswap(anon); 1434 } else { 1435 uvm_pageactivate(pg); 1436 } 1437 mutex_exit(&uvm_pageqlock); 1438 1439 /* 1440 * done case 1! finish up by unlocking everything and returning success 1441 */ 1442 1443 if (anon != oanon) 1444 mutex_exit(&anon->an_lock); 1445 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1446 pmap_update(ufi.orig_map->pmap); 1447 error = 0; 1448 goto done; 1449 1450 Case2: 1451 /* 1452 * handle case 2: faulting on backing object or zero fill 1453 */ 1454 1455 /* 1456 * locked: 1457 * maps(read), amap(if there), uobj(if !null), uobjpage(if !null) 1458 */ 1459 KASSERT(amap == NULL || mutex_owned(&amap->am_l)); 1460 KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock)); 1461 KASSERT(uobjpage == NULL || (uobjpage->flags & PG_BUSY) != 0); 1462 1463 /* 1464 * note that uobjpage can not be PGO_DONTCARE at this point. we now 1465 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we 1466 * have a backing object, check and see if we are going to promote 1467 * the data up to an anon during the fault. 1468 */ 1469 1470 if (uobj == NULL) { 1471 uobjpage = PGO_DONTCARE; 1472 promote = true; /* always need anon here */ 1473 } else { 1474 KASSERT(uobjpage != PGO_DONTCARE); 1475 promote = cow_now && UVM_ET_ISCOPYONWRITE(ufi.entry); 1476 } 1477 UVMHIST_LOG(maphist, " case 2 fault: promote=%d, zfill=%d", 1478 promote, (uobj == NULL), 0,0); 1479 1480 /* 1481 * if uobjpage is not null then we do not need to do I/O to get the 1482 * uobjpage. 1483 * 1484 * if uobjpage is null, then we need to unlock and ask the pager to 1485 * get the data for us. once we have the data, we need to reverify 1486 * the state the world. we are currently not holding any resources. 1487 */ 1488 1489 if (uobjpage) { 1490 /* update rusage counters */ 1491 curlwp->l_ru.ru_minflt++; 1492 } else { 1493 /* update rusage counters */ 1494 curlwp->l_ru.ru_majflt++; 1495 1496 /* locked: maps(read), amap(if there), uobj */ 1497 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1498 /* locked: uobj */ 1499 1500 uvmexp.fltget++; 1501 gotpages = 1; 1502 uoff = (ufi.orig_rvaddr - ufi.entry->start) + ufi.entry->offset; 1503 error = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages, 1504 0, access_type & MASK(ufi.entry), ufi.entry->advice, 1505 PGO_SYNCIO); 1506 /* locked: uobjpage(if no error) */ 1507 KASSERT(error != 0 || (uobjpage->flags & PG_BUSY) != 0); 1508 1509 /* 1510 * recover from I/O 1511 */ 1512 1513 if (error) { 1514 if (error == EAGAIN) { 1515 UVMHIST_LOG(maphist, 1516 " pgo_get says TRY AGAIN!",0,0,0,0); 1517 kpause("fltagain2", false, hz/2, NULL); 1518 goto ReFault; 1519 } 1520 1521 UVMHIST_LOG(maphist, "<- pgo_get failed (code %d)", 1522 error, 0,0,0); 1523 goto done; 1524 } 1525 1526 /* locked: uobjpage */ 1527 1528 mutex_enter(&uvm_pageqlock); 1529 uvm_pageactivate(uobjpage); 1530 mutex_exit(&uvm_pageqlock); 1531 1532 /* 1533 * re-verify the state of the world by first trying to relock 1534 * the maps. always relock the object. 1535 */ 1536 1537 locked = uvmfault_relock(&ufi); 1538 if (locked && amap) 1539 amap_lock(amap); 1540 uobj = uobjpage->uobject; 1541 mutex_enter(&uobj->vmobjlock); 1542 1543 /* locked(locked): maps(read), amap(if !null), uobj, uobjpage */ 1544 /* locked(!locked): uobj, uobjpage */ 1545 1546 /* 1547 * verify that the page has not be released and re-verify 1548 * that amap slot is still free. if there is a problem, 1549 * we unlock and clean up. 1550 */ 1551 1552 if ((uobjpage->flags & PG_RELEASED) != 0 || 1553 (locked && amap && 1554 amap_lookup(&ufi.entry->aref, 1555 ufi.orig_rvaddr - ufi.entry->start))) { 1556 if (locked) 1557 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1558 locked = false; 1559 } 1560 1561 /* 1562 * didn't get the lock? release the page and retry. 1563 */ 1564 1565 if (locked == false) { 1566 UVMHIST_LOG(maphist, 1567 " wasn't able to relock after fault: retry", 1568 0,0,0,0); 1569 if (uobjpage->flags & PG_WANTED) 1570 wakeup(uobjpage); 1571 if (uobjpage->flags & PG_RELEASED) { 1572 uvmexp.fltpgrele++; 1573 uvm_pagefree(uobjpage); 1574 goto ReFault; 1575 } 1576 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1577 UVM_PAGE_OWN(uobjpage, NULL); 1578 mutex_exit(&uobj->vmobjlock); 1579 goto ReFault; 1580 } 1581 1582 /* 1583 * we have the data in uobjpage which is busy and 1584 * not released. we are holding object lock (so the page 1585 * can't be released on us). 1586 */ 1587 1588 /* locked: maps(read), amap(if !null), uobj, uobjpage */ 1589 } 1590 1591 /* 1592 * locked: 1593 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj) 1594 */ 1595 KASSERT(amap == NULL || mutex_owned(&amap->am_l)); 1596 KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock)); 1597 KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0); 1598 1599 /* 1600 * notes: 1601 * - at this point uobjpage can not be NULL 1602 * - at this point uobjpage can not be PG_RELEASED (since we checked 1603 * for it above) 1604 * - at this point uobjpage could be PG_WANTED (handle later) 1605 */ 1606 1607 KASSERT(uobj == NULL || uobj == uobjpage->uobject); 1608 KASSERT(uobj == NULL || !UVM_OBJ_IS_CLEAN(uobjpage->uobject) || 1609 (uobjpage->flags & PG_CLEAN) != 0); 1610 if (promote == false) { 1611 1612 /* 1613 * we are not promoting. if the mapping is COW ensure that we 1614 * don't give more access than we should (e.g. when doing a read 1615 * fault on a COPYONWRITE mapping we want to map the COW page in 1616 * R/O even though the entry protection could be R/W). 1617 * 1618 * set "pg" to the page we want to map in (uobjpage, usually) 1619 */ 1620 1621 /* no anon in this case. */ 1622 anon = NULL; 1623 1624 uvmexp.flt_obj++; 1625 if (UVM_ET_ISCOPYONWRITE(ufi.entry) || 1626 UVM_OBJ_NEEDS_WRITEFAULT(uobjpage->uobject)) 1627 enter_prot &= ~VM_PROT_WRITE; 1628 pg = uobjpage; /* map in the actual object */ 1629 1630 KASSERT(uobjpage != PGO_DONTCARE); 1631 1632 /* 1633 * we are faulting directly on the page. be careful 1634 * about writing to loaned pages... 1635 */ 1636 1637 if (uobjpage->loan_count) { 1638 if (!cow_now) { 1639 /* read fault: cap the protection at readonly */ 1640 /* cap! */ 1641 enter_prot = enter_prot & ~VM_PROT_WRITE; 1642 } else { 1643 /* write fault: must break the loan here */ 1644 1645 pg = uvm_loanbreak(uobjpage); 1646 if (pg == NULL) { 1647 1648 /* 1649 * drop ownership of page, it can't 1650 * be released 1651 */ 1652 1653 if (uobjpage->flags & PG_WANTED) 1654 wakeup(uobjpage); 1655 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1656 UVM_PAGE_OWN(uobjpage, NULL); 1657 1658 uvmfault_unlockall(&ufi, amap, uobj, 1659 NULL); 1660 UVMHIST_LOG(maphist, 1661 " out of RAM breaking loan, waiting", 1662 0,0,0,0); 1663 uvmexp.fltnoram++; 1664 uvm_wait("flt_noram4"); 1665 goto ReFault; 1666 } 1667 uobjpage = pg; 1668 } 1669 } 1670 } else { 1671 1672 /* 1673 * if we are going to promote the data to an anon we 1674 * allocate a blank anon here and plug it into our amap. 1675 */ 1676 #if DIAGNOSTIC 1677 if (amap == NULL) 1678 panic("uvm_fault: want to promote data, but no anon"); 1679 #endif 1680 error = uvmfault_promote(&ufi, NULL, uobjpage, 1681 &anon, &anon_spare); 1682 switch (error) { 1683 case 0: 1684 break; 1685 case ERESTART: 1686 goto ReFault; 1687 default: 1688 goto done; 1689 } 1690 1691 pg = anon->an_page; 1692 1693 /* 1694 * fill in the data 1695 */ 1696 1697 if (uobjpage != PGO_DONTCARE) { 1698 uvmexp.flt_prcopy++; 1699 1700 /* 1701 * promote to shared amap? make sure all sharing 1702 * procs see it 1703 */ 1704 1705 if ((amap_flags(amap) & AMAP_SHARED) != 0) { 1706 pmap_page_protect(uobjpage, VM_PROT_NONE); 1707 /* 1708 * XXX: PAGE MIGHT BE WIRED! 1709 */ 1710 } 1711 1712 /* 1713 * dispose of uobjpage. it can't be PG_RELEASED 1714 * since we still hold the object lock. 1715 * drop handle to uobj as well. 1716 */ 1717 1718 if (uobjpage->flags & PG_WANTED) 1719 /* still have the obj lock */ 1720 wakeup(uobjpage); 1721 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1722 UVM_PAGE_OWN(uobjpage, NULL); 1723 mutex_exit(&uobj->vmobjlock); 1724 uobj = NULL; 1725 1726 UVMHIST_LOG(maphist, 1727 " promote uobjpage 0x%x to anon/page 0x%x/0x%x", 1728 uobjpage, anon, pg, 0); 1729 1730 } else { 1731 uvmexp.flt_przero++; 1732 1733 /* 1734 * Page is zero'd and marked dirty by 1735 * uvmfault_promote(). 1736 */ 1737 1738 UVMHIST_LOG(maphist," zero fill anon/page 0x%x/0%x", 1739 anon, pg, 0, 0); 1740 } 1741 } 1742 1743 /* 1744 * locked: 1745 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj), 1746 * anon(if !null), pg(if anon) 1747 * 1748 * note: pg is either the uobjpage or the new page in the new anon 1749 */ 1750 KASSERT(amap == NULL || mutex_owned(&amap->am_l)); 1751 KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock)); 1752 KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0); 1753 KASSERT(anon == NULL || mutex_owned(&anon->an_lock)); 1754 KASSERT((pg->flags & PG_BUSY) != 0); 1755 1756 /* 1757 * all resources are present. we can now map it in and free our 1758 * resources. 1759 */ 1760 1761 UVMHIST_LOG(maphist, 1762 " MAPPING: case2: pm=0x%x, va=0x%x, pg=0x%x, promote=%d", 1763 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, promote); 1764 KASSERT((access_type & VM_PROT_WRITE) == 0 || 1765 (pg->flags & PG_RDONLY) == 0); 1766 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1767 pg->flags & PG_RDONLY ? enter_prot & ~VM_PROT_WRITE : enter_prot, 1768 access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) != 0) { 1769 1770 /* 1771 * No need to undo what we did; we can simply think of 1772 * this as the pmap throwing away the mapping information. 1773 * 1774 * We do, however, have to go through the ReFault path, 1775 * as the map may change while we're asleep. 1776 */ 1777 1778 if (pg->flags & PG_WANTED) 1779 wakeup(pg); 1780 1781 /* 1782 * note that pg can't be PG_RELEASED since we did not drop 1783 * the object lock since the last time we checked. 1784 */ 1785 KASSERT((pg->flags & PG_RELEASED) == 0); 1786 1787 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); 1788 UVM_PAGE_OWN(pg, NULL); 1789 uvmfault_unlockall(&ufi, amap, uobj, anon); 1790 if (!uvm_reclaimable()) { 1791 UVMHIST_LOG(maphist, 1792 "<- failed. out of VM",0,0,0,0); 1793 /* XXX instrumentation */ 1794 error = ENOMEM; 1795 goto done; 1796 } 1797 /* XXX instrumentation */ 1798 uvm_wait("flt_pmfail2"); 1799 goto ReFault; 1800 } 1801 1802 mutex_enter(&uvm_pageqlock); 1803 if (wire_fault) { 1804 uvm_pagewire(pg); 1805 if (pg->pqflags & PQ_AOBJ) { 1806 1807 /* 1808 * since the now-wired page cannot be paged out, 1809 * release its swap resources for others to use. 1810 * since an aobj page with no swap cannot be PG_CLEAN, 1811 * clear its clean flag now. 1812 */ 1813 1814 KASSERT(uobj != NULL); 1815 pg->flags &= ~(PG_CLEAN); 1816 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT); 1817 } 1818 } else { 1819 uvm_pageactivate(pg); 1820 } 1821 mutex_exit(&uvm_pageqlock); 1822 if (pg->flags & PG_WANTED) 1823 wakeup(pg); 1824 1825 /* 1826 * note that pg can't be PG_RELEASED since we did not drop the object 1827 * lock since the last time we checked. 1828 */ 1829 KASSERT((pg->flags & PG_RELEASED) == 0); 1830 1831 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); 1832 UVM_PAGE_OWN(pg, NULL); 1833 uvmfault_unlockall(&ufi, amap, uobj, anon); 1834 pmap_update(ufi.orig_map->pmap); 1835 UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0); 1836 error = 0; 1837 done: 1838 if (anon_spare != NULL) { 1839 anon_spare->an_ref--; 1840 uvm_anfree(anon_spare); 1841 } 1842 return error; 1843 } 1844 1845 1846 /* 1847 * uvm_fault_wire: wire down a range of virtual addresses in a map. 1848 * 1849 * => map may be read-locked by caller, but MUST NOT be write-locked. 1850 * => if map is read-locked, any operations which may cause map to 1851 * be write-locked in uvm_fault() must be taken care of by 1852 * the caller. See uvm_map_pageable(). 1853 */ 1854 1855 int 1856 uvm_fault_wire(struct vm_map *map, vaddr_t start, vaddr_t end, 1857 vm_prot_t access_type, int wiremax) 1858 { 1859 vaddr_t va; 1860 int error; 1861 1862 /* 1863 * now fault it in a page at a time. if the fault fails then we have 1864 * to undo what we have done. note that in uvm_fault VM_PROT_NONE 1865 * is replaced with the max protection if fault_type is VM_FAULT_WIRE. 1866 */ 1867 1868 /* 1869 * XXX work around overflowing a vaddr_t. this prevents us from 1870 * wiring the last page in the address space, though. 1871 */ 1872 if (start > end) { 1873 return EFAULT; 1874 } 1875 1876 for (va = start ; va < end ; va += PAGE_SIZE) { 1877 error = uvm_fault_internal(map, va, access_type, 1878 wiremax ? UVM_FAULT_WIREMAX : UVM_FAULT_WIRE); 1879 if (error) { 1880 if (va != start) { 1881 uvm_fault_unwire(map, start, va); 1882 } 1883 return error; 1884 } 1885 } 1886 return 0; 1887 } 1888 1889 /* 1890 * uvm_fault_unwire(): unwire range of virtual space. 1891 */ 1892 1893 void 1894 uvm_fault_unwire(struct vm_map *map, vaddr_t start, vaddr_t end) 1895 { 1896 vm_map_lock_read(map); 1897 uvm_fault_unwire_locked(map, start, end); 1898 vm_map_unlock_read(map); 1899 } 1900 1901 /* 1902 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire(). 1903 * 1904 * => map must be at least read-locked. 1905 */ 1906 1907 void 1908 uvm_fault_unwire_locked(struct vm_map *map, vaddr_t start, vaddr_t end) 1909 { 1910 struct vm_map_entry *entry; 1911 pmap_t pmap = vm_map_pmap(map); 1912 vaddr_t va; 1913 paddr_t pa; 1914 struct vm_page *pg; 1915 1916 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0); 1917 1918 /* 1919 * we assume that the area we are unwiring has actually been wired 1920 * in the first place. this means that we should be able to extract 1921 * the PAs from the pmap. we also lock out the page daemon so that 1922 * we can call uvm_pageunwire. 1923 */ 1924 1925 mutex_enter(&uvm_pageqlock); 1926 1927 /* 1928 * find the beginning map entry for the region. 1929 */ 1930 1931 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map)); 1932 if (uvm_map_lookup_entry(map, start, &entry) == false) 1933 panic("uvm_fault_unwire_locked: address not in map"); 1934 1935 for (va = start; va < end; va += PAGE_SIZE) { 1936 if (pmap_extract(pmap, va, &pa) == false) 1937 continue; 1938 1939 /* 1940 * find the map entry for the current address. 1941 */ 1942 1943 KASSERT(va >= entry->start); 1944 while (va >= entry->end) { 1945 KASSERT(entry->next != &map->header && 1946 entry->next->start <= entry->end); 1947 entry = entry->next; 1948 } 1949 1950 /* 1951 * if the entry is no longer wired, tell the pmap. 1952 */ 1953 1954 if (VM_MAPENT_ISWIRED(entry) == 0) 1955 pmap_unwire(pmap, va); 1956 1957 pg = PHYS_TO_VM_PAGE(pa); 1958 if (pg) 1959 uvm_pageunwire(pg); 1960 } 1961 1962 mutex_exit(&uvm_pageqlock); 1963 } 1964