1 /* $NetBSD: uvm_fault.c,v 1.126 2008/12/20 11:33:38 ad Exp $ */ 2 3 /* 4 * 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by Charles D. Cranor and 19 * Washington University. 20 * 4. The name of the author may not be used to endorse or promote products 21 * derived from this software without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 24 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 25 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 26 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 27 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 28 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 32 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 * 34 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp 35 */ 36 37 /* 38 * uvm_fault.c: fault handler 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.126 2008/12/20 11:33:38 ad Exp $"); 43 44 #include "opt_uvmhist.h" 45 46 #include <sys/param.h> 47 #include <sys/systm.h> 48 #include <sys/kernel.h> 49 #include <sys/proc.h> 50 #include <sys/malloc.h> 51 #include <sys/mman.h> 52 #include <sys/user.h> 53 54 #include <uvm/uvm.h> 55 56 /* 57 * 58 * a word on page faults: 59 * 60 * types of page faults we handle: 61 * 62 * CASE 1: upper layer faults CASE 2: lower layer faults 63 * 64 * CASE 1A CASE 1B CASE 2A CASE 2B 65 * read/write1 write>1 read/write +-cow_write/zero 66 * | | | | 67 * +--|--+ +--|--+ +-----+ + | + | +-----+ 68 * amap | V | | ----------->new| | | | ^ | 69 * +-----+ +-----+ +-----+ + | + | +--|--+ 70 * | | | 71 * +-----+ +-----+ +--|--+ | +--|--+ 72 * uobj | d/c | | d/c | | V | +----| | 73 * +-----+ +-----+ +-----+ +-----+ 74 * 75 * d/c = don't care 76 * 77 * case [0]: layerless fault 78 * no amap or uobj is present. this is an error. 79 * 80 * case [1]: upper layer fault [anon active] 81 * 1A: [read] or [write with anon->an_ref == 1] 82 * I/O takes place in top level anon and uobj is not touched. 83 * 1B: [write with anon->an_ref > 1] 84 * new anon is alloc'd and data is copied off ["COW"] 85 * 86 * case [2]: lower layer fault [uobj] 87 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area] 88 * I/O takes place directly in object. 89 * 2B: [write to copy_on_write] or [read on NULL uobj] 90 * data is "promoted" from uobj to a new anon. 91 * if uobj is null, then we zero fill. 92 * 93 * we follow the standard UVM locking protocol ordering: 94 * 95 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ) 96 * we hold a PG_BUSY page if we unlock for I/O 97 * 98 * 99 * the code is structured as follows: 100 * 101 * - init the "IN" params in the ufi structure 102 * ReFault: 103 * - do lookups [locks maps], check protection, handle needs_copy 104 * - check for case 0 fault (error) 105 * - establish "range" of fault 106 * - if we have an amap lock it and extract the anons 107 * - if sequential advice deactivate pages behind us 108 * - at the same time check pmap for unmapped areas and anon for pages 109 * that we could map in (and do map it if found) 110 * - check object for resident pages that we could map in 111 * - if (case 2) goto Case2 112 * - >>> handle case 1 113 * - ensure source anon is resident in RAM 114 * - if case 1B alloc new anon and copy from source 115 * - map the correct page in 116 * Case2: 117 * - >>> handle case 2 118 * - ensure source page is resident (if uobj) 119 * - if case 2B alloc new anon and copy from source (could be zero 120 * fill if uobj == NULL) 121 * - map the correct page in 122 * - done! 123 * 124 * note on paging: 125 * if we have to do I/O we place a PG_BUSY page in the correct object, 126 * unlock everything, and do the I/O. when I/O is done we must reverify 127 * the state of the world before assuming that our data structures are 128 * valid. [because mappings could change while the map is unlocked] 129 * 130 * alternative 1: unbusy the page in question and restart the page fault 131 * from the top (ReFault). this is easy but does not take advantage 132 * of the information that we already have from our previous lookup, 133 * although it is possible that the "hints" in the vm_map will help here. 134 * 135 * alternative 2: the system already keeps track of a "version" number of 136 * a map. [i.e. every time you write-lock a map (e.g. to change a 137 * mapping) you bump the version number up by one...] so, we can save 138 * the version number of the map before we release the lock and start I/O. 139 * then when I/O is done we can relock and check the version numbers 140 * to see if anything changed. this might save us some over 1 because 141 * we don't have to unbusy the page and may be less compares(?). 142 * 143 * alternative 3: put in backpointers or a way to "hold" part of a map 144 * in place while I/O is in progress. this could be complex to 145 * implement (especially with structures like amap that can be referenced 146 * by multiple map entries, and figuring out what should wait could be 147 * complex as well...). 148 * 149 * we use alternative 2. given that we are multi-threaded now we may want 150 * to reconsider the choice. 151 */ 152 153 /* 154 * local data structures 155 */ 156 157 struct uvm_advice { 158 int advice; 159 int nback; 160 int nforw; 161 }; 162 163 /* 164 * page range array: 165 * note: index in array must match "advice" value 166 * XXX: borrowed numbers from freebsd. do they work well for us? 167 */ 168 169 static const struct uvm_advice uvmadvice[] = { 170 { MADV_NORMAL, 3, 4 }, 171 { MADV_RANDOM, 0, 0 }, 172 { MADV_SEQUENTIAL, 8, 7}, 173 }; 174 175 #define UVM_MAXRANGE 16 /* must be MAX() of nback+nforw+1 */ 176 177 /* 178 * private prototypes 179 */ 180 181 /* 182 * inline functions 183 */ 184 185 /* 186 * uvmfault_anonflush: try and deactivate pages in specified anons 187 * 188 * => does not have to deactivate page if it is busy 189 */ 190 191 static inline void 192 uvmfault_anonflush(struct vm_anon **anons, int n) 193 { 194 int lcv; 195 struct vm_page *pg; 196 197 for (lcv = 0 ; lcv < n ; lcv++) { 198 if (anons[lcv] == NULL) 199 continue; 200 mutex_enter(&anons[lcv]->an_lock); 201 pg = anons[lcv]->an_page; 202 if (pg && (pg->flags & PG_BUSY) == 0) { 203 mutex_enter(&uvm_pageqlock); 204 if (pg->wire_count == 0) { 205 uvm_pagedeactivate(pg); 206 } 207 mutex_exit(&uvm_pageqlock); 208 } 209 mutex_exit(&anons[lcv]->an_lock); 210 } 211 } 212 213 /* 214 * normal functions 215 */ 216 217 /* 218 * uvmfault_amapcopy: clear "needs_copy" in a map. 219 * 220 * => called with VM data structures unlocked (usually, see below) 221 * => we get a write lock on the maps and clear needs_copy for a VA 222 * => if we are out of RAM we sleep (waiting for more) 223 */ 224 225 static void 226 uvmfault_amapcopy(struct uvm_faultinfo *ufi) 227 { 228 for (;;) { 229 230 /* 231 * no mapping? give up. 232 */ 233 234 if (uvmfault_lookup(ufi, true) == false) 235 return; 236 237 /* 238 * copy if needed. 239 */ 240 241 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) 242 amap_copy(ufi->map, ufi->entry, AMAP_COPY_NOWAIT, 243 ufi->orig_rvaddr, ufi->orig_rvaddr + 1); 244 245 /* 246 * didn't work? must be out of RAM. unlock and sleep. 247 */ 248 249 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 250 uvmfault_unlockmaps(ufi, true); 251 uvm_wait("fltamapcopy"); 252 continue; 253 } 254 255 /* 256 * got it! unlock and return. 257 */ 258 259 uvmfault_unlockmaps(ufi, true); 260 return; 261 } 262 /*NOTREACHED*/ 263 } 264 265 /* 266 * uvmfault_anonget: get data in an anon into a non-busy, non-released 267 * page in that anon. 268 * 269 * => maps, amap, and anon locked by caller. 270 * => if we fail (result != 0) we unlock everything. 271 * => if we are successful, we return with everything still locked. 272 * => we don't move the page on the queues [gets moved later] 273 * => if we allocate a new page [we_own], it gets put on the queues. 274 * either way, the result is that the page is on the queues at return time 275 * => for pages which are on loan from a uvm_object (and thus are not 276 * owned by the anon): if successful, we return with the owning object 277 * locked. the caller must unlock this object when it unlocks everything 278 * else. 279 */ 280 281 int 282 uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap, 283 struct vm_anon *anon) 284 { 285 bool we_own; /* we own anon's page? */ 286 bool locked; /* did we relock? */ 287 struct vm_page *pg; 288 int error; 289 UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist); 290 291 KASSERT(mutex_owned(&anon->an_lock)); 292 293 error = 0; 294 uvmexp.fltanget++; 295 /* bump rusage counters */ 296 if (anon->an_page) 297 curlwp->l_ru.ru_minflt++; 298 else 299 curlwp->l_ru.ru_majflt++; 300 301 /* 302 * loop until we get it, or fail. 303 */ 304 305 for (;;) { 306 we_own = false; /* true if we set PG_BUSY on a page */ 307 pg = anon->an_page; 308 309 /* 310 * if there is a resident page and it is loaned, then anon 311 * may not own it. call out to uvm_anon_lockpage() to ensure 312 * the real owner of the page has been identified and locked. 313 */ 314 315 if (pg && pg->loan_count) 316 pg = uvm_anon_lockloanpg(anon); 317 318 /* 319 * page there? make sure it is not busy/released. 320 */ 321 322 if (pg) { 323 324 /* 325 * at this point, if the page has a uobject [meaning 326 * we have it on loan], then that uobject is locked 327 * by us! if the page is busy, we drop all the 328 * locks (including uobject) and try again. 329 */ 330 331 if ((pg->flags & PG_BUSY) == 0) { 332 UVMHIST_LOG(maphist, "<- OK",0,0,0,0); 333 return (0); 334 } 335 pg->flags |= PG_WANTED; 336 uvmexp.fltpgwait++; 337 338 /* 339 * the last unlock must be an atomic unlock+wait on 340 * the owner of page 341 */ 342 343 if (pg->uobject) { /* owner is uobject ? */ 344 uvmfault_unlockall(ufi, amap, NULL, anon); 345 UVMHIST_LOG(maphist, " unlock+wait on uobj",0, 346 0,0,0); 347 UVM_UNLOCK_AND_WAIT(pg, 348 &pg->uobject->vmobjlock, 349 false, "anonget1",0); 350 } else { 351 /* anon owns page */ 352 uvmfault_unlockall(ufi, amap, NULL, NULL); 353 UVMHIST_LOG(maphist, " unlock+wait on anon",0, 354 0,0,0); 355 UVM_UNLOCK_AND_WAIT(pg,&anon->an_lock,0, 356 "anonget2",0); 357 } 358 } else { 359 #if defined(VMSWAP) 360 361 /* 362 * no page, we must try and bring it in. 363 */ 364 365 pg = uvm_pagealloc(NULL, 0, anon, 0); 366 if (pg == NULL) { /* out of RAM. */ 367 uvmfault_unlockall(ufi, amap, NULL, anon); 368 uvmexp.fltnoram++; 369 UVMHIST_LOG(maphist, " noram -- UVM_WAIT",0, 370 0,0,0); 371 if (!uvm_reclaimable()) { 372 return ENOMEM; 373 } 374 uvm_wait("flt_noram1"); 375 } else { 376 /* we set the PG_BUSY bit */ 377 we_own = true; 378 uvmfault_unlockall(ufi, amap, NULL, anon); 379 380 /* 381 * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN 382 * page into the uvm_swap_get function with 383 * all data structures unlocked. note that 384 * it is ok to read an_swslot here because 385 * we hold PG_BUSY on the page. 386 */ 387 uvmexp.pageins++; 388 error = uvm_swap_get(pg, anon->an_swslot, 389 PGO_SYNCIO); 390 391 /* 392 * we clean up after the i/o below in the 393 * "we_own" case 394 */ 395 } 396 #else /* defined(VMSWAP) */ 397 panic("%s: no page", __func__); 398 #endif /* defined(VMSWAP) */ 399 } 400 401 /* 402 * now relock and try again 403 */ 404 405 locked = uvmfault_relock(ufi); 406 if (locked && amap != NULL) { 407 amap_lock(amap); 408 } 409 if (locked || we_own) 410 mutex_enter(&anon->an_lock); 411 412 /* 413 * if we own the page (i.e. we set PG_BUSY), then we need 414 * to clean up after the I/O. there are three cases to 415 * consider: 416 * [1] page released during I/O: free anon and ReFault. 417 * [2] I/O not OK. free the page and cause the fault 418 * to fail. 419 * [3] I/O OK! activate the page and sync with the 420 * non-we_own case (i.e. drop anon lock if not locked). 421 */ 422 423 if (we_own) { 424 #if defined(VMSWAP) 425 if (pg->flags & PG_WANTED) { 426 wakeup(pg); 427 } 428 if (error) { 429 430 /* 431 * remove the swap slot from the anon 432 * and mark the anon as having no real slot. 433 * don't free the swap slot, thus preventing 434 * it from being used again. 435 */ 436 437 if (anon->an_swslot > 0) 438 uvm_swap_markbad(anon->an_swslot, 1); 439 anon->an_swslot = SWSLOT_BAD; 440 441 if ((pg->flags & PG_RELEASED) != 0) 442 goto released; 443 444 /* 445 * note: page was never !PG_BUSY, so it 446 * can't be mapped and thus no need to 447 * pmap_page_protect it... 448 */ 449 450 mutex_enter(&uvm_pageqlock); 451 uvm_pagefree(pg); 452 mutex_exit(&uvm_pageqlock); 453 454 if (locked) 455 uvmfault_unlockall(ufi, amap, NULL, 456 anon); 457 else 458 mutex_exit(&anon->an_lock); 459 UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0); 460 return error; 461 } 462 463 if ((pg->flags & PG_RELEASED) != 0) { 464 released: 465 KASSERT(anon->an_ref == 0); 466 467 /* 468 * released while we unlocked amap. 469 */ 470 471 if (locked) 472 uvmfault_unlockall(ufi, amap, NULL, 473 NULL); 474 475 uvm_anon_release(anon); 476 477 if (error) { 478 UVMHIST_LOG(maphist, 479 "<- ERROR/RELEASED", 0,0,0,0); 480 return error; 481 } 482 483 UVMHIST_LOG(maphist, "<- RELEASED", 0,0,0,0); 484 return ERESTART; 485 } 486 487 /* 488 * we've successfully read the page, activate it. 489 */ 490 491 mutex_enter(&uvm_pageqlock); 492 uvm_pageactivate(pg); 493 mutex_exit(&uvm_pageqlock); 494 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE); 495 UVM_PAGE_OWN(pg, NULL); 496 if (!locked) 497 mutex_exit(&anon->an_lock); 498 #else /* defined(VMSWAP) */ 499 panic("%s: we_own", __func__); 500 #endif /* defined(VMSWAP) */ 501 } 502 503 /* 504 * we were not able to relock. restart fault. 505 */ 506 507 if (!locked) { 508 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 509 return (ERESTART); 510 } 511 512 /* 513 * verify no one has touched the amap and moved the anon on us. 514 */ 515 516 if (ufi != NULL && 517 amap_lookup(&ufi->entry->aref, 518 ufi->orig_rvaddr - ufi->entry->start) != anon) { 519 520 uvmfault_unlockall(ufi, amap, NULL, anon); 521 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 522 return (ERESTART); 523 } 524 525 /* 526 * try it again! 527 */ 528 529 uvmexp.fltanretry++; 530 continue; 531 } 532 /*NOTREACHED*/ 533 } 534 535 /* 536 * uvmfault_promote: promote data to a new anon. used for 1B and 2B. 537 * 538 * 1. allocate an anon and a page. 539 * 2. fill its contents. 540 * 3. put it into amap. 541 * 542 * => if we fail (result != 0) we unlock everything. 543 * => on success, return a new locked anon via 'nanon'. 544 * (*nanon)->an_page will be a resident, locked, dirty page. 545 */ 546 547 static int 548 uvmfault_promote(struct uvm_faultinfo *ufi, 549 struct vm_anon *oanon, 550 struct vm_page *uobjpage, 551 struct vm_anon **nanon, /* OUT: allocated anon */ 552 struct vm_anon **spare) 553 { 554 struct vm_amap *amap = ufi->entry->aref.ar_amap; 555 struct uvm_object *uobj; 556 struct vm_anon *anon; 557 struct vm_page *pg; 558 struct vm_page *opg; 559 int error; 560 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 561 562 if (oanon) { 563 /* anon COW */ 564 opg = oanon->an_page; 565 KASSERT(opg != NULL); 566 KASSERT(opg->uobject == NULL || opg->loan_count > 0); 567 } else if (uobjpage != PGO_DONTCARE) { 568 /* object-backed COW */ 569 opg = uobjpage; 570 } else { 571 /* ZFOD */ 572 opg = NULL; 573 } 574 if (opg != NULL) { 575 uobj = opg->uobject; 576 } else { 577 uobj = NULL; 578 } 579 580 KASSERT(amap != NULL); 581 KASSERT(uobjpage != NULL); 582 KASSERT(uobjpage == PGO_DONTCARE || (uobjpage->flags & PG_BUSY) != 0); 583 KASSERT(mutex_owned(&amap->am_l)); 584 KASSERT(oanon == NULL || mutex_owned(&oanon->an_lock)); 585 KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock)); 586 #if 0 587 KASSERT(*spare == NULL || !mutex_owned(&(*spare)->an_lock)); 588 #endif 589 590 if (*spare != NULL) { 591 anon = *spare; 592 *spare = NULL; 593 mutex_enter(&anon->an_lock); 594 } else if (ufi->map != kernel_map) { 595 anon = uvm_analloc(); 596 } else { 597 UVMHIST_LOG(maphist, "kernel_map, unlock and retry", 0,0,0,0); 598 599 /* 600 * we can't allocate anons with kernel_map locked. 601 */ 602 603 uvm_page_unbusy(&uobjpage, 1); 604 uvmfault_unlockall(ufi, amap, uobj, oanon); 605 606 *spare = uvm_analloc(); 607 if (*spare == NULL) { 608 goto nomem; 609 } 610 mutex_exit(&(*spare)->an_lock); 611 error = ERESTART; 612 goto done; 613 } 614 if (anon) { 615 616 /* 617 * The new anon is locked. 618 * 619 * if opg == NULL, we want a zero'd, dirty page, 620 * so have uvm_pagealloc() do that for us. 621 */ 622 623 pg = uvm_pagealloc(NULL, 0, anon, 624 (opg == NULL) ? UVM_PGA_ZERO : 0); 625 } else { 626 pg = NULL; 627 } 628 629 /* 630 * out of memory resources? 631 */ 632 633 if (pg == NULL) { 634 /* save anon for the next try. */ 635 if (anon != NULL) { 636 mutex_exit(&anon->an_lock); 637 *spare = anon; 638 } 639 640 /* unlock and fail ... */ 641 uvm_page_unbusy(&uobjpage, 1); 642 uvmfault_unlockall(ufi, amap, uobj, oanon); 643 nomem: 644 if (!uvm_reclaimable()) { 645 UVMHIST_LOG(maphist, "out of VM", 0,0,0,0); 646 uvmexp.fltnoanon++; 647 error = ENOMEM; 648 goto done; 649 } 650 651 UVMHIST_LOG(maphist, "out of RAM, waiting for more", 0,0,0,0); 652 uvmexp.fltnoram++; 653 uvm_wait("flt_noram5"); 654 error = ERESTART; 655 goto done; 656 } 657 658 /* copy page [pg now dirty] */ 659 if (opg) { 660 uvm_pagecopy(opg, pg); 661 } 662 663 amap_add(&ufi->entry->aref, ufi->orig_rvaddr - ufi->entry->start, anon, 664 oanon != NULL); 665 666 *nanon = anon; 667 error = 0; 668 done: 669 return error; 670 } 671 672 673 /* 674 * F A U L T - m a i n e n t r y p o i n t 675 */ 676 677 /* 678 * uvm_fault: page fault handler 679 * 680 * => called from MD code to resolve a page fault 681 * => VM data structures usually should be unlocked. however, it is 682 * possible to call here with the main map locked if the caller 683 * gets a write lock, sets it recusive, and then calls us (c.f. 684 * uvm_map_pageable). this should be avoided because it keeps 685 * the map locked off during I/O. 686 * => MUST NEVER BE CALLED IN INTERRUPT CONTEXT 687 */ 688 689 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 690 ~VM_PROT_WRITE : VM_PROT_ALL) 691 692 /* fault_flag values passed from uvm_fault_wire to uvm_fault_internal */ 693 #define UVM_FAULT_WIRE 1 694 #define UVM_FAULT_WIREMAX 2 695 696 int 697 uvm_fault_internal(struct vm_map *orig_map, vaddr_t vaddr, 698 vm_prot_t access_type, int fault_flag) 699 { 700 struct uvm_faultinfo ufi; 701 vm_prot_t enter_prot, check_prot; 702 bool wired, narrow, promote, locked, shadowed, wire_fault, cow_now; 703 int npages, nback, nforw, centeridx, error, lcv, gotpages; 704 vaddr_t startva, currva; 705 voff_t uoff; 706 struct vm_amap *amap; 707 struct uvm_object *uobj; 708 struct vm_anon *anons_store[UVM_MAXRANGE], **anons, *anon, *oanon; 709 struct vm_anon *anon_spare; 710 struct vm_page *pages[UVM_MAXRANGE], *pg, *uobjpage; 711 UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist); 712 713 UVMHIST_LOG(maphist, "(map=0x%x, vaddr=0x%x, at=%d, ff=%d)", 714 orig_map, vaddr, access_type, fault_flag); 715 716 anon = anon_spare = NULL; 717 pg = NULL; 718 719 uvmexp.faults++; /* XXX: locking? */ 720 721 /* 722 * init the IN parameters in the ufi 723 */ 724 725 ufi.orig_map = orig_map; 726 ufi.orig_rvaddr = trunc_page(vaddr); 727 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ 728 wire_fault = (fault_flag > 0); 729 if (wire_fault) 730 narrow = true; /* don't look for neighborhood 731 * pages on wire */ 732 else 733 narrow = false; /* normal fault */ 734 735 /* 736 * "goto ReFault" means restart the page fault from ground zero. 737 */ 738 ReFault: 739 740 /* 741 * lookup and lock the maps 742 */ 743 744 if (uvmfault_lookup(&ufi, false) == false) { 745 UVMHIST_LOG(maphist, "<- no mapping @ 0x%x", vaddr, 0,0,0); 746 error = EFAULT; 747 goto done; 748 } 749 /* locked: maps(read) */ 750 751 #ifdef DIAGNOSTIC 752 if ((ufi.map->flags & VM_MAP_PAGEABLE) == 0) { 753 printf("Page fault on non-pageable map:\n"); 754 printf("ufi.map = %p\n", ufi.map); 755 printf("ufi.orig_map = %p\n", ufi.orig_map); 756 printf("ufi.orig_rvaddr = 0x%lx\n", (u_long) ufi.orig_rvaddr); 757 panic("uvm_fault: (ufi.map->flags & VM_MAP_PAGEABLE) == 0"); 758 } 759 #endif 760 761 /* 762 * check protection 763 */ 764 765 check_prot = fault_flag == UVM_FAULT_WIREMAX ? 766 ufi.entry->max_protection : ufi.entry->protection; 767 if ((check_prot & access_type) != access_type) { 768 UVMHIST_LOG(maphist, 769 "<- protection failure (prot=0x%x, access=0x%x)", 770 ufi.entry->protection, access_type, 0, 0); 771 uvmfault_unlockmaps(&ufi, false); 772 error = EACCES; 773 goto done; 774 } 775 776 /* 777 * "enter_prot" is the protection we want to enter the page in at. 778 * for certain pages (e.g. copy-on-write pages) this protection can 779 * be more strict than ufi.entry->protection. "wired" means either 780 * the entry is wired or we are fault-wiring the pg. 781 */ 782 783 enter_prot = ufi.entry->protection; 784 wired = VM_MAPENT_ISWIRED(ufi.entry) || wire_fault; 785 if (wired) { 786 access_type = enter_prot; /* full access for wired */ 787 cow_now = (check_prot & VM_PROT_WRITE) != 0; 788 } else { 789 cow_now = (access_type & VM_PROT_WRITE) != 0; 790 } 791 792 /* 793 * handle "needs_copy" case. if we need to copy the amap we will 794 * have to drop our readlock and relock it with a write lock. (we 795 * need a write lock to change anything in a map entry [e.g. 796 * needs_copy]). 797 */ 798 799 if (UVM_ET_ISNEEDSCOPY(ufi.entry)) { 800 if (cow_now || (ufi.entry->object.uvm_obj == NULL)) { 801 KASSERT(fault_flag != UVM_FAULT_WIREMAX); 802 /* need to clear */ 803 UVMHIST_LOG(maphist, 804 " need to clear needs_copy and refault",0,0,0,0); 805 uvmfault_unlockmaps(&ufi, false); 806 uvmfault_amapcopy(&ufi); 807 uvmexp.fltamcopy++; 808 goto ReFault; 809 810 } else { 811 812 /* 813 * ensure that we pmap_enter page R/O since 814 * needs_copy is still true 815 */ 816 817 enter_prot &= ~VM_PROT_WRITE; 818 } 819 } 820 821 /* 822 * identify the players 823 */ 824 825 amap = ufi.entry->aref.ar_amap; /* top layer */ 826 uobj = ufi.entry->object.uvm_obj; /* bottom layer */ 827 828 /* 829 * check for a case 0 fault. if nothing backing the entry then 830 * error now. 831 */ 832 833 if (amap == NULL && uobj == NULL) { 834 uvmfault_unlockmaps(&ufi, false); 835 UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0); 836 error = EFAULT; 837 goto done; 838 } 839 840 /* 841 * establish range of interest based on advice from mapper 842 * and then clip to fit map entry. note that we only want 843 * to do this the first time through the fault. if we 844 * ReFault we will disable this by setting "narrow" to true. 845 */ 846 847 if (narrow == false) { 848 849 /* wide fault (!narrow) */ 850 KASSERT(uvmadvice[ufi.entry->advice].advice == 851 ufi.entry->advice); 852 nback = MIN(uvmadvice[ufi.entry->advice].nback, 853 (ufi.orig_rvaddr - ufi.entry->start) >> PAGE_SHIFT); 854 startva = ufi.orig_rvaddr - (nback << PAGE_SHIFT); 855 nforw = MIN(uvmadvice[ufi.entry->advice].nforw, 856 ((ufi.entry->end - ufi.orig_rvaddr) >> 857 PAGE_SHIFT) - 1); 858 /* 859 * note: "-1" because we don't want to count the 860 * faulting page as forw 861 */ 862 npages = nback + nforw + 1; 863 centeridx = nback; 864 865 narrow = true; /* ensure only once per-fault */ 866 867 } else { 868 869 /* narrow fault! */ 870 nback = nforw = 0; 871 startva = ufi.orig_rvaddr; 872 npages = 1; 873 centeridx = 0; 874 875 } 876 877 /* locked: maps(read) */ 878 UVMHIST_LOG(maphist, " narrow=%d, back=%d, forw=%d, startva=0x%x", 879 narrow, nback, nforw, startva); 880 UVMHIST_LOG(maphist, " entry=0x%x, amap=0x%x, obj=0x%x", ufi.entry, 881 amap, uobj, 0); 882 883 /* 884 * if we've got an amap, lock it and extract current anons. 885 */ 886 887 if (amap) { 888 amap_lock(amap); 889 anons = anons_store; 890 amap_lookups(&ufi.entry->aref, startva - ufi.entry->start, 891 anons, npages); 892 } else { 893 anons = NULL; /* to be safe */ 894 } 895 896 /* locked: maps(read), amap(if there) */ 897 KASSERT(amap == NULL || mutex_owned(&amap->am_l)); 898 899 /* 900 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages 901 * now and then forget about them (for the rest of the fault). 902 */ 903 904 if (ufi.entry->advice == MADV_SEQUENTIAL && nback != 0) { 905 906 UVMHIST_LOG(maphist, " MADV_SEQUENTIAL: flushing backpages", 907 0,0,0,0); 908 /* flush back-page anons? */ 909 if (amap) 910 uvmfault_anonflush(anons, nback); 911 912 /* flush object? */ 913 if (uobj) { 914 uoff = (startva - ufi.entry->start) + ufi.entry->offset; 915 mutex_enter(&uobj->vmobjlock); 916 (void) (uobj->pgops->pgo_put)(uobj, uoff, uoff + 917 (nback << PAGE_SHIFT), PGO_DEACTIVATE); 918 } 919 920 /* now forget about the backpages */ 921 if (amap) 922 anons += nback; 923 startva += (nback << PAGE_SHIFT); 924 npages -= nback; 925 nback = centeridx = 0; 926 } 927 928 /* locked: maps(read), amap(if there) */ 929 KASSERT(amap == NULL || mutex_owned(&amap->am_l)); 930 931 /* 932 * map in the backpages and frontpages we found in the amap in hopes 933 * of preventing future faults. we also init the pages[] array as 934 * we go. 935 */ 936 937 currva = startva; 938 shadowed = false; 939 for (lcv = 0 ; lcv < npages ; lcv++, currva += PAGE_SIZE) { 940 941 /* 942 * dont play with VAs that are already mapped 943 * except for center) 944 */ 945 if (lcv != centeridx && 946 pmap_extract(ufi.orig_map->pmap, currva, NULL)) { 947 pages[lcv] = PGO_DONTCARE; 948 continue; 949 } 950 951 /* 952 * unmapped or center page. check if any anon at this level. 953 */ 954 if (amap == NULL || anons[lcv] == NULL) { 955 pages[lcv] = NULL; 956 continue; 957 } 958 959 /* 960 * check for present page and map if possible. re-activate it. 961 */ 962 963 pages[lcv] = PGO_DONTCARE; 964 if (lcv == centeridx) { /* save center for later! */ 965 shadowed = true; 966 continue; 967 } 968 anon = anons[lcv]; 969 mutex_enter(&anon->an_lock); 970 /* ignore loaned pages */ 971 if (anon->an_page && anon->an_page->loan_count == 0 && 972 (anon->an_page->flags & PG_BUSY) == 0) { 973 mutex_enter(&uvm_pageqlock); 974 uvm_pageenqueue(anon->an_page); 975 mutex_exit(&uvm_pageqlock); 976 UVMHIST_LOG(maphist, 977 " MAPPING: n anon: pm=0x%x, va=0x%x, pg=0x%x", 978 ufi.orig_map->pmap, currva, anon->an_page, 0); 979 uvmexp.fltnamap++; 980 981 /* 982 * Since this isn't the page that's actually faulting, 983 * ignore pmap_enter() failures; it's not critical 984 * that we enter these right now. 985 */ 986 987 (void) pmap_enter(ufi.orig_map->pmap, currva, 988 VM_PAGE_TO_PHYS(anon->an_page), 989 (anon->an_ref > 1) ? (enter_prot & ~VM_PROT_WRITE) : 990 enter_prot, 991 PMAP_CANFAIL | 992 (VM_MAPENT_ISWIRED(ufi.entry) ? PMAP_WIRED : 0)); 993 } 994 pmap_update(ufi.orig_map->pmap); 995 mutex_exit(&anon->an_lock); 996 } 997 998 /* locked: maps(read), amap(if there) */ 999 KASSERT(amap == NULL || mutex_owned(&amap->am_l)); 1000 /* (shadowed == true) if there is an anon at the faulting address */ 1001 UVMHIST_LOG(maphist, " shadowed=%d, will_get=%d", shadowed, 1002 (uobj && shadowed == false),0,0); 1003 1004 /* 1005 * note that if we are really short of RAM we could sleep in the above 1006 * call to pmap_enter with everything locked. bad? 1007 * 1008 * XXX Actually, that is bad; pmap_enter() should just fail in that 1009 * XXX case. --thorpej 1010 */ 1011 1012 /* 1013 * if the desired page is not shadowed by the amap and we have a 1014 * backing object, then we check to see if the backing object would 1015 * prefer to handle the fault itself (rather than letting us do it 1016 * with the usual pgo_get hook). the backing object signals this by 1017 * providing a pgo_fault routine. 1018 */ 1019 1020 if (uobj && shadowed == false && uobj->pgops->pgo_fault != NULL) { 1021 mutex_enter(&uobj->vmobjlock); 1022 /* locked: maps(read), amap (if there), uobj */ 1023 error = uobj->pgops->pgo_fault(&ufi, startva, pages, npages, 1024 centeridx, access_type, PGO_LOCKED|PGO_SYNCIO); 1025 1026 /* locked: nothing, pgo_fault has unlocked everything */ 1027 1028 if (error == ERESTART) 1029 goto ReFault; /* try again! */ 1030 /* 1031 * object fault routine responsible for pmap_update(). 1032 */ 1033 goto done; 1034 } 1035 1036 /* 1037 * now, if the desired page is not shadowed by the amap and we have 1038 * a backing object that does not have a special fault routine, then 1039 * we ask (with pgo_get) the object for resident pages that we care 1040 * about and attempt to map them in. we do not let pgo_get block 1041 * (PGO_LOCKED). 1042 */ 1043 1044 if (uobj && shadowed == false) { 1045 mutex_enter(&uobj->vmobjlock); 1046 /* locked (!shadowed): maps(read), amap (if there), uobj */ 1047 /* 1048 * the following call to pgo_get does _not_ change locking state 1049 */ 1050 1051 uvmexp.fltlget++; 1052 gotpages = npages; 1053 (void) uobj->pgops->pgo_get(uobj, ufi.entry->offset + 1054 (startva - ufi.entry->start), 1055 pages, &gotpages, centeridx, 1056 access_type & MASK(ufi.entry), 1057 ufi.entry->advice, PGO_LOCKED); 1058 1059 /* 1060 * check for pages to map, if we got any 1061 */ 1062 1063 uobjpage = NULL; 1064 1065 if (gotpages) { 1066 currva = startva; 1067 for (lcv = 0; lcv < npages; 1068 lcv++, currva += PAGE_SIZE) { 1069 struct vm_page *curpg; 1070 bool readonly; 1071 1072 curpg = pages[lcv]; 1073 if (curpg == NULL || curpg == PGO_DONTCARE) { 1074 continue; 1075 } 1076 KASSERT(curpg->uobject == uobj); 1077 1078 /* 1079 * if center page is resident and not 1080 * PG_BUSY|PG_RELEASED then pgo_get 1081 * made it PG_BUSY for us and gave 1082 * us a handle to it. remember this 1083 * page as "uobjpage." (for later use). 1084 */ 1085 1086 if (lcv == centeridx) { 1087 uobjpage = curpg; 1088 UVMHIST_LOG(maphist, " got uobjpage " 1089 "(0x%x) with locked get", 1090 uobjpage, 0,0,0); 1091 continue; 1092 } 1093 1094 /* 1095 * calling pgo_get with PGO_LOCKED returns us 1096 * pages which are neither busy nor released, 1097 * so we don't need to check for this. 1098 * we can just directly enter the pages. 1099 */ 1100 1101 mutex_enter(&uvm_pageqlock); 1102 uvm_pageenqueue(curpg); 1103 mutex_exit(&uvm_pageqlock); 1104 UVMHIST_LOG(maphist, 1105 " MAPPING: n obj: pm=0x%x, va=0x%x, pg=0x%x", 1106 ufi.orig_map->pmap, currva, curpg, 0); 1107 uvmexp.fltnomap++; 1108 1109 /* 1110 * Since this page isn't the page that's 1111 * actually faulting, ignore pmap_enter() 1112 * failures; it's not critical that we 1113 * enter these right now. 1114 */ 1115 KASSERT((curpg->flags & PG_PAGEOUT) == 0); 1116 KASSERT((curpg->flags & PG_RELEASED) == 0); 1117 KASSERT(!UVM_OBJ_IS_CLEAN(curpg->uobject) || 1118 (curpg->flags & PG_CLEAN) != 0); 1119 readonly = (curpg->flags & PG_RDONLY) 1120 || (curpg->loan_count > 0) 1121 || UVM_OBJ_NEEDS_WRITEFAULT(curpg->uobject); 1122 1123 (void) pmap_enter(ufi.orig_map->pmap, currva, 1124 VM_PAGE_TO_PHYS(curpg), 1125 readonly ? 1126 enter_prot & ~VM_PROT_WRITE : 1127 enter_prot & MASK(ufi.entry), 1128 PMAP_CANFAIL | 1129 (wired ? PMAP_WIRED : 0)); 1130 1131 /* 1132 * NOTE: page can't be PG_WANTED or PG_RELEASED 1133 * because we've held the lock the whole time 1134 * we've had the handle. 1135 */ 1136 KASSERT((curpg->flags & PG_WANTED) == 0); 1137 KASSERT((curpg->flags & PG_RELEASED) == 0); 1138 1139 curpg->flags &= ~(PG_BUSY); 1140 UVM_PAGE_OWN(curpg, NULL); 1141 } 1142 pmap_update(ufi.orig_map->pmap); 1143 } 1144 } else { 1145 uobjpage = NULL; 1146 } 1147 1148 /* locked (shadowed): maps(read), amap */ 1149 /* locked (!shadowed): maps(read), amap(if there), 1150 uobj(if !null), uobjpage(if !null) */ 1151 if (shadowed) { 1152 KASSERT(mutex_owned(&amap->am_l)); 1153 } else { 1154 KASSERT(amap == NULL || mutex_owned(&amap->am_l)); 1155 KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock)); 1156 KASSERT(uobjpage == NULL || (uobjpage->flags & PG_BUSY) != 0); 1157 } 1158 1159 /* 1160 * note that at this point we are done with any front or back pages. 1161 * we are now going to focus on the center page (i.e. the one we've 1162 * faulted on). if we have faulted on the top (anon) layer 1163 * [i.e. case 1], then the anon we want is anons[centeridx] (we have 1164 * not touched it yet). if we have faulted on the bottom (uobj) 1165 * layer [i.e. case 2] and the page was both present and available, 1166 * then we've got a pointer to it as "uobjpage" and we've already 1167 * made it BUSY. 1168 */ 1169 1170 /* 1171 * there are four possible cases we must address: 1A, 1B, 2A, and 2B 1172 */ 1173 1174 /* 1175 * redirect case 2: if we are not shadowed, go to case 2. 1176 */ 1177 1178 if (shadowed == false) 1179 goto Case2; 1180 1181 /* locked: maps(read), amap */ 1182 1183 /* 1184 * handle case 1: fault on an anon in our amap 1185 */ 1186 1187 anon = anons[centeridx]; 1188 UVMHIST_LOG(maphist, " case 1 fault: anon=0x%x", anon, 0,0,0); 1189 mutex_enter(&anon->an_lock); 1190 1191 /* locked: maps(read), amap, anon */ 1192 KASSERT(mutex_owned(&amap->am_l)); 1193 KASSERT(mutex_owned(&anon->an_lock)); 1194 1195 /* 1196 * no matter if we have case 1A or case 1B we are going to need to 1197 * have the anon's memory resident. ensure that now. 1198 */ 1199 1200 /* 1201 * let uvmfault_anonget do the dirty work. 1202 * if it fails (!OK) it will unlock everything for us. 1203 * if it succeeds, locks are still valid and locked. 1204 * also, if it is OK, then the anon's page is on the queues. 1205 * if the page is on loan from a uvm_object, then anonget will 1206 * lock that object for us if it does not fail. 1207 */ 1208 1209 error = uvmfault_anonget(&ufi, amap, anon); 1210 switch (error) { 1211 case 0: 1212 break; 1213 1214 case ERESTART: 1215 goto ReFault; 1216 1217 case EAGAIN: 1218 tsleep(&lbolt, PVM, "fltagain1", 0); 1219 goto ReFault; 1220 1221 default: 1222 goto done; 1223 } 1224 1225 /* 1226 * uobj is non null if the page is on loan from an object (i.e. uobj) 1227 */ 1228 1229 uobj = anon->an_page->uobject; /* locked by anonget if !NULL */ 1230 1231 /* locked: maps(read), amap, anon, uobj(if one) */ 1232 KASSERT(mutex_owned(&amap->am_l)); 1233 KASSERT(mutex_owned(&anon->an_lock)); 1234 KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock)); 1235 1236 /* 1237 * special handling for loaned pages 1238 */ 1239 1240 if (anon->an_page->loan_count) { 1241 1242 if (!cow_now) { 1243 1244 /* 1245 * for read faults on loaned pages we just cap the 1246 * protection at read-only. 1247 */ 1248 1249 enter_prot = enter_prot & ~VM_PROT_WRITE; 1250 1251 } else { 1252 /* 1253 * note that we can't allow writes into a loaned page! 1254 * 1255 * if we have a write fault on a loaned page in an 1256 * anon then we need to look at the anon's ref count. 1257 * if it is greater than one then we are going to do 1258 * a normal copy-on-write fault into a new anon (this 1259 * is not a problem). however, if the reference count 1260 * is one (a case where we would normally allow a 1261 * write directly to the page) then we need to kill 1262 * the loan before we continue. 1263 */ 1264 1265 /* >1 case is already ok */ 1266 if (anon->an_ref == 1) { 1267 1268 /* get new un-owned replacement page */ 1269 pg = uvm_pagealloc(NULL, 0, NULL, 0); 1270 if (pg == NULL) { 1271 uvmfault_unlockall(&ufi, amap, uobj, 1272 anon); 1273 uvm_wait("flt_noram2"); 1274 goto ReFault; 1275 } 1276 1277 /* 1278 * copy data, kill loan, and drop uobj lock 1279 * (if any) 1280 */ 1281 /* copy old -> new */ 1282 uvm_pagecopy(anon->an_page, pg); 1283 1284 /* force reload */ 1285 pmap_page_protect(anon->an_page, VM_PROT_NONE); 1286 mutex_enter(&uvm_pageqlock); /* KILL loan */ 1287 1288 anon->an_page->uanon = NULL; 1289 /* in case we owned */ 1290 anon->an_page->pqflags &= ~PQ_ANON; 1291 1292 if (uobj) { 1293 /* if we were receiver of loan */ 1294 anon->an_page->loan_count--; 1295 } else { 1296 /* 1297 * we were the lender (A->K); need 1298 * to remove the page from pageq's. 1299 */ 1300 uvm_pagedequeue(anon->an_page); 1301 } 1302 1303 if (uobj) { 1304 mutex_exit(&uobj->vmobjlock); 1305 uobj = NULL; 1306 } 1307 1308 /* install new page in anon */ 1309 anon->an_page = pg; 1310 pg->uanon = anon; 1311 pg->pqflags |= PQ_ANON; 1312 1313 uvm_pageactivate(pg); 1314 mutex_exit(&uvm_pageqlock); 1315 1316 pg->flags &= ~(PG_BUSY|PG_FAKE); 1317 UVM_PAGE_OWN(pg, NULL); 1318 1319 /* done! */ 1320 } /* ref == 1 */ 1321 } /* write fault */ 1322 } /* loan count */ 1323 1324 /* 1325 * if we are case 1B then we will need to allocate a new blank 1326 * anon to transfer the data into. note that we have a lock 1327 * on anon, so no one can busy or release the page until we are done. 1328 * also note that the ref count can't drop to zero here because 1329 * it is > 1 and we are only dropping one ref. 1330 * 1331 * in the (hopefully very rare) case that we are out of RAM we 1332 * will unlock, wait for more RAM, and refault. 1333 * 1334 * if we are out of anon VM we kill the process (XXX: could wait?). 1335 */ 1336 1337 if (cow_now && anon->an_ref > 1) { 1338 1339 UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0); 1340 uvmexp.flt_acow++; 1341 oanon = anon; /* oanon = old, locked anon */ 1342 1343 error = uvmfault_promote(&ufi, oanon, PGO_DONTCARE, 1344 &anon, &anon_spare); 1345 switch (error) { 1346 case 0: 1347 break; 1348 case ERESTART: 1349 goto ReFault; 1350 default: 1351 goto done; 1352 } 1353 1354 pg = anon->an_page; 1355 mutex_enter(&uvm_pageqlock); 1356 uvm_pageactivate(pg); 1357 mutex_exit(&uvm_pageqlock); 1358 pg->flags &= ~(PG_BUSY|PG_FAKE); 1359 UVM_PAGE_OWN(pg, NULL); 1360 1361 /* deref: can not drop to zero here by defn! */ 1362 oanon->an_ref--; 1363 1364 /* 1365 * note: oanon is still locked, as is the new anon. we 1366 * need to check for this later when we unlock oanon; if 1367 * oanon != anon, we'll have to unlock anon, too. 1368 */ 1369 1370 } else { 1371 1372 uvmexp.flt_anon++; 1373 oanon = anon; /* old, locked anon is same as anon */ 1374 pg = anon->an_page; 1375 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */ 1376 enter_prot = enter_prot & ~VM_PROT_WRITE; 1377 1378 } 1379 1380 /* locked: maps(read), amap, oanon, anon (if different from oanon) */ 1381 KASSERT(mutex_owned(&amap->am_l)); 1382 KASSERT(mutex_owned(&anon->an_lock)); 1383 KASSERT(mutex_owned(&oanon->an_lock)); 1384 1385 /* 1386 * now map the page in. 1387 */ 1388 1389 UVMHIST_LOG(maphist, " MAPPING: anon: pm=0x%x, va=0x%x, pg=0x%x", 1390 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, 0); 1391 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1392 enter_prot, access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) 1393 != 0) { 1394 1395 /* 1396 * No need to undo what we did; we can simply think of 1397 * this as the pmap throwing away the mapping information. 1398 * 1399 * We do, however, have to go through the ReFault path, 1400 * as the map may change while we're asleep. 1401 */ 1402 1403 if (anon != oanon) 1404 mutex_exit(&anon->an_lock); 1405 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1406 if (!uvm_reclaimable()) { 1407 UVMHIST_LOG(maphist, 1408 "<- failed. out of VM",0,0,0,0); 1409 /* XXX instrumentation */ 1410 error = ENOMEM; 1411 goto done; 1412 } 1413 /* XXX instrumentation */ 1414 uvm_wait("flt_pmfail1"); 1415 goto ReFault; 1416 } 1417 1418 /* 1419 * ... update the page queues. 1420 */ 1421 1422 mutex_enter(&uvm_pageqlock); 1423 if (wire_fault) { 1424 uvm_pagewire(pg); 1425 1426 /* 1427 * since the now-wired page cannot be paged out, 1428 * release its swap resources for others to use. 1429 * since an anon with no swap cannot be PG_CLEAN, 1430 * clear its clean flag now. 1431 */ 1432 1433 pg->flags &= ~(PG_CLEAN); 1434 uvm_anon_dropswap(anon); 1435 } else { 1436 uvm_pageactivate(pg); 1437 } 1438 mutex_exit(&uvm_pageqlock); 1439 1440 /* 1441 * done case 1! finish up by unlocking everything and returning success 1442 */ 1443 1444 if (anon != oanon) 1445 mutex_exit(&anon->an_lock); 1446 uvmfault_unlockall(&ufi, amap, uobj, oanon); 1447 pmap_update(ufi.orig_map->pmap); 1448 error = 0; 1449 goto done; 1450 1451 Case2: 1452 /* 1453 * handle case 2: faulting on backing object or zero fill 1454 */ 1455 1456 /* 1457 * locked: 1458 * maps(read), amap(if there), uobj(if !null), uobjpage(if !null) 1459 */ 1460 KASSERT(amap == NULL || mutex_owned(&amap->am_l)); 1461 KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock)); 1462 KASSERT(uobjpage == NULL || (uobjpage->flags & PG_BUSY) != 0); 1463 1464 /* 1465 * note that uobjpage can not be PGO_DONTCARE at this point. we now 1466 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we 1467 * have a backing object, check and see if we are going to promote 1468 * the data up to an anon during the fault. 1469 */ 1470 1471 if (uobj == NULL) { 1472 uobjpage = PGO_DONTCARE; 1473 promote = true; /* always need anon here */ 1474 } else { 1475 KASSERT(uobjpage != PGO_DONTCARE); 1476 promote = cow_now && UVM_ET_ISCOPYONWRITE(ufi.entry); 1477 } 1478 UVMHIST_LOG(maphist, " case 2 fault: promote=%d, zfill=%d", 1479 promote, (uobj == NULL), 0,0); 1480 1481 /* 1482 * if uobjpage is not null then we do not need to do I/O to get the 1483 * uobjpage. 1484 * 1485 * if uobjpage is null, then we need to unlock and ask the pager to 1486 * get the data for us. once we have the data, we need to reverify 1487 * the state the world. we are currently not holding any resources. 1488 */ 1489 1490 if (uobjpage) { 1491 /* update rusage counters */ 1492 curlwp->l_ru.ru_minflt++; 1493 } else { 1494 /* update rusage counters */ 1495 curlwp->l_ru.ru_majflt++; 1496 1497 /* locked: maps(read), amap(if there), uobj */ 1498 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1499 /* locked: uobj */ 1500 1501 uvmexp.fltget++; 1502 gotpages = 1; 1503 uoff = (ufi.orig_rvaddr - ufi.entry->start) + ufi.entry->offset; 1504 error = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages, 1505 0, access_type & MASK(ufi.entry), ufi.entry->advice, 1506 PGO_SYNCIO); 1507 /* locked: uobjpage(if no error) */ 1508 KASSERT(error != 0 || (uobjpage->flags & PG_BUSY) != 0); 1509 1510 /* 1511 * recover from I/O 1512 */ 1513 1514 if (error) { 1515 if (error == EAGAIN) { 1516 UVMHIST_LOG(maphist, 1517 " pgo_get says TRY AGAIN!",0,0,0,0); 1518 tsleep(&lbolt, PVM, "fltagain2", 0); 1519 goto ReFault; 1520 } 1521 1522 UVMHIST_LOG(maphist, "<- pgo_get failed (code %d)", 1523 error, 0,0,0); 1524 goto done; 1525 } 1526 1527 /* locked: uobjpage */ 1528 1529 mutex_enter(&uvm_pageqlock); 1530 uvm_pageactivate(uobjpage); 1531 mutex_exit(&uvm_pageqlock); 1532 1533 /* 1534 * re-verify the state of the world by first trying to relock 1535 * the maps. always relock the object. 1536 */ 1537 1538 locked = uvmfault_relock(&ufi); 1539 if (locked && amap) 1540 amap_lock(amap); 1541 uobj = uobjpage->uobject; 1542 mutex_enter(&uobj->vmobjlock); 1543 1544 /* locked(locked): maps(read), amap(if !null), uobj, uobjpage */ 1545 /* locked(!locked): uobj, uobjpage */ 1546 1547 /* 1548 * verify that the page has not be released and re-verify 1549 * that amap slot is still free. if there is a problem, 1550 * we unlock and clean up. 1551 */ 1552 1553 if ((uobjpage->flags & PG_RELEASED) != 0 || 1554 (locked && amap && 1555 amap_lookup(&ufi.entry->aref, 1556 ufi.orig_rvaddr - ufi.entry->start))) { 1557 if (locked) 1558 uvmfault_unlockall(&ufi, amap, NULL, NULL); 1559 locked = false; 1560 } 1561 1562 /* 1563 * didn't get the lock? release the page and retry. 1564 */ 1565 1566 if (locked == false) { 1567 UVMHIST_LOG(maphist, 1568 " wasn't able to relock after fault: retry", 1569 0,0,0,0); 1570 if (uobjpage->flags & PG_WANTED) 1571 wakeup(uobjpage); 1572 if (uobjpage->flags & PG_RELEASED) { 1573 uvmexp.fltpgrele++; 1574 uvm_pagefree(uobjpage); 1575 goto ReFault; 1576 } 1577 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1578 UVM_PAGE_OWN(uobjpage, NULL); 1579 mutex_exit(&uobj->vmobjlock); 1580 goto ReFault; 1581 } 1582 1583 /* 1584 * we have the data in uobjpage which is busy and 1585 * not released. we are holding object lock (so the page 1586 * can't be released on us). 1587 */ 1588 1589 /* locked: maps(read), amap(if !null), uobj, uobjpage */ 1590 } 1591 1592 /* 1593 * locked: 1594 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj) 1595 */ 1596 KASSERT(amap == NULL || mutex_owned(&amap->am_l)); 1597 KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock)); 1598 KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0); 1599 1600 /* 1601 * notes: 1602 * - at this point uobjpage can not be NULL 1603 * - at this point uobjpage can not be PG_RELEASED (since we checked 1604 * for it above) 1605 * - at this point uobjpage could be PG_WANTED (handle later) 1606 */ 1607 1608 KASSERT(uobj == NULL || uobj == uobjpage->uobject); 1609 KASSERT(uobj == NULL || !UVM_OBJ_IS_CLEAN(uobjpage->uobject) || 1610 (uobjpage->flags & PG_CLEAN) != 0); 1611 if (promote == false) { 1612 1613 /* 1614 * we are not promoting. if the mapping is COW ensure that we 1615 * don't give more access than we should (e.g. when doing a read 1616 * fault on a COPYONWRITE mapping we want to map the COW page in 1617 * R/O even though the entry protection could be R/W). 1618 * 1619 * set "pg" to the page we want to map in (uobjpage, usually) 1620 */ 1621 1622 /* no anon in this case. */ 1623 anon = NULL; 1624 1625 uvmexp.flt_obj++; 1626 if (UVM_ET_ISCOPYONWRITE(ufi.entry) || 1627 UVM_OBJ_NEEDS_WRITEFAULT(uobjpage->uobject)) 1628 enter_prot &= ~VM_PROT_WRITE; 1629 pg = uobjpage; /* map in the actual object */ 1630 1631 KASSERT(uobjpage != PGO_DONTCARE); 1632 1633 /* 1634 * we are faulting directly on the page. be careful 1635 * about writing to loaned pages... 1636 */ 1637 1638 if (uobjpage->loan_count) { 1639 if (!cow_now) { 1640 /* read fault: cap the protection at readonly */ 1641 /* cap! */ 1642 enter_prot = enter_prot & ~VM_PROT_WRITE; 1643 } else { 1644 /* write fault: must break the loan here */ 1645 1646 pg = uvm_loanbreak(uobjpage); 1647 if (pg == NULL) { 1648 1649 /* 1650 * drop ownership of page, it can't 1651 * be released 1652 */ 1653 1654 if (uobjpage->flags & PG_WANTED) 1655 wakeup(uobjpage); 1656 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1657 UVM_PAGE_OWN(uobjpage, NULL); 1658 1659 uvmfault_unlockall(&ufi, amap, uobj, 1660 NULL); 1661 UVMHIST_LOG(maphist, 1662 " out of RAM breaking loan, waiting", 1663 0,0,0,0); 1664 uvmexp.fltnoram++; 1665 uvm_wait("flt_noram4"); 1666 goto ReFault; 1667 } 1668 uobjpage = pg; 1669 } 1670 } 1671 } else { 1672 1673 /* 1674 * if we are going to promote the data to an anon we 1675 * allocate a blank anon here and plug it into our amap. 1676 */ 1677 #if DIAGNOSTIC 1678 if (amap == NULL) 1679 panic("uvm_fault: want to promote data, but no anon"); 1680 #endif 1681 error = uvmfault_promote(&ufi, NULL, uobjpage, 1682 &anon, &anon_spare); 1683 switch (error) { 1684 case 0: 1685 break; 1686 case ERESTART: 1687 goto ReFault; 1688 default: 1689 goto done; 1690 } 1691 1692 pg = anon->an_page; 1693 1694 /* 1695 * fill in the data 1696 */ 1697 1698 if (uobjpage != PGO_DONTCARE) { 1699 uvmexp.flt_prcopy++; 1700 1701 /* 1702 * promote to shared amap? make sure all sharing 1703 * procs see it 1704 */ 1705 1706 if ((amap_flags(amap) & AMAP_SHARED) != 0) { 1707 pmap_page_protect(uobjpage, VM_PROT_NONE); 1708 /* 1709 * XXX: PAGE MIGHT BE WIRED! 1710 */ 1711 } 1712 1713 /* 1714 * dispose of uobjpage. it can't be PG_RELEASED 1715 * since we still hold the object lock. 1716 * drop handle to uobj as well. 1717 */ 1718 1719 if (uobjpage->flags & PG_WANTED) 1720 /* still have the obj lock */ 1721 wakeup(uobjpage); 1722 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 1723 UVM_PAGE_OWN(uobjpage, NULL); 1724 mutex_exit(&uobj->vmobjlock); 1725 uobj = NULL; 1726 1727 UVMHIST_LOG(maphist, 1728 " promote uobjpage 0x%x to anon/page 0x%x/0x%x", 1729 uobjpage, anon, pg, 0); 1730 1731 } else { 1732 uvmexp.flt_przero++; 1733 1734 /* 1735 * Page is zero'd and marked dirty by 1736 * uvmfault_promote(). 1737 */ 1738 1739 UVMHIST_LOG(maphist," zero fill anon/page 0x%x/0%x", 1740 anon, pg, 0, 0); 1741 } 1742 } 1743 1744 /* 1745 * locked: 1746 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj), 1747 * anon(if !null), pg(if anon) 1748 * 1749 * note: pg is either the uobjpage or the new page in the new anon 1750 */ 1751 KASSERT(amap == NULL || mutex_owned(&amap->am_l)); 1752 KASSERT(uobj == NULL || mutex_owned(&uobj->vmobjlock)); 1753 KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0); 1754 KASSERT(anon == NULL || mutex_owned(&anon->an_lock)); 1755 KASSERT((pg->flags & PG_BUSY) != 0); 1756 1757 /* 1758 * all resources are present. we can now map it in and free our 1759 * resources. 1760 */ 1761 1762 UVMHIST_LOG(maphist, 1763 " MAPPING: case2: pm=0x%x, va=0x%x, pg=0x%x, promote=%d", 1764 ufi.orig_map->pmap, ufi.orig_rvaddr, pg, promote); 1765 KASSERT((access_type & VM_PROT_WRITE) == 0 || 1766 (pg->flags & PG_RDONLY) == 0); 1767 if (pmap_enter(ufi.orig_map->pmap, ufi.orig_rvaddr, VM_PAGE_TO_PHYS(pg), 1768 pg->flags & PG_RDONLY ? enter_prot & ~VM_PROT_WRITE : enter_prot, 1769 access_type | PMAP_CANFAIL | (wired ? PMAP_WIRED : 0)) != 0) { 1770 1771 /* 1772 * No need to undo what we did; we can simply think of 1773 * this as the pmap throwing away the mapping information. 1774 * 1775 * We do, however, have to go through the ReFault path, 1776 * as the map may change while we're asleep. 1777 */ 1778 1779 if (pg->flags & PG_WANTED) 1780 wakeup(pg); 1781 1782 /* 1783 * note that pg can't be PG_RELEASED since we did not drop 1784 * the object lock since the last time we checked. 1785 */ 1786 KASSERT((pg->flags & PG_RELEASED) == 0); 1787 1788 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); 1789 UVM_PAGE_OWN(pg, NULL); 1790 uvmfault_unlockall(&ufi, amap, uobj, anon); 1791 if (!uvm_reclaimable()) { 1792 UVMHIST_LOG(maphist, 1793 "<- failed. out of VM",0,0,0,0); 1794 /* XXX instrumentation */ 1795 error = ENOMEM; 1796 goto done; 1797 } 1798 /* XXX instrumentation */ 1799 uvm_wait("flt_pmfail2"); 1800 goto ReFault; 1801 } 1802 1803 mutex_enter(&uvm_pageqlock); 1804 if (wire_fault) { 1805 uvm_pagewire(pg); 1806 if (pg->pqflags & PQ_AOBJ) { 1807 1808 /* 1809 * since the now-wired page cannot be paged out, 1810 * release its swap resources for others to use. 1811 * since an aobj page with no swap cannot be PG_CLEAN, 1812 * clear its clean flag now. 1813 */ 1814 1815 KASSERT(uobj != NULL); 1816 pg->flags &= ~(PG_CLEAN); 1817 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT); 1818 } 1819 } else { 1820 uvm_pageactivate(pg); 1821 } 1822 mutex_exit(&uvm_pageqlock); 1823 if (pg->flags & PG_WANTED) 1824 wakeup(pg); 1825 1826 /* 1827 * note that pg can't be PG_RELEASED since we did not drop the object 1828 * lock since the last time we checked. 1829 */ 1830 KASSERT((pg->flags & PG_RELEASED) == 0); 1831 1832 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); 1833 UVM_PAGE_OWN(pg, NULL); 1834 uvmfault_unlockall(&ufi, amap, uobj, anon); 1835 pmap_update(ufi.orig_map->pmap); 1836 UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0); 1837 error = 0; 1838 done: 1839 if (anon_spare != NULL) { 1840 anon_spare->an_ref--; 1841 uvm_anfree(anon_spare); 1842 } 1843 return error; 1844 } 1845 1846 1847 /* 1848 * uvm_fault_wire: wire down a range of virtual addresses in a map. 1849 * 1850 * => map may be read-locked by caller, but MUST NOT be write-locked. 1851 * => if map is read-locked, any operations which may cause map to 1852 * be write-locked in uvm_fault() must be taken care of by 1853 * the caller. See uvm_map_pageable(). 1854 */ 1855 1856 int 1857 uvm_fault_wire(struct vm_map *map, vaddr_t start, vaddr_t end, 1858 vm_prot_t access_type, int wiremax) 1859 { 1860 vaddr_t va; 1861 int error; 1862 1863 /* 1864 * now fault it in a page at a time. if the fault fails then we have 1865 * to undo what we have done. note that in uvm_fault VM_PROT_NONE 1866 * is replaced with the max protection if fault_type is VM_FAULT_WIRE. 1867 */ 1868 1869 /* 1870 * XXX work around overflowing a vaddr_t. this prevents us from 1871 * wiring the last page in the address space, though. 1872 */ 1873 if (start > end) { 1874 return EFAULT; 1875 } 1876 1877 for (va = start ; va < end ; va += PAGE_SIZE) { 1878 error = uvm_fault_internal(map, va, access_type, 1879 wiremax ? UVM_FAULT_WIREMAX : UVM_FAULT_WIRE); 1880 if (error) { 1881 if (va != start) { 1882 uvm_fault_unwire(map, start, va); 1883 } 1884 return error; 1885 } 1886 } 1887 return 0; 1888 } 1889 1890 /* 1891 * uvm_fault_unwire(): unwire range of virtual space. 1892 */ 1893 1894 void 1895 uvm_fault_unwire(struct vm_map *map, vaddr_t start, vaddr_t end) 1896 { 1897 vm_map_lock_read(map); 1898 uvm_fault_unwire_locked(map, start, end); 1899 vm_map_unlock_read(map); 1900 } 1901 1902 /* 1903 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire(). 1904 * 1905 * => map must be at least read-locked. 1906 */ 1907 1908 void 1909 uvm_fault_unwire_locked(struct vm_map *map, vaddr_t start, vaddr_t end) 1910 { 1911 struct vm_map_entry *entry; 1912 pmap_t pmap = vm_map_pmap(map); 1913 vaddr_t va; 1914 paddr_t pa; 1915 struct vm_page *pg; 1916 1917 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0); 1918 1919 /* 1920 * we assume that the area we are unwiring has actually been wired 1921 * in the first place. this means that we should be able to extract 1922 * the PAs from the pmap. we also lock out the page daemon so that 1923 * we can call uvm_pageunwire. 1924 */ 1925 1926 mutex_enter(&uvm_pageqlock); 1927 1928 /* 1929 * find the beginning map entry for the region. 1930 */ 1931 1932 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map)); 1933 if (uvm_map_lookup_entry(map, start, &entry) == false) 1934 panic("uvm_fault_unwire_locked: address not in map"); 1935 1936 for (va = start; va < end; va += PAGE_SIZE) { 1937 if (pmap_extract(pmap, va, &pa) == false) 1938 continue; 1939 1940 /* 1941 * find the map entry for the current address. 1942 */ 1943 1944 KASSERT(va >= entry->start); 1945 while (va >= entry->end) { 1946 KASSERT(entry->next != &map->header && 1947 entry->next->start <= entry->end); 1948 entry = entry->next; 1949 } 1950 1951 /* 1952 * if the entry is no longer wired, tell the pmap. 1953 */ 1954 1955 if (VM_MAPENT_ISWIRED(entry) == 0) 1956 pmap_unwire(pmap, va); 1957 1958 pg = PHYS_TO_VM_PAGE(pa); 1959 if (pg) 1960 uvm_pageunwire(pg); 1961 } 1962 1963 mutex_exit(&uvm_pageqlock); 1964 } 1965