1 /* $NetBSD: uvm_fault.c,v 1.186 2011/06/12 03:36:02 rmind Exp $ */ 2 3 /* 4 * Copyright (c) 1997 Charles D. Cranor and Washington University. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 * 27 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp 28 */ 29 30 /* 31 * uvm_fault.c: fault handler 32 */ 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.186 2011/06/12 03:36:02 rmind Exp $"); 36 37 #include "opt_uvmhist.h" 38 39 #include <sys/param.h> 40 #include <sys/systm.h> 41 #include <sys/kernel.h> 42 #include <sys/mman.h> 43 44 #include <uvm/uvm.h> 45 46 /* 47 * 48 * a word on page faults: 49 * 50 * types of page faults we handle: 51 * 52 * CASE 1: upper layer faults CASE 2: lower layer faults 53 * 54 * CASE 1A CASE 1B CASE 2A CASE 2B 55 * read/write1 write>1 read/write +-cow_write/zero 56 * | | | | 57 * +--|--+ +--|--+ +-----+ + | + | +-----+ 58 * amap | V | | ---------> new | | | | ^ | 59 * +-----+ +-----+ +-----+ + | + | +--|--+ 60 * | | | 61 * +-----+ +-----+ +--|--+ | +--|--+ 62 * uobj | d/c | | d/c | | V | +----+ | 63 * +-----+ +-----+ +-----+ +-----+ 64 * 65 * d/c = don't care 66 * 67 * case [0]: layerless fault 68 * no amap or uobj is present. this is an error. 69 * 70 * case [1]: upper layer fault [anon active] 71 * 1A: [read] or [write with anon->an_ref == 1] 72 * I/O takes place in upper level anon and uobj is not touched. 73 * 1B: [write with anon->an_ref > 1] 74 * new anon is alloc'd and data is copied off ["COW"] 75 * 76 * case [2]: lower layer fault [uobj] 77 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area] 78 * I/O takes place directly in object. 79 * 2B: [write to copy_on_write] or [read on NULL uobj] 80 * data is "promoted" from uobj to a new anon. 81 * if uobj is null, then we zero fill. 82 * 83 * we follow the standard UVM locking protocol ordering: 84 * 85 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ) 86 * we hold a PG_BUSY page if we unlock for I/O 87 * 88 * 89 * the code is structured as follows: 90 * 91 * - init the "IN" params in the ufi structure 92 * ReFault: (ERESTART returned to the loop in uvm_fault_internal) 93 * - do lookups [locks maps], check protection, handle needs_copy 94 * - check for case 0 fault (error) 95 * - establish "range" of fault 96 * - if we have an amap lock it and extract the anons 97 * - if sequential advice deactivate pages behind us 98 * - at the same time check pmap for unmapped areas and anon for pages 99 * that we could map in (and do map it if found) 100 * - check object for resident pages that we could map in 101 * - if (case 2) goto Case2 102 * - >>> handle case 1 103 * - ensure source anon is resident in RAM 104 * - if case 1B alloc new anon and copy from source 105 * - map the correct page in 106 * Case2: 107 * - >>> handle case 2 108 * - ensure source page is resident (if uobj) 109 * - if case 2B alloc new anon and copy from source (could be zero 110 * fill if uobj == NULL) 111 * - map the correct page in 112 * - done! 113 * 114 * note on paging: 115 * if we have to do I/O we place a PG_BUSY page in the correct object, 116 * unlock everything, and do the I/O. when I/O is done we must reverify 117 * the state of the world before assuming that our data structures are 118 * valid. [because mappings could change while the map is unlocked] 119 * 120 * alternative 1: unbusy the page in question and restart the page fault 121 * from the top (ReFault). this is easy but does not take advantage 122 * of the information that we already have from our previous lookup, 123 * although it is possible that the "hints" in the vm_map will help here. 124 * 125 * alternative 2: the system already keeps track of a "version" number of 126 * a map. [i.e. every time you write-lock a map (e.g. to change a 127 * mapping) you bump the version number up by one...] so, we can save 128 * the version number of the map before we release the lock and start I/O. 129 * then when I/O is done we can relock and check the version numbers 130 * to see if anything changed. this might save us some over 1 because 131 * we don't have to unbusy the page and may be less compares(?). 132 * 133 * alternative 3: put in backpointers or a way to "hold" part of a map 134 * in place while I/O is in progress. this could be complex to 135 * implement (especially with structures like amap that can be referenced 136 * by multiple map entries, and figuring out what should wait could be 137 * complex as well...). 138 * 139 * we use alternative 2. given that we are multi-threaded now we may want 140 * to reconsider the choice. 141 */ 142 143 /* 144 * local data structures 145 */ 146 147 struct uvm_advice { 148 int advice; 149 int nback; 150 int nforw; 151 }; 152 153 /* 154 * page range array: 155 * note: index in array must match "advice" value 156 * XXX: borrowed numbers from freebsd. do they work well for us? 157 */ 158 159 static const struct uvm_advice uvmadvice[] = { 160 { UVM_ADV_NORMAL, 3, 4 }, 161 { UVM_ADV_RANDOM, 0, 0 }, 162 { UVM_ADV_SEQUENTIAL, 8, 7}, 163 }; 164 165 #define UVM_MAXRANGE 16 /* must be MAX() of nback+nforw+1 */ 166 167 /* 168 * private prototypes 169 */ 170 171 /* 172 * inline functions 173 */ 174 175 /* 176 * uvmfault_anonflush: try and deactivate pages in specified anons 177 * 178 * => does not have to deactivate page if it is busy 179 */ 180 181 static inline void 182 uvmfault_anonflush(struct vm_anon **anons, int n) 183 { 184 int lcv; 185 struct vm_page *pg; 186 187 for (lcv = 0; lcv < n; lcv++) { 188 if (anons[lcv] == NULL) 189 continue; 190 KASSERT(mutex_owned(anons[lcv]->an_lock)); 191 pg = anons[lcv]->an_page; 192 if (pg && (pg->flags & PG_BUSY) == 0) { 193 mutex_enter(&uvm_pageqlock); 194 if (pg->wire_count == 0) { 195 uvm_pagedeactivate(pg); 196 } 197 mutex_exit(&uvm_pageqlock); 198 } 199 } 200 } 201 202 /* 203 * normal functions 204 */ 205 206 /* 207 * uvmfault_amapcopy: clear "needs_copy" in a map. 208 * 209 * => called with VM data structures unlocked (usually, see below) 210 * => we get a write lock on the maps and clear needs_copy for a VA 211 * => if we are out of RAM we sleep (waiting for more) 212 */ 213 214 static void 215 uvmfault_amapcopy(struct uvm_faultinfo *ufi) 216 { 217 for (;;) { 218 219 /* 220 * no mapping? give up. 221 */ 222 223 if (uvmfault_lookup(ufi, true) == false) 224 return; 225 226 /* 227 * copy if needed. 228 */ 229 230 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) 231 amap_copy(ufi->map, ufi->entry, AMAP_COPY_NOWAIT, 232 ufi->orig_rvaddr, ufi->orig_rvaddr + 1); 233 234 /* 235 * didn't work? must be out of RAM. unlock and sleep. 236 */ 237 238 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 239 uvmfault_unlockmaps(ufi, true); 240 uvm_wait("fltamapcopy"); 241 continue; 242 } 243 244 /* 245 * got it! unlock and return. 246 */ 247 248 uvmfault_unlockmaps(ufi, true); 249 return; 250 } 251 /*NOTREACHED*/ 252 } 253 254 /* 255 * uvmfault_anonget: get data in an anon into a non-busy, non-released 256 * page in that anon. 257 * 258 * => maps, amap, and anon locked by caller. 259 * => if we fail (result != 0) we unlock everything. 260 * => if we are successful, we return with everything still locked. 261 * => we don't move the page on the queues [gets moved later] 262 * => if we allocate a new page [we_own], it gets put on the queues. 263 * either way, the result is that the page is on the queues at return time 264 * => for pages which are on loan from a uvm_object (and thus are not 265 * owned by the anon): if successful, we return with the owning object 266 * locked. the caller must unlock this object when it unlocks everything 267 * else. 268 */ 269 270 int 271 uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap, 272 struct vm_anon *anon) 273 { 274 bool we_own; /* we own anon's page? */ 275 bool locked; /* did we relock? */ 276 struct vm_page *pg; 277 int error; 278 UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist); 279 280 KASSERT(mutex_owned(anon->an_lock)); 281 KASSERT(amap == NULL || anon->an_lock == amap->am_lock); 282 283 error = 0; 284 uvmexp.fltanget++; 285 /* bump rusage counters */ 286 if (anon->an_page) 287 curlwp->l_ru.ru_minflt++; 288 else 289 curlwp->l_ru.ru_majflt++; 290 291 /* 292 * loop until we get it, or fail. 293 */ 294 295 for (;;) { 296 we_own = false; /* true if we set PG_BUSY on a page */ 297 pg = anon->an_page; 298 299 /* 300 * if there is a resident page and it is loaned, then anon 301 * may not own it. call out to uvm_anon_lockpage() to ensure 302 * the real owner of the page has been identified and locked. 303 */ 304 305 if (pg && pg->loan_count) 306 pg = uvm_anon_lockloanpg(anon); 307 308 /* 309 * page there? make sure it is not busy/released. 310 */ 311 312 if (pg) { 313 314 /* 315 * at this point, if the page has a uobject [meaning 316 * we have it on loan], then that uobject is locked 317 * by us! if the page is busy, we drop all the 318 * locks (including uobject) and try again. 319 */ 320 321 if ((pg->flags & PG_BUSY) == 0) { 322 UVMHIST_LOG(maphist, "<- OK",0,0,0,0); 323 return (0); 324 } 325 pg->flags |= PG_WANTED; 326 uvmexp.fltpgwait++; 327 328 /* 329 * the last unlock must be an atomic unlock+wait on 330 * the owner of page 331 */ 332 333 if (pg->uobject) { /* owner is uobject ? */ 334 uvmfault_unlockall(ufi, amap, NULL); 335 UVMHIST_LOG(maphist, " unlock+wait on uobj",0, 336 0,0,0); 337 UVM_UNLOCK_AND_WAIT(pg, 338 pg->uobject->vmobjlock, 339 false, "anonget1",0); 340 } else { 341 /* anon owns page */ 342 uvmfault_unlockall(ufi, NULL, NULL); 343 UVMHIST_LOG(maphist, " unlock+wait on anon",0, 344 0,0,0); 345 UVM_UNLOCK_AND_WAIT(pg, anon->an_lock, 0, 346 "anonget2",0); 347 } 348 } else { 349 #if defined(VMSWAP) 350 351 /* 352 * no page, we must try and bring it in. 353 */ 354 355 pg = uvm_pagealloc(NULL, 356 ufi != NULL ? ufi->orig_rvaddr : 0, 357 anon, ufi != NULL ? UVM_FLAG_COLORMATCH : 0); 358 if (pg == NULL) { /* out of RAM. */ 359 uvmfault_unlockall(ufi, amap, NULL); 360 uvmexp.fltnoram++; 361 UVMHIST_LOG(maphist, " noram -- UVM_WAIT",0, 362 0,0,0); 363 if (!uvm_reclaimable()) { 364 return ENOMEM; 365 } 366 uvm_wait("flt_noram1"); 367 } else { 368 /* we set the PG_BUSY bit */ 369 we_own = true; 370 uvmfault_unlockall(ufi, amap, NULL); 371 372 /* 373 * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN 374 * page into the uvm_swap_get function with 375 * all data structures unlocked. note that 376 * it is ok to read an_swslot here because 377 * we hold PG_BUSY on the page. 378 */ 379 uvmexp.pageins++; 380 error = uvm_swap_get(pg, anon->an_swslot, 381 PGO_SYNCIO); 382 383 /* 384 * we clean up after the i/o below in the 385 * "we_own" case 386 */ 387 } 388 #else /* defined(VMSWAP) */ 389 panic("%s: no page", __func__); 390 #endif /* defined(VMSWAP) */ 391 } 392 393 /* 394 * now relock and try again 395 */ 396 397 locked = uvmfault_relock(ufi); 398 if (locked || we_own) { 399 mutex_enter(anon->an_lock); 400 } 401 402 /* 403 * if we own the page (i.e. we set PG_BUSY), then we need 404 * to clean up after the I/O. there are three cases to 405 * consider: 406 * [1] page released during I/O: free anon and ReFault. 407 * [2] I/O not OK. free the page and cause the fault 408 * to fail. 409 * [3] I/O OK! activate the page and sync with the 410 * non-we_own case (i.e. drop anon lock if not locked). 411 */ 412 413 if (we_own) { 414 #if defined(VMSWAP) 415 if (pg->flags & PG_WANTED) { 416 wakeup(pg); 417 } 418 if (error) { 419 420 /* 421 * remove the swap slot from the anon 422 * and mark the anon as having no real slot. 423 * don't free the swap slot, thus preventing 424 * it from being used again. 425 */ 426 427 if (anon->an_swslot > 0) 428 uvm_swap_markbad(anon->an_swslot, 1); 429 anon->an_swslot = SWSLOT_BAD; 430 431 if ((pg->flags & PG_RELEASED) != 0) 432 goto released; 433 434 /* 435 * note: page was never !PG_BUSY, so it 436 * can't be mapped and thus no need to 437 * pmap_page_protect it... 438 */ 439 440 mutex_enter(&uvm_pageqlock); 441 uvm_pagefree(pg); 442 mutex_exit(&uvm_pageqlock); 443 444 if (locked) 445 uvmfault_unlockall(ufi, NULL, NULL); 446 mutex_exit(anon->an_lock); 447 UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0); 448 return error; 449 } 450 451 if ((pg->flags & PG_RELEASED) != 0) { 452 released: 453 KASSERT(anon->an_ref == 0); 454 455 /* 456 * released while we unlocked amap. 457 */ 458 459 if (locked) 460 uvmfault_unlockall(ufi, NULL, NULL); 461 462 uvm_anon_release(anon); 463 464 if (error) { 465 UVMHIST_LOG(maphist, 466 "<- ERROR/RELEASED", 0,0,0,0); 467 return error; 468 } 469 470 UVMHIST_LOG(maphist, "<- RELEASED", 0,0,0,0); 471 return ERESTART; 472 } 473 474 /* 475 * we've successfully read the page, activate it. 476 */ 477 478 mutex_enter(&uvm_pageqlock); 479 uvm_pageactivate(pg); 480 mutex_exit(&uvm_pageqlock); 481 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE); 482 UVM_PAGE_OWN(pg, NULL); 483 #else /* defined(VMSWAP) */ 484 panic("%s: we_own", __func__); 485 #endif /* defined(VMSWAP) */ 486 } 487 488 /* 489 * we were not able to relock. restart fault. 490 */ 491 492 if (!locked) { 493 if (we_own) { 494 mutex_exit(anon->an_lock); 495 } 496 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 497 return (ERESTART); 498 } 499 500 /* 501 * verify no one has touched the amap and moved the anon on us. 502 */ 503 504 if (ufi != NULL && amap_lookup(&ufi->entry->aref, 505 ufi->orig_rvaddr - ufi->entry->start) != anon) { 506 507 uvmfault_unlockall(ufi, amap, NULL); 508 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 509 return (ERESTART); 510 } 511 512 /* 513 * try it again! 514 */ 515 516 uvmexp.fltanretry++; 517 continue; 518 } 519 /*NOTREACHED*/ 520 } 521 522 /* 523 * uvmfault_promote: promote data to a new anon. used for 1B and 2B. 524 * 525 * 1. allocate an anon and a page. 526 * 2. fill its contents. 527 * 3. put it into amap. 528 * 529 * => if we fail (result != 0) we unlock everything. 530 * => on success, return a new locked anon via 'nanon'. 531 * (*nanon)->an_page will be a resident, locked, dirty page. 532 * => it's caller's responsibility to put the promoted nanon->an_page to the 533 * page queue. 534 */ 535 536 static int 537 uvmfault_promote(struct uvm_faultinfo *ufi, 538 struct vm_anon *oanon, 539 struct vm_page *uobjpage, 540 struct vm_anon **nanon, /* OUT: allocated anon */ 541 struct vm_anon **spare) 542 { 543 struct vm_amap *amap = ufi->entry->aref.ar_amap; 544 struct uvm_object *uobj; 545 struct vm_anon *anon; 546 struct vm_page *pg; 547 struct vm_page *opg; 548 int error; 549 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 550 551 if (oanon) { 552 /* anon COW */ 553 opg = oanon->an_page; 554 KASSERT(opg != NULL); 555 KASSERT(opg->uobject == NULL || opg->loan_count > 0); 556 } else if (uobjpage != PGO_DONTCARE) { 557 /* object-backed COW */ 558 opg = uobjpage; 559 } else { 560 /* ZFOD */ 561 opg = NULL; 562 } 563 if (opg != NULL) { 564 uobj = opg->uobject; 565 } else { 566 uobj = NULL; 567 } 568 569 KASSERT(amap != NULL); 570 KASSERT(uobjpage != NULL); 571 KASSERT(uobjpage == PGO_DONTCARE || (uobjpage->flags & PG_BUSY) != 0); 572 KASSERT(mutex_owned(amap->am_lock)); 573 KASSERT(oanon == NULL || amap->am_lock == oanon->an_lock); 574 KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock)); 575 576 if (*spare != NULL) { 577 anon = *spare; 578 *spare = NULL; 579 } else if (ufi->map != kernel_map) { 580 anon = uvm_analloc(); 581 } else { 582 UVMHIST_LOG(maphist, "kernel_map, unlock and retry", 0,0,0,0); 583 584 /* 585 * we can't allocate anons with kernel_map locked. 586 */ 587 588 uvm_page_unbusy(&uobjpage, 1); 589 uvmfault_unlockall(ufi, amap, uobj); 590 591 *spare = uvm_analloc(); 592 if (*spare == NULL) { 593 goto nomem; 594 } 595 KASSERT((*spare)->an_lock == NULL); 596 error = ERESTART; 597 goto done; 598 } 599 if (anon) { 600 601 /* 602 * The new anon is locked. 603 * 604 * if opg == NULL, we want a zero'd, dirty page, 605 * so have uvm_pagealloc() do that for us. 606 */ 607 608 KASSERT(anon->an_lock == NULL); 609 anon->an_lock = amap->am_lock; 610 mutex_obj_hold(anon->an_lock); 611 pg = uvm_pagealloc(NULL, ufi->orig_rvaddr, anon, 612 UVM_FLAG_COLORMATCH | (opg == NULL ? UVM_PGA_ZERO : 0)); 613 if (pg == NULL) { 614 mutex_obj_free(anon->an_lock); 615 anon->an_lock = NULL; 616 } 617 } else { 618 pg = NULL; 619 } 620 621 /* 622 * out of memory resources? 623 */ 624 625 if (pg == NULL) { 626 /* save anon for the next try. */ 627 if (anon != NULL) { 628 *spare = anon; 629 } 630 631 /* unlock and fail ... */ 632 uvm_page_unbusy(&uobjpage, 1); 633 uvmfault_unlockall(ufi, amap, uobj); 634 nomem: 635 if (!uvm_reclaimable()) { 636 UVMHIST_LOG(maphist, "out of VM", 0,0,0,0); 637 uvmexp.fltnoanon++; 638 error = ENOMEM; 639 goto done; 640 } 641 642 UVMHIST_LOG(maphist, "out of RAM, waiting for more", 0,0,0,0); 643 uvmexp.fltnoram++; 644 uvm_wait("flt_noram5"); 645 error = ERESTART; 646 goto done; 647 } 648 649 /* copy page [pg now dirty] */ 650 if (opg) { 651 uvm_pagecopy(opg, pg); 652 } 653 654 amap_add(&ufi->entry->aref, ufi->orig_rvaddr - ufi->entry->start, anon, 655 oanon != NULL); 656 657 *nanon = anon; 658 error = 0; 659 done: 660 return error; 661 } 662 663 664 /* 665 * F A U L T - m a i n e n t r y p o i n t 666 */ 667 668 /* 669 * uvm_fault: page fault handler 670 * 671 * => called from MD code to resolve a page fault 672 * => VM data structures usually should be unlocked. however, it is 673 * possible to call here with the main map locked if the caller 674 * gets a write lock, sets it recusive, and then calls us (c.f. 675 * uvm_map_pageable). this should be avoided because it keeps 676 * the map locked off during I/O. 677 * => MUST NEVER BE CALLED IN INTERRUPT CONTEXT 678 */ 679 680 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 681 ~VM_PROT_WRITE : VM_PROT_ALL) 682 683 /* fault_flag values passed from uvm_fault_wire to uvm_fault_internal */ 684 #define UVM_FAULT_WIRE (1 << 0) 685 #define UVM_FAULT_MAXPROT (1 << 1) 686 687 struct uvm_faultctx { 688 vm_prot_t access_type; 689 vm_prot_t enter_prot; 690 vaddr_t startva; 691 int npages; 692 int centeridx; 693 struct vm_anon *anon_spare; 694 bool wire_mapping; 695 bool narrow; 696 bool wire_paging; 697 bool cow_now; 698 bool promote; 699 }; 700 701 static inline int uvm_fault_check( 702 struct uvm_faultinfo *, struct uvm_faultctx *, 703 struct vm_anon ***, bool); 704 705 static int uvm_fault_upper( 706 struct uvm_faultinfo *, struct uvm_faultctx *, 707 struct vm_anon **); 708 static inline int uvm_fault_upper_lookup( 709 struct uvm_faultinfo *, const struct uvm_faultctx *, 710 struct vm_anon **, struct vm_page **); 711 static inline void uvm_fault_upper_neighbor( 712 struct uvm_faultinfo *, const struct uvm_faultctx *, 713 vaddr_t, struct vm_page *, bool); 714 static inline int uvm_fault_upper_loan( 715 struct uvm_faultinfo *, struct uvm_faultctx *, 716 struct vm_anon *, struct uvm_object **); 717 static inline int uvm_fault_upper_promote( 718 struct uvm_faultinfo *, struct uvm_faultctx *, 719 struct uvm_object *, struct vm_anon *); 720 static inline int uvm_fault_upper_direct( 721 struct uvm_faultinfo *, struct uvm_faultctx *, 722 struct uvm_object *, struct vm_anon *); 723 static int uvm_fault_upper_enter( 724 struct uvm_faultinfo *, const struct uvm_faultctx *, 725 struct uvm_object *, struct vm_anon *, 726 struct vm_page *, struct vm_anon *); 727 static inline void uvm_fault_upper_done( 728 struct uvm_faultinfo *, const struct uvm_faultctx *, 729 struct vm_anon *, struct vm_page *); 730 731 static int uvm_fault_lower( 732 struct uvm_faultinfo *, struct uvm_faultctx *, 733 struct vm_page **); 734 static inline void uvm_fault_lower_lookup( 735 struct uvm_faultinfo *, const struct uvm_faultctx *, 736 struct vm_page **); 737 static inline void uvm_fault_lower_neighbor( 738 struct uvm_faultinfo *, const struct uvm_faultctx *, 739 vaddr_t, struct vm_page *, bool); 740 static inline int uvm_fault_lower_io( 741 struct uvm_faultinfo *, const struct uvm_faultctx *, 742 struct uvm_object **, struct vm_page **); 743 static inline int uvm_fault_lower_direct( 744 struct uvm_faultinfo *, struct uvm_faultctx *, 745 struct uvm_object *, struct vm_page *); 746 static inline int uvm_fault_lower_direct_loan( 747 struct uvm_faultinfo *, struct uvm_faultctx *, 748 struct uvm_object *, struct vm_page **, 749 struct vm_page **); 750 static inline int uvm_fault_lower_promote( 751 struct uvm_faultinfo *, struct uvm_faultctx *, 752 struct uvm_object *, struct vm_page *); 753 static int uvm_fault_lower_enter( 754 struct uvm_faultinfo *, const struct uvm_faultctx *, 755 struct uvm_object *, 756 struct vm_anon *, struct vm_page *); 757 static inline void uvm_fault_lower_done( 758 struct uvm_faultinfo *, const struct uvm_faultctx *, 759 struct uvm_object *, struct vm_page *); 760 761 int 762 uvm_fault_internal(struct vm_map *orig_map, vaddr_t vaddr, 763 vm_prot_t access_type, int fault_flag) 764 { 765 struct uvm_faultinfo ufi; 766 struct uvm_faultctx flt = { 767 .access_type = access_type, 768 769 /* don't look for neighborhood * pages on "wire" fault */ 770 .narrow = (fault_flag & UVM_FAULT_WIRE) != 0, 771 772 /* "wire" fault causes wiring of both mapping and paging */ 773 .wire_mapping = (fault_flag & UVM_FAULT_WIRE) != 0, 774 .wire_paging = (fault_flag & UVM_FAULT_WIRE) != 0, 775 }; 776 const bool maxprot = (fault_flag & UVM_FAULT_MAXPROT) != 0; 777 struct vm_anon *anons_store[UVM_MAXRANGE], **anons; 778 struct vm_page *pages_store[UVM_MAXRANGE], **pages; 779 int error; 780 UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist); 781 782 UVMHIST_LOG(maphist, "(map=0x%x, vaddr=0x%x, at=%d, ff=%d)", 783 orig_map, vaddr, access_type, fault_flag); 784 785 curcpu()->ci_data.cpu_nfault++; 786 787 /* 788 * init the IN parameters in the ufi 789 */ 790 791 ufi.orig_map = orig_map; 792 ufi.orig_rvaddr = trunc_page(vaddr); 793 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ 794 795 error = ERESTART; 796 while (error == ERESTART) { /* ReFault: */ 797 anons = anons_store; 798 pages = pages_store; 799 800 error = uvm_fault_check(&ufi, &flt, &anons, maxprot); 801 if (error != 0) 802 continue; 803 804 error = uvm_fault_upper_lookup(&ufi, &flt, anons, pages); 805 if (error != 0) 806 continue; 807 808 if (pages[flt.centeridx] == PGO_DONTCARE) 809 error = uvm_fault_upper(&ufi, &flt, anons); 810 else { 811 struct uvm_object * const uobj = 812 ufi.entry->object.uvm_obj; 813 814 if (uobj && uobj->pgops->pgo_fault != NULL) { 815 /* 816 * invoke "special" fault routine. 817 */ 818 mutex_enter(uobj->vmobjlock); 819 /* locked: maps(read), amap(if there), uobj */ 820 error = uobj->pgops->pgo_fault(&ufi, 821 flt.startva, pages, flt.npages, 822 flt.centeridx, flt.access_type, 823 PGO_LOCKED|PGO_SYNCIO); 824 825 /* 826 * locked: nothing, pgo_fault has unlocked 827 * everything 828 */ 829 830 /* 831 * object fault routine responsible for 832 * pmap_update(). 833 */ 834 } else { 835 error = uvm_fault_lower(&ufi, &flt, pages); 836 } 837 } 838 } 839 840 if (flt.anon_spare != NULL) { 841 flt.anon_spare->an_ref--; 842 KASSERT(flt.anon_spare->an_ref == 0); 843 KASSERT(flt.anon_spare->an_lock == NULL); 844 uvm_anfree(flt.anon_spare); 845 } 846 return error; 847 } 848 849 /* 850 * uvm_fault_check: check prot, handle needs-copy, etc. 851 * 852 * 1. lookup entry. 853 * 2. check protection. 854 * 3. adjust fault condition (mainly for simulated fault). 855 * 4. handle needs-copy (lazy amap copy). 856 * 5. establish range of interest for neighbor fault (aka pre-fault). 857 * 6. look up anons (if amap exists). 858 * 7. flush pages (if MADV_SEQUENTIAL) 859 * 860 * => called with nothing locked. 861 * => if we fail (result != 0) we unlock everything. 862 * => initialize/adjust many members of flt. 863 */ 864 865 static int 866 uvm_fault_check( 867 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 868 struct vm_anon ***ranons, bool maxprot) 869 { 870 struct vm_amap *amap; 871 struct uvm_object *uobj; 872 vm_prot_t check_prot; 873 int nback, nforw; 874 UVMHIST_FUNC("uvm_fault_check"); UVMHIST_CALLED(maphist); 875 876 /* 877 * lookup and lock the maps 878 */ 879 880 if (uvmfault_lookup(ufi, false) == false) { 881 UVMHIST_LOG(maphist, "<- no mapping @ 0x%x", ufi->orig_rvaddr, 882 0,0,0); 883 return EFAULT; 884 } 885 /* locked: maps(read) */ 886 887 #ifdef DIAGNOSTIC 888 if ((ufi->map->flags & VM_MAP_PAGEABLE) == 0) { 889 printf("Page fault on non-pageable map:\n"); 890 printf("ufi->map = %p\n", ufi->map); 891 printf("ufi->orig_map = %p\n", ufi->orig_map); 892 printf("ufi->orig_rvaddr = 0x%lx\n", (u_long) ufi->orig_rvaddr); 893 panic("uvm_fault: (ufi->map->flags & VM_MAP_PAGEABLE) == 0"); 894 } 895 #endif 896 897 /* 898 * check protection 899 */ 900 901 check_prot = maxprot ? 902 ufi->entry->max_protection : ufi->entry->protection; 903 if ((check_prot & flt->access_type) != flt->access_type) { 904 UVMHIST_LOG(maphist, 905 "<- protection failure (prot=0x%x, access=0x%x)", 906 ufi->entry->protection, flt->access_type, 0, 0); 907 uvmfault_unlockmaps(ufi, false); 908 return EACCES; 909 } 910 911 /* 912 * "enter_prot" is the protection we want to enter the page in at. 913 * for certain pages (e.g. copy-on-write pages) this protection can 914 * be more strict than ufi->entry->protection. "wired" means either 915 * the entry is wired or we are fault-wiring the pg. 916 */ 917 918 flt->enter_prot = ufi->entry->protection; 919 if (VM_MAPENT_ISWIRED(ufi->entry)) 920 flt->wire_mapping = true; 921 922 if (flt->wire_mapping) { 923 flt->access_type = flt->enter_prot; /* full access for wired */ 924 flt->cow_now = (check_prot & VM_PROT_WRITE) != 0; 925 } else { 926 flt->cow_now = (flt->access_type & VM_PROT_WRITE) != 0; 927 } 928 929 flt->promote = false; 930 931 /* 932 * handle "needs_copy" case. if we need to copy the amap we will 933 * have to drop our readlock and relock it with a write lock. (we 934 * need a write lock to change anything in a map entry [e.g. 935 * needs_copy]). 936 */ 937 938 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 939 if (flt->cow_now || (ufi->entry->object.uvm_obj == NULL)) { 940 KASSERT(!maxprot); 941 /* need to clear */ 942 UVMHIST_LOG(maphist, 943 " need to clear needs_copy and refault",0,0,0,0); 944 uvmfault_unlockmaps(ufi, false); 945 uvmfault_amapcopy(ufi); 946 uvmexp.fltamcopy++; 947 return ERESTART; 948 949 } else { 950 951 /* 952 * ensure that we pmap_enter page R/O since 953 * needs_copy is still true 954 */ 955 956 flt->enter_prot &= ~VM_PROT_WRITE; 957 } 958 } 959 960 /* 961 * identify the players 962 */ 963 964 amap = ufi->entry->aref.ar_amap; /* upper layer */ 965 uobj = ufi->entry->object.uvm_obj; /* lower layer */ 966 967 /* 968 * check for a case 0 fault. if nothing backing the entry then 969 * error now. 970 */ 971 972 if (amap == NULL && uobj == NULL) { 973 uvmfault_unlockmaps(ufi, false); 974 UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0); 975 return EFAULT; 976 } 977 978 /* 979 * establish range of interest based on advice from mapper 980 * and then clip to fit map entry. note that we only want 981 * to do this the first time through the fault. if we 982 * ReFault we will disable this by setting "narrow" to true. 983 */ 984 985 if (flt->narrow == false) { 986 987 /* wide fault (!narrow) */ 988 KASSERT(uvmadvice[ufi->entry->advice].advice == 989 ufi->entry->advice); 990 nback = MIN(uvmadvice[ufi->entry->advice].nback, 991 (ufi->orig_rvaddr - ufi->entry->start) >> PAGE_SHIFT); 992 flt->startva = ufi->orig_rvaddr - (nback << PAGE_SHIFT); 993 /* 994 * note: "-1" because we don't want to count the 995 * faulting page as forw 996 */ 997 nforw = MIN(uvmadvice[ufi->entry->advice].nforw, 998 ((ufi->entry->end - ufi->orig_rvaddr) >> 999 PAGE_SHIFT) - 1); 1000 flt->npages = nback + nforw + 1; 1001 flt->centeridx = nback; 1002 1003 flt->narrow = true; /* ensure only once per-fault */ 1004 1005 } else { 1006 1007 /* narrow fault! */ 1008 nback = nforw = 0; 1009 flt->startva = ufi->orig_rvaddr; 1010 flt->npages = 1; 1011 flt->centeridx = 0; 1012 1013 } 1014 /* offset from entry's start to pgs' start */ 1015 const voff_t eoff = flt->startva - ufi->entry->start; 1016 1017 /* locked: maps(read) */ 1018 UVMHIST_LOG(maphist, " narrow=%d, back=%d, forw=%d, startva=0x%x", 1019 flt->narrow, nback, nforw, flt->startva); 1020 UVMHIST_LOG(maphist, " entry=0x%x, amap=0x%x, obj=0x%x", ufi->entry, 1021 amap, uobj, 0); 1022 1023 /* 1024 * if we've got an amap, lock it and extract current anons. 1025 */ 1026 1027 if (amap) { 1028 amap_lock(amap); 1029 amap_lookups(&ufi->entry->aref, eoff, *ranons, flt->npages); 1030 } else { 1031 *ranons = NULL; /* to be safe */ 1032 } 1033 1034 /* locked: maps(read), amap(if there) */ 1035 KASSERT(amap == NULL || mutex_owned(amap->am_lock)); 1036 1037 /* 1038 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages 1039 * now and then forget about them (for the rest of the fault). 1040 */ 1041 1042 if (ufi->entry->advice == MADV_SEQUENTIAL && nback != 0) { 1043 1044 UVMHIST_LOG(maphist, " MADV_SEQUENTIAL: flushing backpages", 1045 0,0,0,0); 1046 /* flush back-page anons? */ 1047 if (amap) 1048 uvmfault_anonflush(*ranons, nback); 1049 1050 /* flush object? */ 1051 if (uobj) { 1052 voff_t uoff; 1053 1054 uoff = ufi->entry->offset + eoff; 1055 mutex_enter(uobj->vmobjlock); 1056 (void) (uobj->pgops->pgo_put)(uobj, uoff, uoff + 1057 (nback << PAGE_SHIFT), PGO_DEACTIVATE); 1058 } 1059 1060 /* now forget about the backpages */ 1061 if (amap) 1062 *ranons += nback; 1063 flt->startva += (nback << PAGE_SHIFT); 1064 flt->npages -= nback; 1065 flt->centeridx = 0; 1066 } 1067 /* 1068 * => startva is fixed 1069 * => npages is fixed 1070 */ 1071 KASSERT(flt->startva <= ufi->orig_rvaddr); 1072 KASSERT(ufi->orig_rvaddr + ufi->orig_size <= 1073 flt->startva + (flt->npages << PAGE_SHIFT)); 1074 return 0; 1075 } 1076 1077 /* 1078 * uvm_fault_upper_lookup: look up existing h/w mapping and amap. 1079 * 1080 * iterate range of interest: 1081 * 1. check if h/w mapping exists. if yes, we don't care 1082 * 2. check if anon exists. if not, page is lower. 1083 * 3. if anon exists, enter h/w mapping for neighbors. 1084 * 1085 * => called with amap locked (if exists). 1086 */ 1087 1088 static int 1089 uvm_fault_upper_lookup( 1090 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 1091 struct vm_anon **anons, struct vm_page **pages) 1092 { 1093 struct vm_amap *amap = ufi->entry->aref.ar_amap; 1094 int lcv; 1095 vaddr_t currva; 1096 bool shadowed; 1097 UVMHIST_FUNC("uvm_fault_upper_lookup"); UVMHIST_CALLED(maphist); 1098 1099 /* locked: maps(read), amap(if there) */ 1100 KASSERT(amap == NULL || mutex_owned(amap->am_lock)); 1101 1102 /* 1103 * map in the backpages and frontpages we found in the amap in hopes 1104 * of preventing future faults. we also init the pages[] array as 1105 * we go. 1106 */ 1107 1108 currva = flt->startva; 1109 shadowed = false; 1110 for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) { 1111 /* 1112 * don't play with VAs that are already mapped 1113 * (except for center) 1114 */ 1115 if (lcv != flt->centeridx && 1116 pmap_extract(ufi->orig_map->pmap, currva, NULL)) { 1117 pages[lcv] = PGO_DONTCARE; 1118 continue; 1119 } 1120 1121 /* 1122 * unmapped or center page. check if any anon at this level. 1123 */ 1124 if (amap == NULL || anons[lcv] == NULL) { 1125 pages[lcv] = NULL; 1126 continue; 1127 } 1128 1129 /* 1130 * check for present page and map if possible. re-activate it. 1131 */ 1132 1133 pages[lcv] = PGO_DONTCARE; 1134 if (lcv == flt->centeridx) { /* save center for later! */ 1135 shadowed = true; 1136 continue; 1137 } 1138 1139 struct vm_anon *anon = anons[lcv]; 1140 struct vm_page *pg = anon->an_page; 1141 1142 KASSERT(anon->an_lock == amap->am_lock); 1143 1144 /* Ignore loaned and busy pages. */ 1145 if (pg && pg->loan_count == 0 && (pg->flags & PG_BUSY) == 0) { 1146 uvm_fault_upper_neighbor(ufi, flt, currva, 1147 pg, anon->an_ref > 1); 1148 } 1149 } 1150 1151 /* locked: maps(read), amap(if there) */ 1152 KASSERT(amap == NULL || mutex_owned(amap->am_lock)); 1153 /* (shadowed == true) if there is an anon at the faulting address */ 1154 UVMHIST_LOG(maphist, " shadowed=%d, will_get=%d", shadowed, 1155 (ufi->entry->object.uvm_obj && shadowed != false),0,0); 1156 1157 /* 1158 * note that if we are really short of RAM we could sleep in the above 1159 * call to pmap_enter with everything locked. bad? 1160 * 1161 * XXX Actually, that is bad; pmap_enter() should just fail in that 1162 * XXX case. --thorpej 1163 */ 1164 1165 return 0; 1166 } 1167 1168 /* 1169 * uvm_fault_upper_neighbor: enter single lower neighbor page. 1170 * 1171 * => called with amap and anon locked. 1172 */ 1173 1174 static void 1175 uvm_fault_upper_neighbor( 1176 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 1177 vaddr_t currva, struct vm_page *pg, bool readonly) 1178 { 1179 UVMHIST_FUNC("uvm_fault_upper_neighbor"); UVMHIST_CALLED(maphist); 1180 1181 /* locked: amap, anon */ 1182 1183 mutex_enter(&uvm_pageqlock); 1184 uvm_pageenqueue(pg); 1185 mutex_exit(&uvm_pageqlock); 1186 UVMHIST_LOG(maphist, 1187 " MAPPING: n anon: pm=0x%x, va=0x%x, pg=0x%x", 1188 ufi->orig_map->pmap, currva, pg, 0); 1189 uvmexp.fltnamap++; 1190 1191 /* 1192 * Since this page isn't the page that's actually faulting, 1193 * ignore pmap_enter() failures; it's not critical that we 1194 * enter these right now. 1195 */ 1196 1197 (void) pmap_enter(ufi->orig_map->pmap, currva, 1198 VM_PAGE_TO_PHYS(pg), 1199 readonly ? (flt->enter_prot & ~VM_PROT_WRITE) : 1200 flt->enter_prot, 1201 PMAP_CANFAIL | (flt->wire_mapping ? PMAP_WIRED : 0)); 1202 1203 pmap_update(ufi->orig_map->pmap); 1204 } 1205 1206 /* 1207 * uvm_fault_upper: handle upper fault. 1208 * 1209 * 1. acquire anon lock. 1210 * 2. get anon. let uvmfault_anonget do the dirty work. 1211 * 3. handle loan. 1212 * 4. dispatch direct or promote handlers. 1213 */ 1214 1215 static int 1216 uvm_fault_upper( 1217 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1218 struct vm_anon **anons) 1219 { 1220 struct vm_amap * const amap = ufi->entry->aref.ar_amap; 1221 struct vm_anon * const anon = anons[flt->centeridx]; 1222 struct uvm_object *uobj; 1223 int error; 1224 UVMHIST_FUNC("uvm_fault_upper"); UVMHIST_CALLED(maphist); 1225 1226 /* locked: maps(read), amap, anon */ 1227 KASSERT(mutex_owned(amap->am_lock)); 1228 KASSERT(anon->an_lock == amap->am_lock); 1229 1230 /* 1231 * handle case 1: fault on an anon in our amap 1232 */ 1233 1234 UVMHIST_LOG(maphist, " case 1 fault: anon=0x%x", anon, 0,0,0); 1235 1236 /* 1237 * no matter if we have case 1A or case 1B we are going to need to 1238 * have the anon's memory resident. ensure that now. 1239 */ 1240 1241 /* 1242 * let uvmfault_anonget do the dirty work. 1243 * if it fails (!OK) it will unlock everything for us. 1244 * if it succeeds, locks are still valid and locked. 1245 * also, if it is OK, then the anon's page is on the queues. 1246 * if the page is on loan from a uvm_object, then anonget will 1247 * lock that object for us if it does not fail. 1248 */ 1249 1250 error = uvmfault_anonget(ufi, amap, anon); 1251 switch (error) { 1252 case 0: 1253 break; 1254 1255 case ERESTART: 1256 return ERESTART; 1257 1258 case EAGAIN: 1259 kpause("fltagain1", false, hz/2, NULL); 1260 return ERESTART; 1261 1262 default: 1263 return error; 1264 } 1265 1266 /* 1267 * uobj is non null if the page is on loan from an object (i.e. uobj) 1268 */ 1269 1270 uobj = anon->an_page->uobject; /* locked by anonget if !NULL */ 1271 1272 /* locked: maps(read), amap, anon, uobj(if one) */ 1273 KASSERT(mutex_owned(amap->am_lock)); 1274 KASSERT(anon->an_lock == amap->am_lock); 1275 KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock)); 1276 1277 /* 1278 * special handling for loaned pages 1279 */ 1280 1281 if (anon->an_page->loan_count) { 1282 error = uvm_fault_upper_loan(ufi, flt, anon, &uobj); 1283 if (error != 0) 1284 return error; 1285 } 1286 1287 /* 1288 * if we are case 1B then we will need to allocate a new blank 1289 * anon to transfer the data into. note that we have a lock 1290 * on anon, so no one can busy or release the page until we are done. 1291 * also note that the ref count can't drop to zero here because 1292 * it is > 1 and we are only dropping one ref. 1293 * 1294 * in the (hopefully very rare) case that we are out of RAM we 1295 * will unlock, wait for more RAM, and refault. 1296 * 1297 * if we are out of anon VM we kill the process (XXX: could wait?). 1298 */ 1299 1300 if (flt->cow_now && anon->an_ref > 1) { 1301 flt->promote = true; 1302 error = uvm_fault_upper_promote(ufi, flt, uobj, anon); 1303 } else { 1304 error = uvm_fault_upper_direct(ufi, flt, uobj, anon); 1305 } 1306 return error; 1307 } 1308 1309 /* 1310 * uvm_fault_upper_loan: handle loaned upper page. 1311 * 1312 * 1. if not cow'ing now, simply adjust flt->enter_prot. 1313 * 2. if cow'ing now, and if ref count is 1, break loan. 1314 */ 1315 1316 static int 1317 uvm_fault_upper_loan( 1318 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1319 struct vm_anon *anon, struct uvm_object **ruobj) 1320 { 1321 struct vm_amap * const amap = ufi->entry->aref.ar_amap; 1322 int error = 0; 1323 UVMHIST_FUNC("uvm_fault_upper_loan"); UVMHIST_CALLED(maphist); 1324 1325 if (!flt->cow_now) { 1326 1327 /* 1328 * for read faults on loaned pages we just cap the 1329 * protection at read-only. 1330 */ 1331 1332 flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE; 1333 1334 } else { 1335 /* 1336 * note that we can't allow writes into a loaned page! 1337 * 1338 * if we have a write fault on a loaned page in an 1339 * anon then we need to look at the anon's ref count. 1340 * if it is greater than one then we are going to do 1341 * a normal copy-on-write fault into a new anon (this 1342 * is not a problem). however, if the reference count 1343 * is one (a case where we would normally allow a 1344 * write directly to the page) then we need to kill 1345 * the loan before we continue. 1346 */ 1347 1348 /* >1 case is already ok */ 1349 if (anon->an_ref == 1) { 1350 error = uvm_loanbreak_anon(anon, *ruobj); 1351 if (error != 0) { 1352 uvmfault_unlockall(ufi, amap, *ruobj); 1353 uvm_wait("flt_noram2"); 1354 return ERESTART; 1355 } 1356 /* if we were a loan reciever uobj is gone */ 1357 if (*ruobj) 1358 *ruobj = NULL; 1359 } 1360 } 1361 return error; 1362 } 1363 1364 /* 1365 * uvm_fault_upper_promote: promote upper page. 1366 * 1367 * 1. call uvmfault_promote. 1368 * 2. enqueue page. 1369 * 3. deref. 1370 * 4. pass page to uvm_fault_upper_enter. 1371 */ 1372 1373 static int 1374 uvm_fault_upper_promote( 1375 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1376 struct uvm_object *uobj, struct vm_anon *anon) 1377 { 1378 struct vm_anon * const oanon = anon; 1379 struct vm_page *pg; 1380 int error; 1381 UVMHIST_FUNC("uvm_fault_upper_promote"); UVMHIST_CALLED(maphist); 1382 1383 UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0); 1384 uvmexp.flt_acow++; 1385 1386 error = uvmfault_promote(ufi, oanon, PGO_DONTCARE, &anon, 1387 &flt->anon_spare); 1388 switch (error) { 1389 case 0: 1390 break; 1391 case ERESTART: 1392 return ERESTART; 1393 default: 1394 return error; 1395 } 1396 1397 KASSERT(anon == NULL || anon->an_lock == oanon->an_lock); 1398 1399 pg = anon->an_page; 1400 mutex_enter(&uvm_pageqlock); 1401 uvm_pageenqueue(pg); /* uvm_fault_upper_done will activate the page */ 1402 mutex_exit(&uvm_pageqlock); 1403 pg->flags &= ~(PG_BUSY|PG_FAKE); 1404 UVM_PAGE_OWN(pg, NULL); 1405 1406 /* deref: can not drop to zero here by defn! */ 1407 KASSERT(oanon->an_ref > 1); 1408 oanon->an_ref--; 1409 1410 /* 1411 * note: oanon is still locked, as is the new anon. we 1412 * need to check for this later when we unlock oanon; if 1413 * oanon != anon, we'll have to unlock anon, too. 1414 */ 1415 1416 return uvm_fault_upper_enter(ufi, flt, uobj, anon, pg, oanon); 1417 } 1418 1419 /* 1420 * uvm_fault_upper_direct: handle direct fault. 1421 */ 1422 1423 static int 1424 uvm_fault_upper_direct( 1425 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1426 struct uvm_object *uobj, struct vm_anon *anon) 1427 { 1428 struct vm_anon * const oanon = anon; 1429 struct vm_page *pg; 1430 UVMHIST_FUNC("uvm_fault_upper_direct"); UVMHIST_CALLED(maphist); 1431 1432 uvmexp.flt_anon++; 1433 pg = anon->an_page; 1434 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */ 1435 flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE; 1436 1437 return uvm_fault_upper_enter(ufi, flt, uobj, anon, pg, oanon); 1438 } 1439 1440 /* 1441 * uvm_fault_upper_enter: enter h/w mapping of upper page. 1442 */ 1443 1444 static int 1445 uvm_fault_upper_enter( 1446 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 1447 struct uvm_object *uobj, struct vm_anon *anon, struct vm_page *pg, 1448 struct vm_anon *oanon) 1449 { 1450 struct vm_amap * const amap = ufi->entry->aref.ar_amap; 1451 UVMHIST_FUNC("uvm_fault_upper_enter"); UVMHIST_CALLED(maphist); 1452 1453 /* locked: maps(read), amap, oanon, anon(if different from oanon) */ 1454 KASSERT(mutex_owned(amap->am_lock)); 1455 KASSERT(anon->an_lock == amap->am_lock); 1456 KASSERT(oanon->an_lock == amap->am_lock); 1457 KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock)); 1458 1459 /* 1460 * now map the page in. 1461 */ 1462 1463 UVMHIST_LOG(maphist, 1464 " MAPPING: anon: pm=0x%x, va=0x%x, pg=0x%x, promote=%d", 1465 ufi->orig_map->pmap, ufi->orig_rvaddr, pg, flt->promote); 1466 if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr, 1467 VM_PAGE_TO_PHYS(pg), 1468 flt->enter_prot, flt->access_type | PMAP_CANFAIL | 1469 (flt->wire_mapping ? PMAP_WIRED : 0)) != 0) { 1470 1471 /* 1472 * No need to undo what we did; we can simply think of 1473 * this as the pmap throwing away the mapping information. 1474 * 1475 * We do, however, have to go through the ReFault path, 1476 * as the map may change while we're asleep. 1477 */ 1478 1479 uvmfault_unlockall(ufi, amap, uobj); 1480 if (!uvm_reclaimable()) { 1481 UVMHIST_LOG(maphist, 1482 "<- failed. out of VM",0,0,0,0); 1483 /* XXX instrumentation */ 1484 return ENOMEM; 1485 } 1486 /* XXX instrumentation */ 1487 uvm_wait("flt_pmfail1"); 1488 return ERESTART; 1489 } 1490 1491 uvm_fault_upper_done(ufi, flt, anon, pg); 1492 1493 /* 1494 * done case 1! finish up by unlocking everything and returning success 1495 */ 1496 1497 pmap_update(ufi->orig_map->pmap); 1498 uvmfault_unlockall(ufi, amap, uobj); 1499 return 0; 1500 } 1501 1502 /* 1503 * uvm_fault_upper_done: queue upper center page. 1504 */ 1505 1506 static void 1507 uvm_fault_upper_done( 1508 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 1509 struct vm_anon *anon, struct vm_page *pg) 1510 { 1511 const bool wire_paging = flt->wire_paging; 1512 1513 UVMHIST_FUNC("uvm_fault_upper_done"); UVMHIST_CALLED(maphist); 1514 1515 /* 1516 * ... update the page queues. 1517 */ 1518 1519 mutex_enter(&uvm_pageqlock); 1520 if (wire_paging) { 1521 uvm_pagewire(pg); 1522 1523 /* 1524 * since the now-wired page cannot be paged out, 1525 * release its swap resources for others to use. 1526 * since an anon with no swap cannot be PG_CLEAN, 1527 * clear its clean flag now. 1528 */ 1529 1530 pg->flags &= ~(PG_CLEAN); 1531 1532 } else { 1533 uvm_pageactivate(pg); 1534 } 1535 mutex_exit(&uvm_pageqlock); 1536 1537 if (wire_paging) { 1538 uvm_anon_dropswap(anon); 1539 } 1540 } 1541 1542 /* 1543 * uvm_fault_lower: handle lower fault. 1544 * 1545 * 1. check uobj 1546 * 1.1. if null, ZFOD. 1547 * 1.2. if not null, look up unnmapped neighbor pages. 1548 * 2. for center page, check if promote. 1549 * 2.1. ZFOD always needs promotion. 1550 * 2.2. other uobjs, when entry is marked COW (usually MAP_PRIVATE vnode). 1551 * 3. if uobj is not ZFOD and page is not found, do i/o. 1552 * 4. dispatch either direct / promote fault. 1553 */ 1554 1555 static int 1556 uvm_fault_lower( 1557 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1558 struct vm_page **pages) 1559 { 1560 #ifdef DIAGNOSTIC 1561 struct vm_amap *amap = ufi->entry->aref.ar_amap; 1562 #endif 1563 struct uvm_object *uobj = ufi->entry->object.uvm_obj; 1564 struct vm_page *uobjpage; 1565 int error; 1566 UVMHIST_FUNC("uvm_fault_lower"); UVMHIST_CALLED(maphist); 1567 1568 /* 1569 * now, if the desired page is not shadowed by the amap and we have 1570 * a backing object that does not have a special fault routine, then 1571 * we ask (with pgo_get) the object for resident pages that we care 1572 * about and attempt to map them in. we do not let pgo_get block 1573 * (PGO_LOCKED). 1574 */ 1575 1576 if (uobj == NULL) { 1577 /* zero fill; don't care neighbor pages */ 1578 uobjpage = NULL; 1579 } else { 1580 uvm_fault_lower_lookup(ufi, flt, pages); 1581 uobjpage = pages[flt->centeridx]; 1582 } 1583 1584 /* 1585 * note that at this point we are done with any front or back pages. 1586 * we are now going to focus on the center page (i.e. the one we've 1587 * faulted on). if we have faulted on the upper (anon) layer 1588 * [i.e. case 1], then the anon we want is anons[centeridx] (we have 1589 * not touched it yet). if we have faulted on the bottom (uobj) 1590 * layer [i.e. case 2] and the page was both present and available, 1591 * then we've got a pointer to it as "uobjpage" and we've already 1592 * made it BUSY. 1593 */ 1594 1595 /* 1596 * locked: 1597 * maps(read), amap(if there), uobj(if !null), uobjpage(if !null) 1598 */ 1599 KASSERT(amap == NULL || mutex_owned(amap->am_lock)); 1600 KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock)); 1601 KASSERT(uobjpage == NULL || (uobjpage->flags & PG_BUSY) != 0); 1602 1603 /* 1604 * note that uobjpage can not be PGO_DONTCARE at this point. we now 1605 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we 1606 * have a backing object, check and see if we are going to promote 1607 * the data up to an anon during the fault. 1608 */ 1609 1610 if (uobj == NULL) { 1611 uobjpage = PGO_DONTCARE; 1612 flt->promote = true; /* always need anon here */ 1613 } else { 1614 KASSERT(uobjpage != PGO_DONTCARE); 1615 flt->promote = flt->cow_now && UVM_ET_ISCOPYONWRITE(ufi->entry); 1616 } 1617 UVMHIST_LOG(maphist, " case 2 fault: promote=%d, zfill=%d", 1618 flt->promote, (uobj == NULL), 0,0); 1619 1620 /* 1621 * if uobjpage is not null then we do not need to do I/O to get the 1622 * uobjpage. 1623 * 1624 * if uobjpage is null, then we need to unlock and ask the pager to 1625 * get the data for us. once we have the data, we need to reverify 1626 * the state the world. we are currently not holding any resources. 1627 */ 1628 1629 if (uobjpage) { 1630 /* update rusage counters */ 1631 curlwp->l_ru.ru_minflt++; 1632 } else { 1633 error = uvm_fault_lower_io(ufi, flt, &uobj, &uobjpage); 1634 if (error != 0) 1635 return error; 1636 } 1637 1638 /* 1639 * locked: 1640 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj) 1641 */ 1642 KASSERT(amap == NULL || mutex_owned(amap->am_lock)); 1643 KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock)); 1644 KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0); 1645 1646 /* 1647 * notes: 1648 * - at this point uobjpage can not be NULL 1649 * - at this point uobjpage can not be PG_RELEASED (since we checked 1650 * for it above) 1651 * - at this point uobjpage could be PG_WANTED (handle later) 1652 */ 1653 1654 KASSERT(uobjpage != NULL); 1655 KASSERT(uobj == NULL || uobj == uobjpage->uobject); 1656 KASSERT(uobj == NULL || !UVM_OBJ_IS_CLEAN(uobjpage->uobject) || 1657 (uobjpage->flags & PG_CLEAN) != 0); 1658 1659 if (!flt->promote) { 1660 error = uvm_fault_lower_direct(ufi, flt, uobj, uobjpage); 1661 } else { 1662 error = uvm_fault_lower_promote(ufi, flt, uobj, uobjpage); 1663 } 1664 return error; 1665 } 1666 1667 /* 1668 * uvm_fault_lower_lookup: look up on-memory uobj pages. 1669 * 1670 * 1. get on-memory pages. 1671 * 2. if failed, give up (get only center page later). 1672 * 3. if succeeded, enter h/w mapping of neighbor pages. 1673 */ 1674 1675 static void 1676 uvm_fault_lower_lookup( 1677 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 1678 struct vm_page **pages) 1679 { 1680 struct uvm_object *uobj = ufi->entry->object.uvm_obj; 1681 int lcv, gotpages; 1682 vaddr_t currva; 1683 UVMHIST_FUNC("uvm_fault_lower_lookup"); UVMHIST_CALLED(maphist); 1684 1685 mutex_enter(uobj->vmobjlock); 1686 /* Locked: maps(read), amap(if there), uobj */ 1687 1688 uvmexp.fltlget++; 1689 gotpages = flt->npages; 1690 (void) uobj->pgops->pgo_get(uobj, 1691 ufi->entry->offset + flt->startva - ufi->entry->start, 1692 pages, &gotpages, flt->centeridx, 1693 flt->access_type & MASK(ufi->entry), ufi->entry->advice, PGO_LOCKED); 1694 1695 KASSERT(mutex_owned(uobj->vmobjlock)); 1696 1697 /* 1698 * check for pages to map, if we got any 1699 */ 1700 1701 if (gotpages == 0) { 1702 pages[flt->centeridx] = NULL; 1703 return; 1704 } 1705 1706 currva = flt->startva; 1707 for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) { 1708 struct vm_page *curpg; 1709 1710 curpg = pages[lcv]; 1711 if (curpg == NULL || curpg == PGO_DONTCARE) { 1712 continue; 1713 } 1714 KASSERT(curpg->uobject == uobj); 1715 1716 /* 1717 * if center page is resident and not PG_BUSY|PG_RELEASED 1718 * then pgo_get made it PG_BUSY for us and gave us a handle 1719 * to it. 1720 */ 1721 1722 if (lcv == flt->centeridx) { 1723 UVMHIST_LOG(maphist, " got uobjpage " 1724 "(0x%x) with locked get", 1725 curpg, 0,0,0); 1726 } else { 1727 bool readonly = (curpg->flags & PG_RDONLY) 1728 || (curpg->loan_count > 0) 1729 || UVM_OBJ_NEEDS_WRITEFAULT(curpg->uobject); 1730 1731 uvm_fault_lower_neighbor(ufi, flt, 1732 currva, curpg, readonly); 1733 } 1734 } 1735 pmap_update(ufi->orig_map->pmap); 1736 } 1737 1738 /* 1739 * uvm_fault_lower_neighbor: enter h/w mapping of lower neighbor page. 1740 */ 1741 1742 static void 1743 uvm_fault_lower_neighbor( 1744 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 1745 vaddr_t currva, struct vm_page *pg, bool readonly) 1746 { 1747 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 1748 1749 /* locked: maps(read), amap(if there), uobj */ 1750 1751 /* 1752 * calling pgo_get with PGO_LOCKED returns us pages which 1753 * are neither busy nor released, so we don't need to check 1754 * for this. we can just directly enter the pages. 1755 */ 1756 1757 mutex_enter(&uvm_pageqlock); 1758 uvm_pageenqueue(pg); 1759 mutex_exit(&uvm_pageqlock); 1760 UVMHIST_LOG(maphist, 1761 " MAPPING: n obj: pm=0x%x, va=0x%x, pg=0x%x", 1762 ufi->orig_map->pmap, currva, pg, 0); 1763 uvmexp.fltnomap++; 1764 1765 /* 1766 * Since this page isn't the page that's actually faulting, 1767 * ignore pmap_enter() failures; it's not critical that we 1768 * enter these right now. 1769 * NOTE: page can't be PG_WANTED or PG_RELEASED because we've 1770 * held the lock the whole time we've had the handle. 1771 */ 1772 KASSERT((pg->flags & PG_PAGEOUT) == 0); 1773 KASSERT((pg->flags & PG_RELEASED) == 0); 1774 KASSERT((pg->flags & PG_WANTED) == 0); 1775 KASSERT(!UVM_OBJ_IS_CLEAN(pg->uobject) || (pg->flags & PG_CLEAN) != 0); 1776 pg->flags &= ~(PG_BUSY); 1777 UVM_PAGE_OWN(pg, NULL); 1778 1779 KASSERT(mutex_owned(pg->uobject->vmobjlock)); 1780 (void) pmap_enter(ufi->orig_map->pmap, currva, 1781 VM_PAGE_TO_PHYS(pg), 1782 readonly ? (flt->enter_prot & ~VM_PROT_WRITE) : 1783 flt->enter_prot & MASK(ufi->entry), 1784 PMAP_CANFAIL | (flt->wire_mapping ? PMAP_WIRED : 0)); 1785 } 1786 1787 /* 1788 * uvm_fault_lower_io: get lower page from backing store. 1789 * 1790 * 1. unlock everything, because i/o will block. 1791 * 2. call pgo_get. 1792 * 3. if failed, recover. 1793 * 4. if succeeded, relock everything and verify things. 1794 */ 1795 1796 static int 1797 uvm_fault_lower_io( 1798 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 1799 struct uvm_object **ruobj, struct vm_page **ruobjpage) 1800 { 1801 struct vm_amap * const amap = ufi->entry->aref.ar_amap; 1802 struct uvm_object *uobj = *ruobj; 1803 struct vm_page *pg; 1804 bool locked; 1805 int gotpages; 1806 int error; 1807 voff_t uoff; 1808 UVMHIST_FUNC("uvm_fault_lower_io"); UVMHIST_CALLED(maphist); 1809 1810 /* update rusage counters */ 1811 curlwp->l_ru.ru_majflt++; 1812 1813 /* Locked: maps(read), amap(if there), uobj */ 1814 uvmfault_unlockall(ufi, amap, NULL); 1815 1816 /* Locked: uobj */ 1817 KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock)); 1818 1819 uvmexp.fltget++; 1820 gotpages = 1; 1821 pg = NULL; 1822 uoff = (ufi->orig_rvaddr - ufi->entry->start) + ufi->entry->offset; 1823 error = uobj->pgops->pgo_get(uobj, uoff, &pg, &gotpages, 1824 0, flt->access_type & MASK(ufi->entry), ufi->entry->advice, 1825 PGO_SYNCIO); 1826 /* locked: pg(if no error) */ 1827 1828 /* 1829 * recover from I/O 1830 */ 1831 1832 if (error) { 1833 if (error == EAGAIN) { 1834 UVMHIST_LOG(maphist, 1835 " pgo_get says TRY AGAIN!",0,0,0,0); 1836 kpause("fltagain2", false, hz/2, NULL); 1837 return ERESTART; 1838 } 1839 1840 #if 0 1841 KASSERT(error != ERESTART); 1842 #else 1843 /* XXXUEBS don't re-fault? */ 1844 if (error == ERESTART) 1845 error = EIO; 1846 #endif 1847 1848 UVMHIST_LOG(maphist, "<- pgo_get failed (code %d)", 1849 error, 0,0,0); 1850 return error; 1851 } 1852 1853 /* 1854 * re-verify the state of the world by first trying to relock 1855 * the maps. always relock the object. 1856 */ 1857 1858 locked = uvmfault_relock(ufi); 1859 if (locked && amap) 1860 amap_lock(amap); 1861 1862 /* might be changed */ 1863 uobj = pg->uobject; 1864 1865 mutex_enter(uobj->vmobjlock); 1866 KASSERT((pg->flags & PG_BUSY) != 0); 1867 1868 mutex_enter(&uvm_pageqlock); 1869 uvm_pageactivate(pg); 1870 mutex_exit(&uvm_pageqlock); 1871 1872 /* locked(locked): maps(read), amap(if !null), uobj, pg */ 1873 /* locked(!locked): uobj, pg */ 1874 1875 /* 1876 * verify that the page has not be released and re-verify 1877 * that amap slot is still free. if there is a problem, 1878 * we unlock and clean up. 1879 */ 1880 1881 if ((pg->flags & PG_RELEASED) != 0 || 1882 (locked && amap && amap_lookup(&ufi->entry->aref, 1883 ufi->orig_rvaddr - ufi->entry->start))) { 1884 if (locked) 1885 uvmfault_unlockall(ufi, amap, NULL); 1886 locked = false; 1887 } 1888 1889 /* 1890 * didn't get the lock? release the page and retry. 1891 */ 1892 1893 if (locked == false) { 1894 UVMHIST_LOG(maphist, 1895 " wasn't able to relock after fault: retry", 1896 0,0,0,0); 1897 if (pg->flags & PG_WANTED) { 1898 wakeup(pg); 1899 } 1900 if ((pg->flags & PG_RELEASED) == 0) { 1901 pg->flags &= ~(PG_BUSY | PG_WANTED); 1902 UVM_PAGE_OWN(pg, NULL); 1903 } else { 1904 uvmexp.fltpgrele++; 1905 uvm_pagefree(pg); 1906 } 1907 mutex_exit(uobj->vmobjlock); 1908 return ERESTART; 1909 } 1910 1911 /* 1912 * we have the data in pg which is busy and 1913 * not released. we are holding object lock (so the page 1914 * can't be released on us). 1915 */ 1916 1917 /* locked: maps(read), amap(if !null), uobj, pg */ 1918 1919 *ruobj = uobj; 1920 *ruobjpage = pg; 1921 return 0; 1922 } 1923 1924 /* 1925 * uvm_fault_lower_direct: fault lower center page 1926 * 1927 * 1. adjust flt->enter_prot. 1928 * 2. if page is loaned, resolve. 1929 */ 1930 1931 int 1932 uvm_fault_lower_direct( 1933 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1934 struct uvm_object *uobj, struct vm_page *uobjpage) 1935 { 1936 struct vm_page *pg; 1937 UVMHIST_FUNC("uvm_fault_lower_direct"); UVMHIST_CALLED(maphist); 1938 1939 /* 1940 * we are not promoting. if the mapping is COW ensure that we 1941 * don't give more access than we should (e.g. when doing a read 1942 * fault on a COPYONWRITE mapping we want to map the COW page in 1943 * R/O even though the entry protection could be R/W). 1944 * 1945 * set "pg" to the page we want to map in (uobjpage, usually) 1946 */ 1947 1948 uvmexp.flt_obj++; 1949 if (UVM_ET_ISCOPYONWRITE(ufi->entry) || 1950 UVM_OBJ_NEEDS_WRITEFAULT(uobjpage->uobject)) 1951 flt->enter_prot &= ~VM_PROT_WRITE; 1952 pg = uobjpage; /* map in the actual object */ 1953 1954 KASSERT(uobjpage != PGO_DONTCARE); 1955 1956 /* 1957 * we are faulting directly on the page. be careful 1958 * about writing to loaned pages... 1959 */ 1960 1961 if (uobjpage->loan_count) { 1962 uvm_fault_lower_direct_loan(ufi, flt, uobj, &pg, &uobjpage); 1963 } 1964 KASSERT(pg == uobjpage); 1965 1966 KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0); 1967 return uvm_fault_lower_enter(ufi, flt, uobj, NULL, pg); 1968 } 1969 1970 /* 1971 * uvm_fault_lower_direct_loan: resolve loaned page. 1972 * 1973 * 1. if not cow'ing, adjust flt->enter_prot. 1974 * 2. if cow'ing, break loan. 1975 */ 1976 1977 static int 1978 uvm_fault_lower_direct_loan( 1979 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1980 struct uvm_object *uobj, struct vm_page **rpg, 1981 struct vm_page **ruobjpage) 1982 { 1983 struct vm_amap * const amap = ufi->entry->aref.ar_amap; 1984 struct vm_page *pg; 1985 struct vm_page *uobjpage = *ruobjpage; 1986 UVMHIST_FUNC("uvm_fault_lower_direct_loan"); UVMHIST_CALLED(maphist); 1987 1988 if (!flt->cow_now) { 1989 /* read fault: cap the protection at readonly */ 1990 /* cap! */ 1991 flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE; 1992 } else { 1993 /* write fault: must break the loan here */ 1994 1995 pg = uvm_loanbreak(uobjpage); 1996 if (pg == NULL) { 1997 1998 /* 1999 * drop ownership of page, it can't be released 2000 */ 2001 2002 if (uobjpage->flags & PG_WANTED) 2003 wakeup(uobjpage); 2004 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 2005 UVM_PAGE_OWN(uobjpage, NULL); 2006 2007 uvmfault_unlockall(ufi, amap, uobj); 2008 UVMHIST_LOG(maphist, 2009 " out of RAM breaking loan, waiting", 2010 0,0,0,0); 2011 uvmexp.fltnoram++; 2012 uvm_wait("flt_noram4"); 2013 return ERESTART; 2014 } 2015 *rpg = pg; 2016 *ruobjpage = pg; 2017 } 2018 return 0; 2019 } 2020 2021 /* 2022 * uvm_fault_lower_promote: promote lower page. 2023 * 2024 * 1. call uvmfault_promote. 2025 * 2. fill in data. 2026 * 3. if not ZFOD, dispose old page. 2027 */ 2028 2029 int 2030 uvm_fault_lower_promote( 2031 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 2032 struct uvm_object *uobj, struct vm_page *uobjpage) 2033 { 2034 struct vm_amap * const amap = ufi->entry->aref.ar_amap; 2035 struct vm_anon *anon; 2036 struct vm_page *pg; 2037 int error; 2038 UVMHIST_FUNC("uvm_fault_lower_promote"); UVMHIST_CALLED(maphist); 2039 2040 KASSERT(amap != NULL); 2041 2042 /* 2043 * If we are going to promote the data to an anon we 2044 * allocate a blank anon here and plug it into our amap. 2045 */ 2046 error = uvmfault_promote(ufi, NULL, uobjpage, 2047 &anon, &flt->anon_spare); 2048 switch (error) { 2049 case 0: 2050 break; 2051 case ERESTART: 2052 return ERESTART; 2053 default: 2054 return error; 2055 } 2056 2057 pg = anon->an_page; 2058 2059 /* 2060 * Fill in the data. 2061 */ 2062 KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0); 2063 2064 if (uobjpage != PGO_DONTCARE) { 2065 uvmexp.flt_prcopy++; 2066 2067 /* 2068 * promote to shared amap? make sure all sharing 2069 * procs see it 2070 */ 2071 2072 if ((amap_flags(amap) & AMAP_SHARED) != 0) { 2073 pmap_page_protect(uobjpage, VM_PROT_NONE); 2074 /* 2075 * XXX: PAGE MIGHT BE WIRED! 2076 */ 2077 } 2078 2079 /* 2080 * dispose of uobjpage. it can't be PG_RELEASED 2081 * since we still hold the object lock. 2082 */ 2083 2084 if (uobjpage->flags & PG_WANTED) { 2085 /* still have the obj lock */ 2086 wakeup(uobjpage); 2087 } 2088 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 2089 UVM_PAGE_OWN(uobjpage, NULL); 2090 2091 UVMHIST_LOG(maphist, 2092 " promote uobjpage 0x%x to anon/page 0x%x/0x%x", 2093 uobjpage, anon, pg, 0); 2094 2095 } else { 2096 uvmexp.flt_przero++; 2097 2098 /* 2099 * Page is zero'd and marked dirty by 2100 * uvmfault_promote(). 2101 */ 2102 2103 UVMHIST_LOG(maphist," zero fill anon/page 0x%x/0%x", 2104 anon, pg, 0, 0); 2105 } 2106 2107 return uvm_fault_lower_enter(ufi, flt, uobj, anon, pg); 2108 } 2109 2110 /* 2111 * uvm_fault_lower_enter: enter h/w mapping of lower page or anon page promoted 2112 * from the lower page. 2113 */ 2114 2115 int 2116 uvm_fault_lower_enter( 2117 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 2118 struct uvm_object *uobj, 2119 struct vm_anon *anon, struct vm_page *pg) 2120 { 2121 struct vm_amap * const amap = ufi->entry->aref.ar_amap; 2122 int error; 2123 UVMHIST_FUNC("uvm_fault_lower_enter"); UVMHIST_CALLED(maphist); 2124 2125 /* 2126 * Locked: 2127 * 2128 * maps(read), amap(if !null), uobj(if !null), 2129 * anon(if !null), pg(if anon), unlock_uobj(if !null) 2130 * 2131 * Note: pg is either the uobjpage or the new page in the new anon. 2132 */ 2133 KASSERT(amap == NULL || mutex_owned(amap->am_lock)); 2134 KASSERT(uobj == NULL || mutex_owned(uobj->vmobjlock)); 2135 KASSERT(anon == NULL || anon->an_lock == amap->am_lock); 2136 KASSERT((pg->flags & PG_BUSY) != 0); 2137 2138 /* 2139 * all resources are present. we can now map it in and free our 2140 * resources. 2141 */ 2142 2143 UVMHIST_LOG(maphist, 2144 " MAPPING: case2: pm=0x%x, va=0x%x, pg=0x%x, promote=%d", 2145 ufi->orig_map->pmap, ufi->orig_rvaddr, pg, flt->promote); 2146 KASSERT((flt->access_type & VM_PROT_WRITE) == 0 || 2147 (pg->flags & PG_RDONLY) == 0); 2148 if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr, 2149 VM_PAGE_TO_PHYS(pg), 2150 (pg->flags & PG_RDONLY) != 0 ? 2151 flt->enter_prot & ~VM_PROT_WRITE : flt->enter_prot, 2152 flt->access_type | PMAP_CANFAIL | 2153 (flt->wire_mapping ? PMAP_WIRED : 0)) != 0) { 2154 2155 /* 2156 * No need to undo what we did; we can simply think of 2157 * this as the pmap throwing away the mapping information. 2158 * 2159 * We do, however, have to go through the ReFault path, 2160 * as the map may change while we're asleep. 2161 */ 2162 2163 /* 2164 * ensure that the page is queued in the case that 2165 * we just promoted the page. 2166 */ 2167 2168 mutex_enter(&uvm_pageqlock); 2169 uvm_pageenqueue(pg); 2170 mutex_exit(&uvm_pageqlock); 2171 2172 if (pg->flags & PG_WANTED) 2173 wakeup(pg); 2174 2175 /* 2176 * note that pg can't be PG_RELEASED since we did not drop 2177 * the object lock since the last time we checked. 2178 */ 2179 KASSERT((pg->flags & PG_RELEASED) == 0); 2180 2181 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); 2182 UVM_PAGE_OWN(pg, NULL); 2183 2184 uvmfault_unlockall(ufi, amap, uobj); 2185 if (!uvm_reclaimable()) { 2186 UVMHIST_LOG(maphist, 2187 "<- failed. out of VM",0,0,0,0); 2188 /* XXX instrumentation */ 2189 error = ENOMEM; 2190 return error; 2191 } 2192 /* XXX instrumentation */ 2193 uvm_wait("flt_pmfail2"); 2194 return ERESTART; 2195 } 2196 2197 uvm_fault_lower_done(ufi, flt, uobj, pg); 2198 2199 /* 2200 * note that pg can't be PG_RELEASED since we did not drop the object 2201 * lock since the last time we checked. 2202 */ 2203 KASSERT((pg->flags & PG_RELEASED) == 0); 2204 if (pg->flags & PG_WANTED) 2205 wakeup(pg); 2206 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); 2207 UVM_PAGE_OWN(pg, NULL); 2208 2209 pmap_update(ufi->orig_map->pmap); 2210 uvmfault_unlockall(ufi, amap, uobj); 2211 2212 UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0); 2213 return 0; 2214 } 2215 2216 /* 2217 * uvm_fault_lower_done: queue lower center page. 2218 */ 2219 2220 void 2221 uvm_fault_lower_done( 2222 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 2223 struct uvm_object *uobj, struct vm_page *pg) 2224 { 2225 bool dropswap = false; 2226 2227 UVMHIST_FUNC("uvm_fault_lower_done"); UVMHIST_CALLED(maphist); 2228 2229 mutex_enter(&uvm_pageqlock); 2230 if (flt->wire_paging) { 2231 uvm_pagewire(pg); 2232 if (pg->pqflags & PQ_AOBJ) { 2233 2234 /* 2235 * since the now-wired page cannot be paged out, 2236 * release its swap resources for others to use. 2237 * since an aobj page with no swap cannot be PG_CLEAN, 2238 * clear its clean flag now. 2239 */ 2240 2241 KASSERT(uobj != NULL); 2242 pg->flags &= ~(PG_CLEAN); 2243 dropswap = true; 2244 } 2245 } else { 2246 uvm_pageactivate(pg); 2247 } 2248 mutex_exit(&uvm_pageqlock); 2249 2250 if (dropswap) { 2251 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT); 2252 } 2253 } 2254 2255 2256 /* 2257 * uvm_fault_wire: wire down a range of virtual addresses in a map. 2258 * 2259 * => map may be read-locked by caller, but MUST NOT be write-locked. 2260 * => if map is read-locked, any operations which may cause map to 2261 * be write-locked in uvm_fault() must be taken care of by 2262 * the caller. See uvm_map_pageable(). 2263 */ 2264 2265 int 2266 uvm_fault_wire(struct vm_map *map, vaddr_t start, vaddr_t end, 2267 vm_prot_t access_type, int maxprot) 2268 { 2269 vaddr_t va; 2270 int error; 2271 2272 /* 2273 * now fault it in a page at a time. if the fault fails then we have 2274 * to undo what we have done. note that in uvm_fault VM_PROT_NONE 2275 * is replaced with the max protection if fault_type is VM_FAULT_WIRE. 2276 */ 2277 2278 /* 2279 * XXX work around overflowing a vaddr_t. this prevents us from 2280 * wiring the last page in the address space, though. 2281 */ 2282 if (start > end) { 2283 return EFAULT; 2284 } 2285 2286 for (va = start; va < end; va += PAGE_SIZE) { 2287 error = uvm_fault_internal(map, va, access_type, 2288 (maxprot ? UVM_FAULT_MAXPROT : 0) | UVM_FAULT_WIRE); 2289 if (error) { 2290 if (va != start) { 2291 uvm_fault_unwire(map, start, va); 2292 } 2293 return error; 2294 } 2295 } 2296 return 0; 2297 } 2298 2299 /* 2300 * uvm_fault_unwire(): unwire range of virtual space. 2301 */ 2302 2303 void 2304 uvm_fault_unwire(struct vm_map *map, vaddr_t start, vaddr_t end) 2305 { 2306 vm_map_lock_read(map); 2307 uvm_fault_unwire_locked(map, start, end); 2308 vm_map_unlock_read(map); 2309 } 2310 2311 /* 2312 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire(). 2313 * 2314 * => map must be at least read-locked. 2315 */ 2316 2317 void 2318 uvm_fault_unwire_locked(struct vm_map *map, vaddr_t start, vaddr_t end) 2319 { 2320 struct vm_map_entry *entry, *oentry; 2321 pmap_t pmap = vm_map_pmap(map); 2322 vaddr_t va; 2323 paddr_t pa; 2324 struct vm_page *pg; 2325 2326 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0); 2327 2328 /* 2329 * we assume that the area we are unwiring has actually been wired 2330 * in the first place. this means that we should be able to extract 2331 * the PAs from the pmap. we also lock out the page daemon so that 2332 * we can call uvm_pageunwire. 2333 */ 2334 2335 /* 2336 * find the beginning map entry for the region. 2337 */ 2338 2339 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map)); 2340 if (uvm_map_lookup_entry(map, start, &entry) == false) 2341 panic("uvm_fault_unwire_locked: address not in map"); 2342 2343 oentry = NULL; 2344 for (va = start; va < end; va += PAGE_SIZE) { 2345 if (pmap_extract(pmap, va, &pa) == false) 2346 continue; 2347 2348 /* 2349 * find the map entry for the current address. 2350 */ 2351 2352 KASSERT(va >= entry->start); 2353 while (va >= entry->end) { 2354 KASSERT(entry->next != &map->header && 2355 entry->next->start <= entry->end); 2356 entry = entry->next; 2357 } 2358 2359 /* 2360 * lock it. 2361 */ 2362 2363 if (entry != oentry) { 2364 if (oentry != NULL) { 2365 mutex_exit(&uvm_pageqlock); 2366 uvm_map_unlock_entry(oentry); 2367 } 2368 uvm_map_lock_entry(entry); 2369 mutex_enter(&uvm_pageqlock); 2370 oentry = entry; 2371 } 2372 2373 /* 2374 * if the entry is no longer wired, tell the pmap. 2375 */ 2376 2377 if (VM_MAPENT_ISWIRED(entry) == 0) 2378 pmap_unwire(pmap, va); 2379 2380 pg = PHYS_TO_VM_PAGE(pa); 2381 if (pg) 2382 uvm_pageunwire(pg); 2383 } 2384 2385 if (oentry != NULL) { 2386 mutex_exit(&uvm_pageqlock); 2387 uvm_map_unlock_entry(entry); 2388 } 2389 } 2390