1 /* $NetBSD: uvm_fault.c,v 1.217 2020/02/24 12:38:57 rin Exp $ */ 2 3 /* 4 * Copyright (c) 1997 Charles D. Cranor and Washington University. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 * 27 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp 28 */ 29 30 /* 31 * uvm_fault.c: fault handler 32 */ 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.217 2020/02/24 12:38:57 rin Exp $"); 36 37 #include "opt_uvmhist.h" 38 39 #include <sys/param.h> 40 #include <sys/systm.h> 41 #include <sys/atomic.h> 42 #include <sys/kernel.h> 43 #include <sys/mman.h> 44 45 #include <uvm/uvm.h> 46 47 /* 48 * 49 * a word on page faults: 50 * 51 * types of page faults we handle: 52 * 53 * CASE 1: upper layer faults CASE 2: lower layer faults 54 * 55 * CASE 1A CASE 1B CASE 2A CASE 2B 56 * read/write1 write>1 read/write +-cow_write/zero 57 * | | | | 58 * +--|--+ +--|--+ +-----+ + | + | +-----+ 59 * amap | V | | ---------> new | | | | ^ | 60 * +-----+ +-----+ +-----+ + | + | +--|--+ 61 * | | | 62 * +-----+ +-----+ +--|--+ | +--|--+ 63 * uobj | d/c | | d/c | | V | +----+ | 64 * +-----+ +-----+ +-----+ +-----+ 65 * 66 * d/c = don't care 67 * 68 * case [0]: layerless fault 69 * no amap or uobj is present. this is an error. 70 * 71 * case [1]: upper layer fault [anon active] 72 * 1A: [read] or [write with anon->an_ref == 1] 73 * I/O takes place in upper level anon and uobj is not touched. 74 * 1B: [write with anon->an_ref > 1] 75 * new anon is alloc'd and data is copied off ["COW"] 76 * 77 * case [2]: lower layer fault [uobj] 78 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area] 79 * I/O takes place directly in object. 80 * 2B: [write to copy_on_write] or [read on NULL uobj] 81 * data is "promoted" from uobj to a new anon. 82 * if uobj is null, then we zero fill. 83 * 84 * we follow the standard UVM locking protocol ordering: 85 * 86 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ) 87 * we hold a PG_BUSY page if we unlock for I/O 88 * 89 * 90 * the code is structured as follows: 91 * 92 * - init the "IN" params in the ufi structure 93 * ReFault: (ERESTART returned to the loop in uvm_fault_internal) 94 * - do lookups [locks maps], check protection, handle needs_copy 95 * - check for case 0 fault (error) 96 * - establish "range" of fault 97 * - if we have an amap lock it and extract the anons 98 * - if sequential advice deactivate pages behind us 99 * - at the same time check pmap for unmapped areas and anon for pages 100 * that we could map in (and do map it if found) 101 * - check object for resident pages that we could map in 102 * - if (case 2) goto Case2 103 * - >>> handle case 1 104 * - ensure source anon is resident in RAM 105 * - if case 1B alloc new anon and copy from source 106 * - map the correct page in 107 * Case2: 108 * - >>> handle case 2 109 * - ensure source page is resident (if uobj) 110 * - if case 2B alloc new anon and copy from source (could be zero 111 * fill if uobj == NULL) 112 * - map the correct page in 113 * - done! 114 * 115 * note on paging: 116 * if we have to do I/O we place a PG_BUSY page in the correct object, 117 * unlock everything, and do the I/O. when I/O is done we must reverify 118 * the state of the world before assuming that our data structures are 119 * valid. [because mappings could change while the map is unlocked] 120 * 121 * alternative 1: unbusy the page in question and restart the page fault 122 * from the top (ReFault). this is easy but does not take advantage 123 * of the information that we already have from our previous lookup, 124 * although it is possible that the "hints" in the vm_map will help here. 125 * 126 * alternative 2: the system already keeps track of a "version" number of 127 * a map. [i.e. every time you write-lock a map (e.g. to change a 128 * mapping) you bump the version number up by one...] so, we can save 129 * the version number of the map before we release the lock and start I/O. 130 * then when I/O is done we can relock and check the version numbers 131 * to see if anything changed. this might save us some over 1 because 132 * we don't have to unbusy the page and may be less compares(?). 133 * 134 * alternative 3: put in backpointers or a way to "hold" part of a map 135 * in place while I/O is in progress. this could be complex to 136 * implement (especially with structures like amap that can be referenced 137 * by multiple map entries, and figuring out what should wait could be 138 * complex as well...). 139 * 140 * we use alternative 2. given that we are multi-threaded now we may want 141 * to reconsider the choice. 142 */ 143 144 /* 145 * local data structures 146 */ 147 148 struct uvm_advice { 149 int advice; 150 int nback; 151 int nforw; 152 }; 153 154 /* 155 * page range array: 156 * note: index in array must match "advice" value 157 * XXX: borrowed numbers from freebsd. do they work well for us? 158 */ 159 160 static const struct uvm_advice uvmadvice[] = { 161 { UVM_ADV_NORMAL, 3, 4 }, 162 { UVM_ADV_RANDOM, 0, 0 }, 163 { UVM_ADV_SEQUENTIAL, 8, 7}, 164 }; 165 166 #define UVM_MAXRANGE 16 /* must be MAX() of nback+nforw+1 */ 167 168 /* 169 * private prototypes 170 */ 171 172 /* 173 * externs from other modules 174 */ 175 176 extern int start_init_exec; /* Is init_main() done / init running? */ 177 178 /* 179 * inline functions 180 */ 181 182 /* 183 * uvmfault_anonflush: try and deactivate pages in specified anons 184 * 185 * => does not have to deactivate page if it is busy 186 */ 187 188 static inline void 189 uvmfault_anonflush(struct vm_anon **anons, int n) 190 { 191 int lcv; 192 struct vm_page *pg; 193 194 for (lcv = 0; lcv < n; lcv++) { 195 if (anons[lcv] == NULL) 196 continue; 197 KASSERT(rw_write_held(anons[lcv]->an_lock)); 198 pg = anons[lcv]->an_page; 199 if (pg && (pg->flags & PG_BUSY) == 0) { 200 uvm_pagelock(pg); 201 uvm_pagedeactivate(pg); 202 uvm_pageunlock(pg); 203 } 204 } 205 } 206 207 /* 208 * normal functions 209 */ 210 211 /* 212 * uvmfault_amapcopy: clear "needs_copy" in a map. 213 * 214 * => called with VM data structures unlocked (usually, see below) 215 * => we get a write lock on the maps and clear needs_copy for a VA 216 * => if we are out of RAM we sleep (waiting for more) 217 */ 218 219 static void 220 uvmfault_amapcopy(struct uvm_faultinfo *ufi) 221 { 222 for (;;) { 223 224 /* 225 * no mapping? give up. 226 */ 227 228 if (uvmfault_lookup(ufi, true) == false) 229 return; 230 231 /* 232 * copy if needed. 233 */ 234 235 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) 236 amap_copy(ufi->map, ufi->entry, AMAP_COPY_NOWAIT, 237 ufi->orig_rvaddr, ufi->orig_rvaddr + 1); 238 239 /* 240 * didn't work? must be out of RAM. unlock and sleep. 241 */ 242 243 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 244 uvmfault_unlockmaps(ufi, true); 245 uvm_wait("fltamapcopy"); 246 continue; 247 } 248 249 /* 250 * got it! unlock and return. 251 */ 252 253 uvmfault_unlockmaps(ufi, true); 254 return; 255 } 256 /*NOTREACHED*/ 257 } 258 259 /* 260 * uvmfault_anonget: get data in an anon into a non-busy, non-released 261 * page in that anon. 262 * 263 * => Map, amap and thus anon should be locked by caller. 264 * => If we fail, we unlock everything and error is returned. 265 * => If we are successful, return with everything still locked. 266 * => We do not move the page on the queues [gets moved later]. If we 267 * allocate a new page [we_own], it gets put on the queues. Either way, 268 * the result is that the page is on the queues at return time 269 * => For pages which are on loan from a uvm_object (and thus are not owned 270 * by the anon): if successful, return with the owning object locked. 271 * The caller must unlock this object when it unlocks everything else. 272 */ 273 274 int 275 uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap, 276 struct vm_anon *anon) 277 { 278 struct vm_page *pg; 279 int error; 280 281 UVMHIST_FUNC("uvmfault_anonget"); UVMHIST_CALLED(maphist); 282 KASSERT(rw_write_held(anon->an_lock)); 283 KASSERT(anon->an_lock == amap->am_lock); 284 285 /* Increment the counters.*/ 286 cpu_count(CPU_COUNT_FLTANGET, 1); 287 if (anon->an_page) { 288 curlwp->l_ru.ru_minflt++; 289 } else { 290 curlwp->l_ru.ru_majflt++; 291 } 292 error = 0; 293 294 /* 295 * Loop until we get the anon data, or fail. 296 */ 297 298 for (;;) { 299 bool we_own, locked; 300 /* 301 * Note: 'we_own' will become true if we set PG_BUSY on a page. 302 */ 303 we_own = false; 304 pg = anon->an_page; 305 306 /* 307 * If there is a resident page and it is loaned, then anon 308 * may not own it. Call out to uvm_anon_lockloanpg() to 309 * identify and lock the real owner of the page. 310 */ 311 312 if (pg && pg->loan_count) 313 pg = uvm_anon_lockloanpg(anon); 314 315 /* 316 * Is page resident? Make sure it is not busy/released. 317 */ 318 319 if (pg) { 320 321 /* 322 * at this point, if the page has a uobject [meaning 323 * we have it on loan], then that uobject is locked 324 * by us! if the page is busy, we drop all the 325 * locks (including uobject) and try again. 326 */ 327 328 if ((pg->flags & PG_BUSY) == 0) { 329 UVMHIST_LOG(maphist, "<- OK",0,0,0,0); 330 return 0; 331 } 332 pg->flags |= PG_WANTED; 333 cpu_count(CPU_COUNT_FLTPGWAIT, 1); 334 335 /* 336 * The last unlock must be an atomic unlock and wait 337 * on the owner of page. 338 */ 339 340 if (pg->uobject) { 341 /* Owner of page is UVM object. */ 342 uvmfault_unlockall(ufi, amap, NULL); 343 UVMHIST_LOG(maphist, " unlock+wait on uobj",0, 344 0,0,0); 345 UVM_UNLOCK_AND_WAIT_RW(pg, 346 pg->uobject->vmobjlock, 347 false, "anonget1", 0); 348 } else { 349 /* Owner of page is anon. */ 350 uvmfault_unlockall(ufi, NULL, NULL); 351 UVMHIST_LOG(maphist, " unlock+wait on anon",0, 352 0,0,0); 353 UVM_UNLOCK_AND_WAIT_RW(pg, anon->an_lock, 354 false, "anonget2", 0); 355 } 356 } else { 357 #if defined(VMSWAP) 358 /* 359 * No page, therefore allocate one. 360 */ 361 362 pg = uvm_pagealloc(NULL, 363 ufi != NULL ? ufi->orig_rvaddr : 0, 364 anon, ufi != NULL ? UVM_FLAG_COLORMATCH : 0); 365 if (pg == NULL) { 366 /* Out of memory. Wait a little. */ 367 uvmfault_unlockall(ufi, amap, NULL); 368 cpu_count(CPU_COUNT_FLTNORAM, 1); 369 UVMHIST_LOG(maphist, " noram -- UVM_WAIT",0, 370 0,0,0); 371 if (!uvm_reclaimable()) { 372 return ENOMEM; 373 } 374 uvm_wait("flt_noram1"); 375 } else { 376 /* PG_BUSY bit is set. */ 377 we_own = true; 378 uvmfault_unlockall(ufi, amap, NULL); 379 380 /* 381 * Pass a PG_BUSY+PG_FAKE clean page into 382 * the uvm_swap_get() function with all data 383 * structures unlocked. Note that it is OK 384 * to read an_swslot here, because we hold 385 * PG_BUSY on the page. 386 */ 387 cpu_count(CPU_COUNT_PAGEINS, 1); 388 error = uvm_swap_get(pg, anon->an_swslot, 389 PGO_SYNCIO); 390 391 /* 392 * We clean up after the I/O below in the 393 * 'we_own' case. 394 */ 395 } 396 #else 397 panic("%s: no page", __func__); 398 #endif /* defined(VMSWAP) */ 399 } 400 401 /* 402 * Re-lock the map and anon. 403 */ 404 405 locked = uvmfault_relock(ufi); 406 if (locked || we_own) { 407 rw_enter(anon->an_lock, RW_WRITER); 408 } 409 410 /* 411 * If we own the page (i.e. we set PG_BUSY), then we need 412 * to clean up after the I/O. There are three cases to 413 * consider: 414 * 415 * 1) Page was released during I/O: free anon and ReFault. 416 * 2) I/O not OK. Free the page and cause the fault to fail. 417 * 3) I/O OK! Activate the page and sync with the non-we_own 418 * case (i.e. drop anon lock if not locked). 419 */ 420 421 if (we_own) { 422 #if defined(VMSWAP) 423 if (pg->flags & PG_WANTED) { 424 wakeup(pg); 425 } 426 if (error) { 427 428 /* 429 * Remove the swap slot from the anon and 430 * mark the anon as having no real slot. 431 * Do not free the swap slot, thus preventing 432 * it from being used again. 433 */ 434 435 if (anon->an_swslot > 0) { 436 uvm_swap_markbad(anon->an_swslot, 1); 437 } 438 anon->an_swslot = SWSLOT_BAD; 439 440 if ((pg->flags & PG_RELEASED) != 0) { 441 goto released; 442 } 443 444 /* 445 * Note: page was never !PG_BUSY, so it 446 * cannot be mapped and thus no need to 447 * pmap_page_protect() it. 448 */ 449 450 uvm_pagefree(pg); 451 452 if (locked) { 453 uvmfault_unlockall(ufi, NULL, NULL); 454 } 455 rw_exit(anon->an_lock); 456 UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0); 457 return error; 458 } 459 460 if ((pg->flags & PG_RELEASED) != 0) { 461 released: 462 KASSERT(anon->an_ref == 0); 463 464 /* 465 * Released while we had unlocked amap. 466 */ 467 468 if (locked) { 469 uvmfault_unlockall(ufi, NULL, NULL); 470 } 471 uvm_anon_release(anon); 472 473 if (error) { 474 UVMHIST_LOG(maphist, 475 "<- ERROR/RELEASED", 0,0,0,0); 476 return error; 477 } 478 479 UVMHIST_LOG(maphist, "<- RELEASED", 0,0,0,0); 480 return ERESTART; 481 } 482 483 /* 484 * We have successfully read the page, activate it. 485 */ 486 487 uvm_pagelock(pg); 488 uvm_pageactivate(pg); 489 uvm_pageunlock(pg); 490 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE); 491 uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_UNKNOWN); 492 UVM_PAGE_OWN(pg, NULL); 493 #else 494 panic("%s: we_own", __func__); 495 #endif /* defined(VMSWAP) */ 496 } 497 498 /* 499 * We were not able to re-lock the map - restart the fault. 500 */ 501 502 if (!locked) { 503 if (we_own) { 504 rw_exit(anon->an_lock); 505 } 506 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 507 return ERESTART; 508 } 509 510 /* 511 * Verify that no one has touched the amap and moved 512 * the anon on us. 513 */ 514 515 if (ufi != NULL && amap_lookup(&ufi->entry->aref, 516 ufi->orig_rvaddr - ufi->entry->start) != anon) { 517 518 uvmfault_unlockall(ufi, amap, NULL); 519 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 520 return ERESTART; 521 } 522 523 /* 524 * Retry.. 525 */ 526 527 cpu_count(CPU_COUNT_FLTANRETRY, 1); 528 continue; 529 } 530 /*NOTREACHED*/ 531 } 532 533 /* 534 * uvmfault_promote: promote data to a new anon. used for 1B and 2B. 535 * 536 * 1. allocate an anon and a page. 537 * 2. fill its contents. 538 * 3. put it into amap. 539 * 540 * => if we fail (result != 0) we unlock everything. 541 * => on success, return a new locked anon via 'nanon'. 542 * (*nanon)->an_page will be a resident, locked, dirty page. 543 * => it's caller's responsibility to put the promoted nanon->an_page to the 544 * page queue. 545 */ 546 547 static int 548 uvmfault_promote(struct uvm_faultinfo *ufi, 549 struct vm_anon *oanon, 550 struct vm_page *uobjpage, 551 struct vm_anon **nanon, /* OUT: allocated anon */ 552 struct vm_anon **spare) 553 { 554 struct vm_amap *amap = ufi->entry->aref.ar_amap; 555 struct uvm_object *uobj; 556 struct vm_anon *anon; 557 struct vm_page *pg; 558 struct vm_page *opg; 559 int error; 560 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 561 562 if (oanon) { 563 /* anon COW */ 564 opg = oanon->an_page; 565 KASSERT(opg != NULL); 566 KASSERT(opg->uobject == NULL || opg->loan_count > 0); 567 } else if (uobjpage != PGO_DONTCARE) { 568 /* object-backed COW */ 569 opg = uobjpage; 570 } else { 571 /* ZFOD */ 572 opg = NULL; 573 } 574 if (opg != NULL) { 575 uobj = opg->uobject; 576 } else { 577 uobj = NULL; 578 } 579 580 KASSERT(amap != NULL); 581 KASSERT(uobjpage != NULL); 582 KASSERT(uobjpage == PGO_DONTCARE || (uobjpage->flags & PG_BUSY) != 0); 583 KASSERT(rw_write_held(amap->am_lock)); 584 KASSERT(oanon == NULL || amap->am_lock == oanon->an_lock); 585 KASSERT(uobj == NULL || rw_write_held(uobj->vmobjlock)); 586 587 if (*spare != NULL) { 588 anon = *spare; 589 *spare = NULL; 590 } else { 591 anon = uvm_analloc(); 592 } 593 if (anon) { 594 595 /* 596 * The new anon is locked. 597 * 598 * if opg == NULL, we want a zero'd, dirty page, 599 * so have uvm_pagealloc() do that for us. 600 */ 601 602 KASSERT(anon->an_lock == NULL); 603 anon->an_lock = amap->am_lock; 604 pg = uvm_pagealloc(NULL, ufi->orig_rvaddr, anon, 605 UVM_FLAG_COLORMATCH | (opg == NULL ? UVM_PGA_ZERO : 0)); 606 if (pg == NULL) { 607 anon->an_lock = NULL; 608 } 609 } else { 610 pg = NULL; 611 } 612 613 /* 614 * out of memory resources? 615 */ 616 617 if (pg == NULL) { 618 /* save anon for the next try. */ 619 if (anon != NULL) { 620 *spare = anon; 621 } 622 623 /* unlock and fail ... */ 624 uvm_page_unbusy(&uobjpage, 1); 625 uvmfault_unlockall(ufi, amap, uobj); 626 if (!uvm_reclaimable()) { 627 UVMHIST_LOG(maphist, "out of VM", 0,0,0,0); 628 cpu_count(CPU_COUNT_FLTNOANON, 1); 629 error = ENOMEM; 630 goto done; 631 } 632 633 UVMHIST_LOG(maphist, "out of RAM, waiting for more", 0,0,0,0); 634 cpu_count(CPU_COUNT_FLTNORAM, 1); 635 uvm_wait("flt_noram5"); 636 error = ERESTART; 637 goto done; 638 } 639 640 /* copy page [pg now dirty] */ 641 if (opg) { 642 uvm_pagecopy(opg, pg); 643 } 644 KASSERT(uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_DIRTY); 645 646 amap_add(&ufi->entry->aref, ufi->orig_rvaddr - ufi->entry->start, anon, 647 oanon != NULL); 648 649 *nanon = anon; 650 error = 0; 651 done: 652 return error; 653 } 654 655 /* 656 * Update statistics after fault resolution. 657 * - maxrss 658 */ 659 void 660 uvmfault_update_stats(struct uvm_faultinfo *ufi) 661 { 662 struct vm_map *map; 663 struct vmspace *vm; 664 struct proc *p; 665 vsize_t res; 666 667 map = ufi->orig_map; 668 669 p = curproc; 670 KASSERT(p != NULL); 671 vm = p->p_vmspace; 672 673 if (&vm->vm_map != map) 674 return; 675 676 res = pmap_resident_count(map->pmap); 677 if (vm->vm_rssmax < res) 678 vm->vm_rssmax = res; 679 } 680 681 /* 682 * F A U L T - m a i n e n t r y p o i n t 683 */ 684 685 /* 686 * uvm_fault: page fault handler 687 * 688 * => called from MD code to resolve a page fault 689 * => VM data structures usually should be unlocked. however, it is 690 * possible to call here with the main map locked if the caller 691 * gets a write lock, sets it recusive, and then calls us (c.f. 692 * uvm_map_pageable). this should be avoided because it keeps 693 * the map locked off during I/O. 694 * => MUST NEVER BE CALLED IN INTERRUPT CONTEXT 695 */ 696 697 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 698 ~VM_PROT_WRITE : VM_PROT_ALL) 699 700 /* fault_flag values passed from uvm_fault_wire to uvm_fault_internal */ 701 #define UVM_FAULT_WIRE (1 << 0) 702 #define UVM_FAULT_MAXPROT (1 << 1) 703 704 struct uvm_faultctx { 705 706 /* 707 * the following members are set up by uvm_fault_check() and 708 * read-only after that. 709 * 710 * note that narrow is used by uvm_fault_check() to change 711 * the behaviour after ERESTART. 712 * 713 * most of them might change after RESTART if the underlying 714 * map entry has been changed behind us. an exception is 715 * wire_paging, which does never change. 716 */ 717 vm_prot_t access_type; 718 vaddr_t startva; 719 int npages; 720 int centeridx; 721 bool narrow; /* work on a single requested page only */ 722 bool wire_mapping; /* request a PMAP_WIRED mapping 723 (UVM_FAULT_WIRE or VM_MAPENT_ISWIRED) */ 724 bool wire_paging; /* request uvm_pagewire 725 (true for UVM_FAULT_WIRE) */ 726 bool cow_now; /* VM_PROT_WRITE is actually requested 727 (ie. should break COW and page loaning) */ 728 729 /* 730 * enter_prot is set up by uvm_fault_check() and clamped 731 * (ie. drop the VM_PROT_WRITE bit) in various places in case 732 * of !cow_now. 733 */ 734 vm_prot_t enter_prot; /* prot at which we want to enter pages in */ 735 736 /* 737 * the following member is for uvmfault_promote() and ERESTART. 738 */ 739 struct vm_anon *anon_spare; 740 741 /* 742 * the folloing is actually a uvm_fault_lower() internal. 743 * it's here merely for debugging. 744 * (or due to the mechanical separation of the function?) 745 */ 746 bool promote; 747 }; 748 749 static inline int uvm_fault_check( 750 struct uvm_faultinfo *, struct uvm_faultctx *, 751 struct vm_anon ***, bool); 752 753 static int uvm_fault_upper( 754 struct uvm_faultinfo *, struct uvm_faultctx *, 755 struct vm_anon **); 756 static inline int uvm_fault_upper_lookup( 757 struct uvm_faultinfo *, const struct uvm_faultctx *, 758 struct vm_anon **, struct vm_page **); 759 static inline void uvm_fault_upper_neighbor( 760 struct uvm_faultinfo *, const struct uvm_faultctx *, 761 vaddr_t, struct vm_page *, bool); 762 static inline int uvm_fault_upper_loan( 763 struct uvm_faultinfo *, struct uvm_faultctx *, 764 struct vm_anon *, struct uvm_object **); 765 static inline int uvm_fault_upper_promote( 766 struct uvm_faultinfo *, struct uvm_faultctx *, 767 struct uvm_object *, struct vm_anon *); 768 static inline int uvm_fault_upper_direct( 769 struct uvm_faultinfo *, struct uvm_faultctx *, 770 struct uvm_object *, struct vm_anon *); 771 static int uvm_fault_upper_enter( 772 struct uvm_faultinfo *, const struct uvm_faultctx *, 773 struct uvm_object *, struct vm_anon *, 774 struct vm_page *, struct vm_anon *); 775 static inline void uvm_fault_upper_done( 776 struct uvm_faultinfo *, const struct uvm_faultctx *, 777 struct vm_anon *, struct vm_page *); 778 779 static int uvm_fault_lower( 780 struct uvm_faultinfo *, struct uvm_faultctx *, 781 struct vm_page **); 782 static inline void uvm_fault_lower_lookup( 783 struct uvm_faultinfo *, const struct uvm_faultctx *, 784 struct vm_page **); 785 static inline void uvm_fault_lower_neighbor( 786 struct uvm_faultinfo *, const struct uvm_faultctx *, 787 vaddr_t, struct vm_page *); 788 static inline int uvm_fault_lower_io( 789 struct uvm_faultinfo *, const struct uvm_faultctx *, 790 struct uvm_object **, struct vm_page **); 791 static inline int uvm_fault_lower_direct( 792 struct uvm_faultinfo *, struct uvm_faultctx *, 793 struct uvm_object *, struct vm_page *); 794 static inline int uvm_fault_lower_direct_loan( 795 struct uvm_faultinfo *, struct uvm_faultctx *, 796 struct uvm_object *, struct vm_page **, 797 struct vm_page **); 798 static inline int uvm_fault_lower_promote( 799 struct uvm_faultinfo *, struct uvm_faultctx *, 800 struct uvm_object *, struct vm_page *); 801 static int uvm_fault_lower_enter( 802 struct uvm_faultinfo *, const struct uvm_faultctx *, 803 struct uvm_object *, 804 struct vm_anon *, struct vm_page *); 805 static inline void uvm_fault_lower_done( 806 struct uvm_faultinfo *, const struct uvm_faultctx *, 807 struct uvm_object *, struct vm_page *); 808 809 int 810 uvm_fault_internal(struct vm_map *orig_map, vaddr_t vaddr, 811 vm_prot_t access_type, int fault_flag) 812 { 813 struct uvm_faultinfo ufi; 814 struct uvm_faultctx flt = { 815 .access_type = access_type, 816 817 /* don't look for neighborhood * pages on "wire" fault */ 818 .narrow = (fault_flag & UVM_FAULT_WIRE) != 0, 819 820 /* "wire" fault causes wiring of both mapping and paging */ 821 .wire_mapping = (fault_flag & UVM_FAULT_WIRE) != 0, 822 .wire_paging = (fault_flag & UVM_FAULT_WIRE) != 0, 823 }; 824 const bool maxprot = (fault_flag & UVM_FAULT_MAXPROT) != 0; 825 struct vm_anon *anons_store[UVM_MAXRANGE], **anons; 826 struct vm_page *pages_store[UVM_MAXRANGE], **pages; 827 int error; 828 829 UVMHIST_FUNC("uvm_fault"); UVMHIST_CALLED(maphist); 830 831 UVMHIST_LOG(maphist, "(map=%#jx, vaddr=%#jx, at=%jd, ff=%jd)", 832 (uintptr_t)orig_map, vaddr, access_type, fault_flag); 833 834 /* Don't count anything until user interaction is possible */ 835 kpreempt_disable(); 836 if (__predict_true(start_init_exec)) { 837 struct cpu_info *ci = curcpu(); 838 CPU_COUNT(CPU_COUNT_NFAULT, 1); 839 /* Don't flood RNG subsystem with samples. */ 840 if (++(ci->ci_faultrng) == 503) { 841 ci->ci_faultrng = 0; 842 rnd_add_uint32(&curcpu()->ci_data.cpu_uvm->rs, 843 sizeof(vaddr_t) == sizeof(uint32_t) ? 844 (uint32_t)vaddr : sizeof(vaddr_t) == 845 sizeof(uint64_t) ? 846 (uint32_t)vaddr : 847 (uint32_t)ci->ci_counts[CPU_COUNT_NFAULT]); 848 } 849 } 850 kpreempt_enable(); 851 852 /* 853 * init the IN parameters in the ufi 854 */ 855 856 ufi.orig_map = orig_map; 857 ufi.orig_rvaddr = trunc_page(vaddr); 858 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ 859 860 error = ERESTART; 861 while (error == ERESTART) { /* ReFault: */ 862 anons = anons_store; 863 pages = pages_store; 864 865 error = uvm_fault_check(&ufi, &flt, &anons, maxprot); 866 if (error != 0) 867 continue; 868 869 error = uvm_fault_upper_lookup(&ufi, &flt, anons, pages); 870 if (error != 0) 871 continue; 872 873 if (pages[flt.centeridx] == PGO_DONTCARE) 874 error = uvm_fault_upper(&ufi, &flt, anons); 875 else { 876 struct uvm_object * const uobj = 877 ufi.entry->object.uvm_obj; 878 879 if (uobj && uobj->pgops->pgo_fault != NULL) { 880 /* 881 * invoke "special" fault routine. 882 */ 883 rw_enter(uobj->vmobjlock, RW_WRITER); 884 /* locked: maps(read), amap(if there), uobj */ 885 error = uobj->pgops->pgo_fault(&ufi, 886 flt.startva, pages, flt.npages, 887 flt.centeridx, flt.access_type, 888 PGO_LOCKED|PGO_SYNCIO); 889 890 /* 891 * locked: nothing, pgo_fault has unlocked 892 * everything 893 */ 894 895 /* 896 * object fault routine responsible for 897 * pmap_update(). 898 */ 899 900 /* 901 * Wake up the pagedaemon if the fault method 902 * failed for lack of memory but some can be 903 * reclaimed. 904 */ 905 if (error == ENOMEM && uvm_reclaimable()) { 906 uvm_wait("pgo_fault"); 907 error = ERESTART; 908 } 909 } else { 910 error = uvm_fault_lower(&ufi, &flt, pages); 911 } 912 } 913 } 914 915 if (flt.anon_spare != NULL) { 916 flt.anon_spare->an_ref--; 917 KASSERT(flt.anon_spare->an_ref == 0); 918 KASSERT(flt.anon_spare->an_lock == NULL); 919 uvm_anon_free(flt.anon_spare); 920 } 921 return error; 922 } 923 924 /* 925 * uvm_fault_check: check prot, handle needs-copy, etc. 926 * 927 * 1. lookup entry. 928 * 2. check protection. 929 * 3. adjust fault condition (mainly for simulated fault). 930 * 4. handle needs-copy (lazy amap copy). 931 * 5. establish range of interest for neighbor fault (aka pre-fault). 932 * 6. look up anons (if amap exists). 933 * 7. flush pages (if MADV_SEQUENTIAL) 934 * 935 * => called with nothing locked. 936 * => if we fail (result != 0) we unlock everything. 937 * => initialize/adjust many members of flt. 938 */ 939 940 static int 941 uvm_fault_check( 942 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 943 struct vm_anon ***ranons, bool maxprot) 944 { 945 struct vm_amap *amap; 946 struct uvm_object *uobj; 947 vm_prot_t check_prot; 948 int nback, nforw; 949 UVMHIST_FUNC("uvm_fault_check"); UVMHIST_CALLED(maphist); 950 951 /* 952 * lookup and lock the maps 953 */ 954 955 if (uvmfault_lookup(ufi, false) == false) { 956 UVMHIST_LOG(maphist, "<- no mapping @ %#jx", ufi->orig_rvaddr, 957 0,0,0); 958 return EFAULT; 959 } 960 /* locked: maps(read) */ 961 962 #ifdef DIAGNOSTIC 963 if ((ufi->map->flags & VM_MAP_PAGEABLE) == 0) { 964 printf("Page fault on non-pageable map:\n"); 965 printf("ufi->map = %p\n", ufi->map); 966 printf("ufi->orig_map = %p\n", ufi->orig_map); 967 printf("ufi->orig_rvaddr = %#lx\n", (u_long) ufi->orig_rvaddr); 968 panic("uvm_fault: (ufi->map->flags & VM_MAP_PAGEABLE) == 0"); 969 } 970 #endif 971 972 /* 973 * check protection 974 */ 975 976 check_prot = maxprot ? 977 ufi->entry->max_protection : ufi->entry->protection; 978 if ((check_prot & flt->access_type) != flt->access_type) { 979 UVMHIST_LOG(maphist, 980 "<- protection failure (prot=%#jx, access=%#jx)", 981 ufi->entry->protection, flt->access_type, 0, 0); 982 uvmfault_unlockmaps(ufi, false); 983 return EFAULT; 984 } 985 986 /* 987 * "enter_prot" is the protection we want to enter the page in at. 988 * for certain pages (e.g. copy-on-write pages) this protection can 989 * be more strict than ufi->entry->protection. "wired" means either 990 * the entry is wired or we are fault-wiring the pg. 991 */ 992 993 flt->enter_prot = ufi->entry->protection; 994 if (VM_MAPENT_ISWIRED(ufi->entry)) { 995 flt->wire_mapping = true; 996 flt->wire_paging = true; 997 flt->narrow = true; 998 } 999 1000 if (flt->wire_mapping) { 1001 flt->access_type = flt->enter_prot; /* full access for wired */ 1002 flt->cow_now = (check_prot & VM_PROT_WRITE) != 0; 1003 } else { 1004 flt->cow_now = (flt->access_type & VM_PROT_WRITE) != 0; 1005 } 1006 1007 flt->promote = false; 1008 1009 /* 1010 * handle "needs_copy" case. if we need to copy the amap we will 1011 * have to drop our readlock and relock it with a write lock. (we 1012 * need a write lock to change anything in a map entry [e.g. 1013 * needs_copy]). 1014 */ 1015 1016 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 1017 if (flt->cow_now || (ufi->entry->object.uvm_obj == NULL)) { 1018 KASSERT(!maxprot); 1019 /* need to clear */ 1020 UVMHIST_LOG(maphist, 1021 " need to clear needs_copy and refault",0,0,0,0); 1022 uvmfault_unlockmaps(ufi, false); 1023 uvmfault_amapcopy(ufi); 1024 cpu_count(CPU_COUNT_FLTAMCOPY, 1); 1025 return ERESTART; 1026 1027 } else { 1028 1029 /* 1030 * ensure that we pmap_enter page R/O since 1031 * needs_copy is still true 1032 */ 1033 1034 flt->enter_prot &= ~VM_PROT_WRITE; 1035 } 1036 } 1037 1038 /* 1039 * identify the players 1040 */ 1041 1042 amap = ufi->entry->aref.ar_amap; /* upper layer */ 1043 uobj = ufi->entry->object.uvm_obj; /* lower layer */ 1044 1045 /* 1046 * check for a case 0 fault. if nothing backing the entry then 1047 * error now. 1048 */ 1049 1050 if (amap == NULL && uobj == NULL) { 1051 uvmfault_unlockmaps(ufi, false); 1052 UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0); 1053 return EFAULT; 1054 } 1055 1056 /* 1057 * establish range of interest based on advice from mapper 1058 * and then clip to fit map entry. note that we only want 1059 * to do this the first time through the fault. if we 1060 * ReFault we will disable this by setting "narrow" to true. 1061 */ 1062 1063 if (flt->narrow == false) { 1064 1065 /* wide fault (!narrow) */ 1066 KASSERT(uvmadvice[ufi->entry->advice].advice == 1067 ufi->entry->advice); 1068 nback = MIN(uvmadvice[ufi->entry->advice].nback, 1069 (ufi->orig_rvaddr - ufi->entry->start) >> PAGE_SHIFT); 1070 flt->startva = ufi->orig_rvaddr - (nback << PAGE_SHIFT); 1071 /* 1072 * note: "-1" because we don't want to count the 1073 * faulting page as forw 1074 */ 1075 nforw = MIN(uvmadvice[ufi->entry->advice].nforw, 1076 ((ufi->entry->end - ufi->orig_rvaddr) >> 1077 PAGE_SHIFT) - 1); 1078 flt->npages = nback + nforw + 1; 1079 flt->centeridx = nback; 1080 1081 flt->narrow = true; /* ensure only once per-fault */ 1082 1083 } else { 1084 1085 /* narrow fault! */ 1086 nback = nforw = 0; 1087 flt->startva = ufi->orig_rvaddr; 1088 flt->npages = 1; 1089 flt->centeridx = 0; 1090 1091 } 1092 /* offset from entry's start to pgs' start */ 1093 const voff_t eoff = flt->startva - ufi->entry->start; 1094 1095 /* locked: maps(read) */ 1096 UVMHIST_LOG(maphist, " narrow=%jd, back=%jd, forw=%jd, startva=%#jx", 1097 flt->narrow, nback, nforw, flt->startva); 1098 UVMHIST_LOG(maphist, " entry=%#jx, amap=%#jx, obj=%#jx", 1099 (uintptr_t)ufi->entry, (uintptr_t)amap, (uintptr_t)uobj, 0); 1100 1101 /* 1102 * if we've got an amap, lock it and extract current anons. 1103 */ 1104 1105 if (amap) { 1106 amap_lock(amap, RW_WRITER); 1107 amap_lookups(&ufi->entry->aref, eoff, *ranons, flt->npages); 1108 } else { 1109 *ranons = NULL; /* to be safe */ 1110 } 1111 1112 /* locked: maps(read), amap(if there) */ 1113 KASSERT(amap == NULL || rw_write_held(amap->am_lock)); 1114 1115 /* 1116 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages 1117 * now and then forget about them (for the rest of the fault). 1118 */ 1119 1120 if (ufi->entry->advice == MADV_SEQUENTIAL && nback != 0) { 1121 1122 UVMHIST_LOG(maphist, " MADV_SEQUENTIAL: flushing backpages", 1123 0,0,0,0); 1124 /* flush back-page anons? */ 1125 if (amap) 1126 uvmfault_anonflush(*ranons, nback); 1127 1128 /* flush object? */ 1129 if (uobj) { 1130 voff_t uoff; 1131 1132 uoff = ufi->entry->offset + eoff; 1133 rw_enter(uobj->vmobjlock, RW_WRITER); 1134 (void) (uobj->pgops->pgo_put)(uobj, uoff, uoff + 1135 (nback << PAGE_SHIFT), PGO_DEACTIVATE); 1136 } 1137 1138 /* now forget about the backpages */ 1139 if (amap) 1140 *ranons += nback; 1141 flt->startva += (nback << PAGE_SHIFT); 1142 flt->npages -= nback; 1143 flt->centeridx = 0; 1144 } 1145 /* 1146 * => startva is fixed 1147 * => npages is fixed 1148 */ 1149 KASSERT(flt->startva <= ufi->orig_rvaddr); 1150 KASSERT(ufi->orig_rvaddr + ufi->orig_size <= 1151 flt->startva + (flt->npages << PAGE_SHIFT)); 1152 return 0; 1153 } 1154 1155 /* 1156 * uvm_fault_upper_lookup: look up existing h/w mapping and amap. 1157 * 1158 * iterate range of interest: 1159 * 1. check if h/w mapping exists. if yes, we don't care 1160 * 2. check if anon exists. if not, page is lower. 1161 * 3. if anon exists, enter h/w mapping for neighbors. 1162 * 1163 * => called with amap locked (if exists). 1164 */ 1165 1166 static int 1167 uvm_fault_upper_lookup( 1168 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 1169 struct vm_anon **anons, struct vm_page **pages) 1170 { 1171 struct vm_amap *amap = ufi->entry->aref.ar_amap; 1172 int lcv; 1173 vaddr_t currva; 1174 bool shadowed __unused; 1175 UVMHIST_FUNC("uvm_fault_upper_lookup"); UVMHIST_CALLED(maphist); 1176 1177 /* locked: maps(read), amap(if there) */ 1178 KASSERT(amap == NULL || rw_write_held(amap->am_lock)); 1179 1180 /* 1181 * map in the backpages and frontpages we found in the amap in hopes 1182 * of preventing future faults. we also init the pages[] array as 1183 * we go. 1184 */ 1185 1186 currva = flt->startva; 1187 shadowed = false; 1188 for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) { 1189 /* 1190 * don't play with VAs that are already mapped 1191 * (except for center) 1192 */ 1193 if (lcv != flt->centeridx && 1194 pmap_extract(ufi->orig_map->pmap, currva, NULL)) { 1195 pages[lcv] = PGO_DONTCARE; 1196 continue; 1197 } 1198 1199 /* 1200 * unmapped or center page. check if any anon at this level. 1201 */ 1202 if (amap == NULL || anons[lcv] == NULL) { 1203 pages[lcv] = NULL; 1204 continue; 1205 } 1206 1207 /* 1208 * check for present page and map if possible. re-activate it. 1209 */ 1210 1211 pages[lcv] = PGO_DONTCARE; 1212 if (lcv == flt->centeridx) { /* save center for later! */ 1213 shadowed = true; 1214 continue; 1215 } 1216 1217 struct vm_anon *anon = anons[lcv]; 1218 struct vm_page *pg = anon->an_page; 1219 1220 KASSERT(anon->an_lock == amap->am_lock); 1221 1222 /* Ignore loaned and busy pages. */ 1223 if (pg && pg->loan_count == 0 && (pg->flags & PG_BUSY) == 0) { 1224 uvm_fault_upper_neighbor(ufi, flt, currva, 1225 pg, anon->an_ref > 1); 1226 } 1227 } 1228 1229 /* locked: maps(read), amap(if there) */ 1230 KASSERT(amap == NULL || rw_write_held(amap->am_lock)); 1231 /* (shadowed == true) if there is an anon at the faulting address */ 1232 UVMHIST_LOG(maphist, " shadowed=%jd, will_get=%jd", shadowed, 1233 (ufi->entry->object.uvm_obj && shadowed != false),0,0); 1234 1235 /* 1236 * note that if we are really short of RAM we could sleep in the above 1237 * call to pmap_enter with everything locked. bad? 1238 * 1239 * XXX Actually, that is bad; pmap_enter() should just fail in that 1240 * XXX case. --thorpej 1241 */ 1242 1243 return 0; 1244 } 1245 1246 /* 1247 * uvm_fault_upper_neighbor: enter single upper neighbor page. 1248 * 1249 * => called with amap and anon locked. 1250 */ 1251 1252 static void 1253 uvm_fault_upper_neighbor( 1254 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 1255 vaddr_t currva, struct vm_page *pg, bool readonly) 1256 { 1257 UVMHIST_FUNC("uvm_fault_upper_neighbor"); UVMHIST_CALLED(maphist); 1258 1259 /* locked: amap, anon */ 1260 1261 KASSERT(pg->uobject == NULL); 1262 KASSERT(pg->uanon != NULL); 1263 KASSERT(rw_write_held(pg->uanon->an_lock)); 1264 KASSERT(uvm_pagegetdirty(pg) != UVM_PAGE_STATUS_CLEAN); 1265 1266 uvm_pagelock(pg); 1267 uvm_pageenqueue(pg); 1268 uvm_pageunlock(pg); 1269 UVMHIST_LOG(maphist, 1270 " MAPPING: n anon: pm=%#jx, va=%#jx, pg=%#jx", 1271 (uintptr_t)ufi->orig_map->pmap, currva, (uintptr_t)pg, 0); 1272 cpu_count(CPU_COUNT_FLTNAMAP, 1); 1273 1274 /* 1275 * Since this page isn't the page that's actually faulting, 1276 * ignore pmap_enter() failures; it's not critical that we 1277 * enter these right now. 1278 */ 1279 1280 (void) pmap_enter(ufi->orig_map->pmap, currva, 1281 VM_PAGE_TO_PHYS(pg), 1282 readonly ? (flt->enter_prot & ~VM_PROT_WRITE) : 1283 flt->enter_prot, 1284 PMAP_CANFAIL | (flt->wire_mapping ? PMAP_WIRED : 0)); 1285 1286 pmap_update(ufi->orig_map->pmap); 1287 } 1288 1289 /* 1290 * uvm_fault_upper: handle upper fault. 1291 * 1292 * 1. acquire anon lock. 1293 * 2. get anon. let uvmfault_anonget do the dirty work. 1294 * 3. handle loan. 1295 * 4. dispatch direct or promote handlers. 1296 */ 1297 1298 static int 1299 uvm_fault_upper( 1300 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1301 struct vm_anon **anons) 1302 { 1303 struct vm_amap * const amap = ufi->entry->aref.ar_amap; 1304 struct vm_anon * const anon = anons[flt->centeridx]; 1305 struct uvm_object *uobj; 1306 int error; 1307 UVMHIST_FUNC("uvm_fault_upper"); UVMHIST_CALLED(maphist); 1308 1309 /* locked: maps(read), amap, anon */ 1310 KASSERT(rw_write_held(amap->am_lock)); 1311 KASSERT(anon->an_lock == amap->am_lock); 1312 1313 /* 1314 * handle case 1: fault on an anon in our amap 1315 */ 1316 1317 UVMHIST_LOG(maphist, " case 1 fault: anon=%#jx", 1318 (uintptr_t)anon, 0, 0, 0); 1319 1320 /* 1321 * no matter if we have case 1A or case 1B we are going to need to 1322 * have the anon's memory resident. ensure that now. 1323 */ 1324 1325 /* 1326 * let uvmfault_anonget do the dirty work. 1327 * if it fails (!OK) it will unlock everything for us. 1328 * if it succeeds, locks are still valid and locked. 1329 * also, if it is OK, then the anon's page is on the queues. 1330 * if the page is on loan from a uvm_object, then anonget will 1331 * lock that object for us if it does not fail. 1332 */ 1333 1334 error = uvmfault_anonget(ufi, amap, anon); 1335 switch (error) { 1336 case 0: 1337 break; 1338 1339 case ERESTART: 1340 return ERESTART; 1341 1342 case EAGAIN: 1343 kpause("fltagain1", false, hz/2, NULL); 1344 return ERESTART; 1345 1346 default: 1347 return error; 1348 } 1349 1350 /* 1351 * uobj is non null if the page is on loan from an object (i.e. uobj) 1352 */ 1353 1354 uobj = anon->an_page->uobject; /* locked by anonget if !NULL */ 1355 1356 /* locked: maps(read), amap, anon, uobj(if one) */ 1357 KASSERT(rw_write_held(amap->am_lock)); 1358 KASSERT(anon->an_lock == amap->am_lock); 1359 KASSERT(uobj == NULL || rw_write_held(uobj->vmobjlock)); 1360 1361 /* 1362 * special handling for loaned pages 1363 */ 1364 1365 if (anon->an_page->loan_count) { 1366 error = uvm_fault_upper_loan(ufi, flt, anon, &uobj); 1367 if (error != 0) 1368 return error; 1369 } 1370 1371 /* 1372 * if we are case 1B then we will need to allocate a new blank 1373 * anon to transfer the data into. note that we have a lock 1374 * on anon, so no one can busy or release the page until we are done. 1375 * also note that the ref count can't drop to zero here because 1376 * it is > 1 and we are only dropping one ref. 1377 * 1378 * in the (hopefully very rare) case that we are out of RAM we 1379 * will unlock, wait for more RAM, and refault. 1380 * 1381 * if we are out of anon VM we kill the process (XXX: could wait?). 1382 */ 1383 1384 if (flt->cow_now && anon->an_ref > 1) { 1385 flt->promote = true; 1386 error = uvm_fault_upper_promote(ufi, flt, uobj, anon); 1387 } else { 1388 error = uvm_fault_upper_direct(ufi, flt, uobj, anon); 1389 } 1390 return error; 1391 } 1392 1393 /* 1394 * uvm_fault_upper_loan: handle loaned upper page. 1395 * 1396 * 1. if not cow'ing now, simply adjust flt->enter_prot. 1397 * 2. if cow'ing now, and if ref count is 1, break loan. 1398 */ 1399 1400 static int 1401 uvm_fault_upper_loan( 1402 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1403 struct vm_anon *anon, struct uvm_object **ruobj) 1404 { 1405 struct vm_amap * const amap = ufi->entry->aref.ar_amap; 1406 int error = 0; 1407 UVMHIST_FUNC("uvm_fault_upper_loan"); UVMHIST_CALLED(maphist); 1408 1409 if (!flt->cow_now) { 1410 1411 /* 1412 * for read faults on loaned pages we just cap the 1413 * protection at read-only. 1414 */ 1415 1416 flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE; 1417 1418 } else { 1419 /* 1420 * note that we can't allow writes into a loaned page! 1421 * 1422 * if we have a write fault on a loaned page in an 1423 * anon then we need to look at the anon's ref count. 1424 * if it is greater than one then we are going to do 1425 * a normal copy-on-write fault into a new anon (this 1426 * is not a problem). however, if the reference count 1427 * is one (a case where we would normally allow a 1428 * write directly to the page) then we need to kill 1429 * the loan before we continue. 1430 */ 1431 1432 /* >1 case is already ok */ 1433 if (anon->an_ref == 1) { 1434 error = uvm_loanbreak_anon(anon, *ruobj); 1435 if (error != 0) { 1436 uvmfault_unlockall(ufi, amap, *ruobj); 1437 uvm_wait("flt_noram2"); 1438 return ERESTART; 1439 } 1440 /* if we were a loan receiver uobj is gone */ 1441 if (*ruobj) 1442 *ruobj = NULL; 1443 } 1444 } 1445 return error; 1446 } 1447 1448 /* 1449 * uvm_fault_upper_promote: promote upper page. 1450 * 1451 * 1. call uvmfault_promote. 1452 * 2. enqueue page. 1453 * 3. deref. 1454 * 4. pass page to uvm_fault_upper_enter. 1455 */ 1456 1457 static int 1458 uvm_fault_upper_promote( 1459 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1460 struct uvm_object *uobj, struct vm_anon *anon) 1461 { 1462 struct vm_anon * const oanon = anon; 1463 struct vm_page *pg; 1464 int error; 1465 UVMHIST_FUNC("uvm_fault_upper_promote"); UVMHIST_CALLED(maphist); 1466 1467 UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0); 1468 cpu_count(CPU_COUNT_FLT_ACOW, 1); 1469 1470 error = uvmfault_promote(ufi, oanon, PGO_DONTCARE, &anon, 1471 &flt->anon_spare); 1472 switch (error) { 1473 case 0: 1474 break; 1475 case ERESTART: 1476 return ERESTART; 1477 default: 1478 return error; 1479 } 1480 1481 KASSERT(anon == NULL || anon->an_lock == oanon->an_lock); 1482 1483 pg = anon->an_page; 1484 /* uvm_fault_upper_done will activate the page */ 1485 uvm_pagelock(pg); 1486 uvm_pageenqueue(pg); 1487 uvm_pageunlock(pg); 1488 pg->flags &= ~(PG_BUSY|PG_FAKE); 1489 UVM_PAGE_OWN(pg, NULL); 1490 1491 /* deref: can not drop to zero here by defn! */ 1492 KASSERT(oanon->an_ref > 1); 1493 oanon->an_ref--; 1494 1495 /* 1496 * note: oanon is still locked, as is the new anon. we 1497 * need to check for this later when we unlock oanon; if 1498 * oanon != anon, we'll have to unlock anon, too. 1499 */ 1500 1501 return uvm_fault_upper_enter(ufi, flt, uobj, anon, pg, oanon); 1502 } 1503 1504 /* 1505 * uvm_fault_upper_direct: handle direct fault. 1506 */ 1507 1508 static int 1509 uvm_fault_upper_direct( 1510 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1511 struct uvm_object *uobj, struct vm_anon *anon) 1512 { 1513 struct vm_anon * const oanon = anon; 1514 struct vm_page *pg; 1515 UVMHIST_FUNC("uvm_fault_upper_direct"); UVMHIST_CALLED(maphist); 1516 1517 cpu_count(CPU_COUNT_FLT_ANON, 1); 1518 pg = anon->an_page; 1519 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */ 1520 flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE; 1521 1522 return uvm_fault_upper_enter(ufi, flt, uobj, anon, pg, oanon); 1523 } 1524 1525 /* 1526 * uvm_fault_upper_enter: enter h/w mapping of upper page. 1527 */ 1528 1529 static int 1530 uvm_fault_upper_enter( 1531 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 1532 struct uvm_object *uobj, struct vm_anon *anon, struct vm_page *pg, 1533 struct vm_anon *oanon) 1534 { 1535 struct pmap *pmap = ufi->orig_map->pmap; 1536 vaddr_t va = ufi->orig_rvaddr; 1537 struct vm_amap * const amap = ufi->entry->aref.ar_amap; 1538 UVMHIST_FUNC("uvm_fault_upper_enter"); UVMHIST_CALLED(maphist); 1539 1540 /* locked: maps(read), amap, oanon, anon(if different from oanon) */ 1541 KASSERT(rw_write_held(amap->am_lock)); 1542 KASSERT(anon->an_lock == amap->am_lock); 1543 KASSERT(oanon->an_lock == amap->am_lock); 1544 KASSERT(uobj == NULL || rw_write_held(uobj->vmobjlock)); 1545 KASSERT(uvm_pagegetdirty(pg) != UVM_PAGE_STATUS_CLEAN); 1546 1547 /* 1548 * now map the page in. 1549 */ 1550 1551 UVMHIST_LOG(maphist, 1552 " MAPPING: anon: pm=%#jx, va=%#jx, pg=%#jx, promote=%jd", 1553 (uintptr_t)pmap, va, (uintptr_t)pg, flt->promote); 1554 if (pmap_enter(pmap, va, VM_PAGE_TO_PHYS(pg), 1555 flt->enter_prot, flt->access_type | PMAP_CANFAIL | 1556 (flt->wire_mapping ? PMAP_WIRED : 0)) != 0) { 1557 1558 /* 1559 * If pmap_enter() fails, it must not leave behind an existing 1560 * pmap entry. In particular, a now-stale entry for a different 1561 * page would leave the pmap inconsistent with the vm_map. 1562 * This is not to imply that pmap_enter() should remove an 1563 * existing mapping in such a situation (since that could create 1564 * different problems, eg. if the existing mapping is wired), 1565 * but rather that the pmap should be designed such that it 1566 * never needs to fail when the new mapping is replacing an 1567 * existing mapping and the new page has no existing mappings. 1568 */ 1569 1570 KASSERT(!pmap_extract(pmap, va, NULL)); 1571 1572 /* 1573 * No need to undo what we did; we can simply think of 1574 * this as the pmap throwing away the mapping information. 1575 * 1576 * We do, however, have to go through the ReFault path, 1577 * as the map may change while we're asleep. 1578 */ 1579 1580 uvmfault_unlockall(ufi, amap, uobj); 1581 if (!uvm_reclaimable()) { 1582 UVMHIST_LOG(maphist, 1583 "<- failed. out of VM",0,0,0,0); 1584 /* XXX instrumentation */ 1585 return ENOMEM; 1586 } 1587 /* XXX instrumentation */ 1588 uvm_wait("flt_pmfail1"); 1589 return ERESTART; 1590 } 1591 1592 uvm_fault_upper_done(ufi, flt, anon, pg); 1593 1594 /* 1595 * done case 1! finish up by unlocking everything and returning success 1596 */ 1597 1598 pmap_update(pmap); 1599 uvmfault_unlockall(ufi, amap, uobj); 1600 return 0; 1601 } 1602 1603 /* 1604 * uvm_fault_upper_done: queue upper center page. 1605 */ 1606 1607 static void 1608 uvm_fault_upper_done( 1609 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 1610 struct vm_anon *anon, struct vm_page *pg) 1611 { 1612 const bool wire_paging = flt->wire_paging; 1613 1614 UVMHIST_FUNC("uvm_fault_upper_done"); UVMHIST_CALLED(maphist); 1615 1616 /* 1617 * ... update the page queues. 1618 */ 1619 1620 uvm_pagelock(pg); 1621 if (wire_paging) { 1622 uvm_pagewire(pg); 1623 } else { 1624 uvm_pageactivate(pg); 1625 } 1626 uvm_pageunlock(pg); 1627 1628 if (wire_paging) { 1629 /* 1630 * since the now-wired page cannot be paged out, 1631 * release its swap resources for others to use. 1632 * and since an anon with no swap cannot be clean, 1633 * mark it dirty now. 1634 */ 1635 1636 uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY); 1637 uvm_anon_dropswap(anon); 1638 } 1639 } 1640 1641 /* 1642 * uvm_fault_lower: handle lower fault. 1643 * 1644 * 1. check uobj 1645 * 1.1. if null, ZFOD. 1646 * 1.2. if not null, look up unnmapped neighbor pages. 1647 * 2. for center page, check if promote. 1648 * 2.1. ZFOD always needs promotion. 1649 * 2.2. other uobjs, when entry is marked COW (usually MAP_PRIVATE vnode). 1650 * 3. if uobj is not ZFOD and page is not found, do i/o. 1651 * 4. dispatch either direct / promote fault. 1652 */ 1653 1654 static int 1655 uvm_fault_lower( 1656 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1657 struct vm_page **pages) 1658 { 1659 struct vm_amap *amap __diagused = ufi->entry->aref.ar_amap; 1660 struct uvm_object *uobj = ufi->entry->object.uvm_obj; 1661 struct vm_page *uobjpage; 1662 int error; 1663 UVMHIST_FUNC("uvm_fault_lower"); UVMHIST_CALLED(maphist); 1664 1665 /* 1666 * now, if the desired page is not shadowed by the amap and we have 1667 * a backing object that does not have a special fault routine, then 1668 * we ask (with pgo_get) the object for resident pages that we care 1669 * about and attempt to map them in. we do not let pgo_get block 1670 * (PGO_LOCKED). 1671 */ 1672 1673 if (uobj == NULL) { 1674 /* zero fill; don't care neighbor pages */ 1675 uobjpage = NULL; 1676 } else { 1677 uvm_fault_lower_lookup(ufi, flt, pages); 1678 uobjpage = pages[flt->centeridx]; 1679 } 1680 1681 /* 1682 * note that at this point we are done with any front or back pages. 1683 * we are now going to focus on the center page (i.e. the one we've 1684 * faulted on). if we have faulted on the upper (anon) layer 1685 * [i.e. case 1], then the anon we want is anons[centeridx] (we have 1686 * not touched it yet). if we have faulted on the bottom (uobj) 1687 * layer [i.e. case 2] and the page was both present and available, 1688 * then we've got a pointer to it as "uobjpage" and we've already 1689 * made it BUSY. 1690 */ 1691 1692 /* 1693 * locked: 1694 * maps(read), amap(if there), uobj(if !null), uobjpage(if !null) 1695 */ 1696 KASSERT(amap == NULL || rw_write_held(amap->am_lock)); 1697 KASSERT(uobj == NULL || rw_write_held(uobj->vmobjlock)); 1698 KASSERT(uobjpage == NULL || (uobjpage->flags & PG_BUSY) != 0); 1699 1700 /* 1701 * note that uobjpage can not be PGO_DONTCARE at this point. we now 1702 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we 1703 * have a backing object, check and see if we are going to promote 1704 * the data up to an anon during the fault. 1705 */ 1706 1707 if (uobj == NULL) { 1708 uobjpage = PGO_DONTCARE; 1709 flt->promote = true; /* always need anon here */ 1710 } else { 1711 KASSERT(uobjpage != PGO_DONTCARE); 1712 flt->promote = flt->cow_now && UVM_ET_ISCOPYONWRITE(ufi->entry); 1713 } 1714 UVMHIST_LOG(maphist, " case 2 fault: promote=%jd, zfill=%jd", 1715 flt->promote, (uobj == NULL), 0,0); 1716 1717 /* 1718 * if uobjpage is not null then we do not need to do I/O to get the 1719 * uobjpage. 1720 * 1721 * if uobjpage is null, then we need to unlock and ask the pager to 1722 * get the data for us. once we have the data, we need to reverify 1723 * the state the world. we are currently not holding any resources. 1724 */ 1725 1726 if (uobjpage) { 1727 /* update rusage counters */ 1728 curlwp->l_ru.ru_minflt++; 1729 } else { 1730 error = uvm_fault_lower_io(ufi, flt, &uobj, &uobjpage); 1731 if (error != 0) 1732 return error; 1733 } 1734 1735 /* 1736 * locked: 1737 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj) 1738 */ 1739 KASSERT(amap == NULL || rw_write_held(amap->am_lock)); 1740 KASSERT(uobj == NULL || rw_write_held(uobj->vmobjlock)); 1741 KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0); 1742 1743 /* 1744 * notes: 1745 * - at this point uobjpage can not be NULL 1746 * - at this point uobjpage can not be PG_RELEASED (since we checked 1747 * for it above) 1748 * - at this point uobjpage could be PG_WANTED (handle later) 1749 */ 1750 1751 KASSERT(uobjpage != NULL); 1752 KASSERT(uobj == NULL || uobj == uobjpage->uobject); 1753 KASSERT(uobj == NULL || !UVM_OBJ_IS_CLEAN(uobjpage->uobject) || 1754 uvm_pagegetdirty(uobjpage) == UVM_PAGE_STATUS_CLEAN); 1755 1756 if (!flt->promote) { 1757 error = uvm_fault_lower_direct(ufi, flt, uobj, uobjpage); 1758 } else { 1759 error = uvm_fault_lower_promote(ufi, flt, uobj, uobjpage); 1760 } 1761 return error; 1762 } 1763 1764 /* 1765 * uvm_fault_lower_lookup: look up on-memory uobj pages. 1766 * 1767 * 1. get on-memory pages. 1768 * 2. if failed, give up (get only center page later). 1769 * 3. if succeeded, enter h/w mapping of neighbor pages. 1770 */ 1771 1772 static void 1773 uvm_fault_lower_lookup( 1774 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 1775 struct vm_page **pages) 1776 { 1777 struct uvm_object *uobj = ufi->entry->object.uvm_obj; 1778 int lcv, gotpages; 1779 vaddr_t currva; 1780 UVMHIST_FUNC("uvm_fault_lower_lookup"); UVMHIST_CALLED(maphist); 1781 1782 rw_enter(uobj->vmobjlock, RW_WRITER); 1783 /* Locked: maps(read), amap(if there), uobj */ 1784 1785 cpu_count(CPU_COUNT_FLTLGET, 1); 1786 gotpages = flt->npages; 1787 (void) uobj->pgops->pgo_get(uobj, 1788 ufi->entry->offset + flt->startva - ufi->entry->start, 1789 pages, &gotpages, flt->centeridx, 1790 flt->access_type & MASK(ufi->entry), ufi->entry->advice, PGO_LOCKED); 1791 1792 KASSERT(rw_write_held(uobj->vmobjlock)); 1793 1794 /* 1795 * check for pages to map, if we got any 1796 */ 1797 1798 if (gotpages == 0) { 1799 pages[flt->centeridx] = NULL; 1800 return; 1801 } 1802 1803 currva = flt->startva; 1804 for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) { 1805 struct vm_page *curpg; 1806 1807 curpg = pages[lcv]; 1808 if (curpg == NULL || curpg == PGO_DONTCARE) { 1809 continue; 1810 } 1811 KASSERT(curpg->uobject == uobj); 1812 1813 /* 1814 * if center page is resident and not PG_BUSY|PG_RELEASED 1815 * then pgo_get made it PG_BUSY for us and gave us a handle 1816 * to it. 1817 */ 1818 1819 if (lcv == flt->centeridx) { 1820 UVMHIST_LOG(maphist, " got uobjpage (%#jx) " 1821 "with locked get", (uintptr_t)curpg, 0, 0, 0); 1822 } else { 1823 uvm_fault_lower_neighbor(ufi, flt, currva, curpg); 1824 } 1825 } 1826 pmap_update(ufi->orig_map->pmap); 1827 } 1828 1829 /* 1830 * uvm_fault_lower_neighbor: enter h/w mapping of lower neighbor page. 1831 */ 1832 1833 static void 1834 uvm_fault_lower_neighbor( 1835 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 1836 vaddr_t currva, struct vm_page *pg) 1837 { 1838 const bool readonly = uvm_pagereadonly_p(pg) || pg->loan_count > 0; 1839 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 1840 1841 /* locked: maps(read), amap(if there), uobj */ 1842 1843 /* 1844 * calling pgo_get with PGO_LOCKED returns us pages which 1845 * are neither busy nor released, so we don't need to check 1846 * for this. we can just directly enter the pages. 1847 */ 1848 1849 uvm_pagelock(pg); 1850 uvm_pageenqueue(pg); 1851 uvm_pageunlock(pg); 1852 UVMHIST_LOG(maphist, 1853 " MAPPING: n obj: pm=%#jx, va=%#jx, pg=%#jx", 1854 (uintptr_t)ufi->orig_map->pmap, currva, (uintptr_t)pg, 0); 1855 cpu_count(CPU_COUNT_FLTNOMAP, 1); 1856 1857 /* 1858 * Since this page isn't the page that's actually faulting, 1859 * ignore pmap_enter() failures; it's not critical that we 1860 * enter these right now. 1861 * NOTE: page can't be PG_WANTED or PG_RELEASED because we've 1862 * held the lock the whole time we've had the handle. 1863 */ 1864 KASSERT((pg->flags & PG_PAGEOUT) == 0); 1865 KASSERT((pg->flags & PG_RELEASED) == 0); 1866 KASSERT((pg->flags & PG_WANTED) == 0); 1867 KASSERT(!UVM_OBJ_IS_CLEAN(pg->uobject) || 1868 uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_CLEAN); 1869 pg->flags &= ~(PG_BUSY); 1870 UVM_PAGE_OWN(pg, NULL); 1871 1872 KASSERT(rw_write_held(pg->uobject->vmobjlock)); 1873 1874 const vm_prot_t mapprot = 1875 readonly ? (flt->enter_prot & ~VM_PROT_WRITE) : 1876 flt->enter_prot & MASK(ufi->entry); 1877 const u_int mapflags = 1878 PMAP_CANFAIL | (flt->wire_mapping ? (mapprot | PMAP_WIRED) : 0); 1879 (void) pmap_enter(ufi->orig_map->pmap, currva, 1880 VM_PAGE_TO_PHYS(pg), mapprot, mapflags); 1881 } 1882 1883 /* 1884 * uvm_fault_lower_io: get lower page from backing store. 1885 * 1886 * 1. unlock everything, because i/o will block. 1887 * 2. call pgo_get. 1888 * 3. if failed, recover. 1889 * 4. if succeeded, relock everything and verify things. 1890 */ 1891 1892 static int 1893 uvm_fault_lower_io( 1894 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 1895 struct uvm_object **ruobj, struct vm_page **ruobjpage) 1896 { 1897 struct vm_amap * const amap = ufi->entry->aref.ar_amap; 1898 struct uvm_object *uobj = *ruobj; 1899 struct vm_page *pg; 1900 bool locked; 1901 int gotpages; 1902 int error; 1903 voff_t uoff; 1904 vm_prot_t access_type; 1905 int advice; 1906 UVMHIST_FUNC("uvm_fault_lower_io"); UVMHIST_CALLED(maphist); 1907 1908 /* update rusage counters */ 1909 curlwp->l_ru.ru_majflt++; 1910 1911 /* grab everything we need from the entry before we unlock */ 1912 uoff = (ufi->orig_rvaddr - ufi->entry->start) + ufi->entry->offset; 1913 access_type = flt->access_type & MASK(ufi->entry); 1914 advice = ufi->entry->advice; 1915 1916 /* Locked: maps(read), amap(if there), uobj */ 1917 uvmfault_unlockall(ufi, amap, NULL); 1918 1919 /* Locked: uobj */ 1920 KASSERT(uobj == NULL || rw_write_held(uobj->vmobjlock)); 1921 1922 cpu_count(CPU_COUNT_FLTGET, 1); 1923 gotpages = 1; 1924 pg = NULL; 1925 error = uobj->pgops->pgo_get(uobj, uoff, &pg, &gotpages, 1926 0, access_type, advice, PGO_SYNCIO); 1927 /* locked: pg(if no error) */ 1928 1929 /* 1930 * recover from I/O 1931 */ 1932 1933 if (error) { 1934 if (error == EAGAIN) { 1935 UVMHIST_LOG(maphist, 1936 " pgo_get says TRY AGAIN!",0,0,0,0); 1937 kpause("fltagain2", false, hz/2, NULL); 1938 return ERESTART; 1939 } 1940 1941 #if 0 1942 KASSERT(error != ERESTART); 1943 #else 1944 /* XXXUEBS don't re-fault? */ 1945 if (error == ERESTART) 1946 error = EIO; 1947 #endif 1948 1949 UVMHIST_LOG(maphist, "<- pgo_get failed (code %jd)", 1950 error, 0,0,0); 1951 return error; 1952 } 1953 1954 /* 1955 * re-verify the state of the world by first trying to relock 1956 * the maps. always relock the object. 1957 */ 1958 1959 locked = uvmfault_relock(ufi); 1960 if (locked && amap) 1961 amap_lock(amap, RW_WRITER); 1962 1963 /* might be changed */ 1964 uobj = pg->uobject; 1965 1966 rw_enter(uobj->vmobjlock, RW_WRITER); 1967 KASSERT((pg->flags & PG_BUSY) != 0); 1968 1969 uvm_pagelock(pg); 1970 uvm_pageactivate(pg); 1971 uvm_pageunlock(pg); 1972 1973 /* locked(locked): maps(read), amap(if !null), uobj, pg */ 1974 /* locked(!locked): uobj, pg */ 1975 1976 /* 1977 * verify that the page has not be released and re-verify 1978 * that amap slot is still free. if there is a problem, 1979 * we unlock and clean up. 1980 */ 1981 1982 if ((pg->flags & PG_RELEASED) != 0 || 1983 (locked && amap && amap_lookup(&ufi->entry->aref, 1984 ufi->orig_rvaddr - ufi->entry->start))) { 1985 if (locked) 1986 uvmfault_unlockall(ufi, amap, NULL); 1987 locked = false; 1988 } 1989 1990 /* 1991 * didn't get the lock? release the page and retry. 1992 */ 1993 1994 if (locked == false) { 1995 UVMHIST_LOG(maphist, 1996 " wasn't able to relock after fault: retry", 1997 0,0,0,0); 1998 if (pg->flags & PG_WANTED) { 1999 wakeup(pg); 2000 } 2001 if ((pg->flags & PG_RELEASED) == 0) { 2002 pg->flags &= ~(PG_BUSY | PG_WANTED); 2003 UVM_PAGE_OWN(pg, NULL); 2004 } else { 2005 cpu_count(CPU_COUNT_FLTPGRELE, 1); 2006 uvm_pagefree(pg); 2007 } 2008 rw_exit(uobj->vmobjlock); 2009 return ERESTART; 2010 } 2011 2012 /* 2013 * we have the data in pg which is busy and 2014 * not released. we are holding object lock (so the page 2015 * can't be released on us). 2016 */ 2017 2018 /* locked: maps(read), amap(if !null), uobj, pg */ 2019 2020 *ruobj = uobj; 2021 *ruobjpage = pg; 2022 return 0; 2023 } 2024 2025 /* 2026 * uvm_fault_lower_direct: fault lower center page 2027 * 2028 * 1. adjust flt->enter_prot. 2029 * 2. if page is loaned, resolve. 2030 */ 2031 2032 int 2033 uvm_fault_lower_direct( 2034 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 2035 struct uvm_object *uobj, struct vm_page *uobjpage) 2036 { 2037 struct vm_page *pg; 2038 UVMHIST_FUNC("uvm_fault_lower_direct"); UVMHIST_CALLED(maphist); 2039 2040 /* 2041 * we are not promoting. if the mapping is COW ensure that we 2042 * don't give more access than we should (e.g. when doing a read 2043 * fault on a COPYONWRITE mapping we want to map the COW page in 2044 * R/O even though the entry protection could be R/W). 2045 * 2046 * set "pg" to the page we want to map in (uobjpage, usually) 2047 */ 2048 2049 cpu_count(CPU_COUNT_FLT_OBJ, 1); 2050 if (UVM_ET_ISCOPYONWRITE(ufi->entry) || 2051 UVM_OBJ_NEEDS_WRITEFAULT(uobjpage->uobject)) 2052 flt->enter_prot &= ~VM_PROT_WRITE; 2053 pg = uobjpage; /* map in the actual object */ 2054 2055 KASSERT(uobjpage != PGO_DONTCARE); 2056 2057 /* 2058 * we are faulting directly on the page. be careful 2059 * about writing to loaned pages... 2060 */ 2061 2062 if (uobjpage->loan_count) { 2063 uvm_fault_lower_direct_loan(ufi, flt, uobj, &pg, &uobjpage); 2064 } 2065 KASSERT(pg == uobjpage); 2066 2067 KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0); 2068 return uvm_fault_lower_enter(ufi, flt, uobj, NULL, pg); 2069 } 2070 2071 /* 2072 * uvm_fault_lower_direct_loan: resolve loaned page. 2073 * 2074 * 1. if not cow'ing, adjust flt->enter_prot. 2075 * 2. if cow'ing, break loan. 2076 */ 2077 2078 static int 2079 uvm_fault_lower_direct_loan( 2080 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 2081 struct uvm_object *uobj, struct vm_page **rpg, 2082 struct vm_page **ruobjpage) 2083 { 2084 struct vm_amap * const amap = ufi->entry->aref.ar_amap; 2085 struct vm_page *pg; 2086 struct vm_page *uobjpage = *ruobjpage; 2087 UVMHIST_FUNC("uvm_fault_lower_direct_loan"); UVMHIST_CALLED(maphist); 2088 2089 if (!flt->cow_now) { 2090 /* read fault: cap the protection at readonly */ 2091 /* cap! */ 2092 flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE; 2093 } else { 2094 /* write fault: must break the loan here */ 2095 2096 pg = uvm_loanbreak(uobjpage); 2097 if (pg == NULL) { 2098 2099 /* 2100 * drop ownership of page, it can't be released 2101 */ 2102 2103 if (uobjpage->flags & PG_WANTED) 2104 wakeup(uobjpage); 2105 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 2106 UVM_PAGE_OWN(uobjpage, NULL); 2107 2108 uvmfault_unlockall(ufi, amap, uobj); 2109 UVMHIST_LOG(maphist, 2110 " out of RAM breaking loan, waiting", 2111 0,0,0,0); 2112 cpu_count(CPU_COUNT_FLTNORAM, 1); 2113 uvm_wait("flt_noram4"); 2114 return ERESTART; 2115 } 2116 *rpg = pg; 2117 *ruobjpage = pg; 2118 } 2119 return 0; 2120 } 2121 2122 /* 2123 * uvm_fault_lower_promote: promote lower page. 2124 * 2125 * 1. call uvmfault_promote. 2126 * 2. fill in data. 2127 * 3. if not ZFOD, dispose old page. 2128 */ 2129 2130 int 2131 uvm_fault_lower_promote( 2132 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 2133 struct uvm_object *uobj, struct vm_page *uobjpage) 2134 { 2135 struct vm_amap * const amap = ufi->entry->aref.ar_amap; 2136 struct vm_anon *anon; 2137 struct vm_page *pg; 2138 int error; 2139 UVMHIST_FUNC("uvm_fault_lower_promote"); UVMHIST_CALLED(maphist); 2140 2141 KASSERT(amap != NULL); 2142 2143 /* 2144 * If we are going to promote the data to an anon we 2145 * allocate a blank anon here and plug it into our amap. 2146 */ 2147 error = uvmfault_promote(ufi, NULL, uobjpage, 2148 &anon, &flt->anon_spare); 2149 switch (error) { 2150 case 0: 2151 break; 2152 case ERESTART: 2153 return ERESTART; 2154 default: 2155 return error; 2156 } 2157 2158 pg = anon->an_page; 2159 2160 /* 2161 * Fill in the data. 2162 */ 2163 KASSERT(uobj == NULL || (uobjpage->flags & PG_BUSY) != 0); 2164 2165 if (uobjpage != PGO_DONTCARE) { 2166 cpu_count(CPU_COUNT_FLT_PRCOPY, 1); 2167 2168 /* 2169 * promote to shared amap? make sure all sharing 2170 * procs see it 2171 */ 2172 2173 if ((amap_flags(amap) & AMAP_SHARED) != 0) { 2174 pmap_page_protect(uobjpage, VM_PROT_NONE); 2175 /* 2176 * XXX: PAGE MIGHT BE WIRED! 2177 */ 2178 } 2179 2180 /* 2181 * dispose of uobjpage. it can't be PG_RELEASED 2182 * since we still hold the object lock. 2183 */ 2184 2185 if (uobjpage->flags & PG_WANTED) { 2186 /* still have the obj lock */ 2187 wakeup(uobjpage); 2188 } 2189 uobjpage->flags &= ~(PG_BUSY|PG_WANTED); 2190 UVM_PAGE_OWN(uobjpage, NULL); 2191 2192 UVMHIST_LOG(maphist, 2193 " promote uobjpage %#jx to anon/page %#jx/%#jx", 2194 (uintptr_t)uobjpage, (uintptr_t)anon, (uintptr_t)pg, 0); 2195 2196 } else { 2197 cpu_count(CPU_COUNT_FLT_PRZERO, 1); 2198 2199 /* 2200 * Page is zero'd and marked dirty by 2201 * uvmfault_promote(). 2202 */ 2203 2204 UVMHIST_LOG(maphist," zero fill anon/page %#jx/%#jx", 2205 (uintptr_t)anon, (uintptr_t)pg, 0, 0); 2206 } 2207 2208 return uvm_fault_lower_enter(ufi, flt, uobj, anon, pg); 2209 } 2210 2211 /* 2212 * uvm_fault_lower_enter: enter h/w mapping of lower page or anon page promoted 2213 * from the lower page. 2214 */ 2215 2216 int 2217 uvm_fault_lower_enter( 2218 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 2219 struct uvm_object *uobj, 2220 struct vm_anon *anon, struct vm_page *pg) 2221 { 2222 struct vm_amap * const amap = ufi->entry->aref.ar_amap; 2223 const bool readonly = uvm_pagereadonly_p(pg); 2224 int error; 2225 UVMHIST_FUNC("uvm_fault_lower_enter"); UVMHIST_CALLED(maphist); 2226 2227 /* 2228 * Locked: 2229 * 2230 * maps(read), amap(if !null), uobj(if !null), 2231 * anon(if !null), pg(if anon), unlock_uobj(if !null) 2232 * 2233 * Note: pg is either the uobjpage or the new page in the new anon. 2234 */ 2235 KASSERT(amap == NULL || rw_write_held(amap->am_lock)); 2236 KASSERT(uobj == NULL || rw_write_held(uobj->vmobjlock)); 2237 KASSERT(anon == NULL || anon->an_lock == amap->am_lock); 2238 KASSERT((pg->flags & PG_BUSY) != 0); 2239 2240 /* 2241 * all resources are present. we can now map it in and free our 2242 * resources. 2243 */ 2244 2245 UVMHIST_LOG(maphist, 2246 " MAPPING: case2: pm=%#jx, va=%#jx, pg=%#jx, promote=%jd", 2247 (uintptr_t)ufi->orig_map->pmap, ufi->orig_rvaddr, 2248 (uintptr_t)pg, flt->promote); 2249 KASSERTMSG((flt->access_type & VM_PROT_WRITE) == 0 || !readonly, 2250 "promote=%u cow_now=%u access_type=%x enter_prot=%x cow=%u " 2251 "entry=%p map=%p orig_rvaddr=%p pg=%p", 2252 flt->promote, flt->cow_now, flt->access_type, flt->enter_prot, 2253 UVM_ET_ISCOPYONWRITE(ufi->entry), ufi->entry, ufi->orig_map, 2254 (void *)ufi->orig_rvaddr, pg); 2255 KASSERT((flt->access_type & VM_PROT_WRITE) == 0 || !readonly); 2256 if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr, 2257 VM_PAGE_TO_PHYS(pg), 2258 readonly ? flt->enter_prot & ~VM_PROT_WRITE : flt->enter_prot, 2259 flt->access_type | PMAP_CANFAIL | 2260 (flt->wire_mapping ? PMAP_WIRED : 0)) != 0) { 2261 2262 /* 2263 * No need to undo what we did; we can simply think of 2264 * this as the pmap throwing away the mapping information. 2265 * 2266 * We do, however, have to go through the ReFault path, 2267 * as the map may change while we're asleep. 2268 */ 2269 2270 /* 2271 * ensure that the page is queued in the case that 2272 * we just promoted the page. 2273 */ 2274 2275 uvm_pagelock(pg); 2276 uvm_pageenqueue(pg); 2277 uvm_pageunlock(pg); 2278 2279 if (pg->flags & PG_WANTED) 2280 wakeup(pg); 2281 2282 /* 2283 * note that pg can't be PG_RELEASED since we did not drop 2284 * the object lock since the last time we checked. 2285 */ 2286 KASSERT((pg->flags & PG_RELEASED) == 0); 2287 2288 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); 2289 UVM_PAGE_OWN(pg, NULL); 2290 2291 uvmfault_unlockall(ufi, amap, uobj); 2292 if (!uvm_reclaimable()) { 2293 UVMHIST_LOG(maphist, 2294 "<- failed. out of VM",0,0,0,0); 2295 /* XXX instrumentation */ 2296 error = ENOMEM; 2297 return error; 2298 } 2299 /* XXX instrumentation */ 2300 uvm_wait("flt_pmfail2"); 2301 return ERESTART; 2302 } 2303 2304 uvm_fault_lower_done(ufi, flt, uobj, pg); 2305 2306 /* 2307 * note that pg can't be PG_RELEASED since we did not drop the object 2308 * lock since the last time we checked. 2309 */ 2310 KASSERT((pg->flags & PG_RELEASED) == 0); 2311 if (pg->flags & PG_WANTED) 2312 wakeup(pg); 2313 pg->flags &= ~(PG_BUSY|PG_FAKE|PG_WANTED); 2314 UVM_PAGE_OWN(pg, NULL); 2315 2316 pmap_update(ufi->orig_map->pmap); 2317 uvmfault_unlockall(ufi, amap, uobj); 2318 2319 UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0); 2320 return 0; 2321 } 2322 2323 /* 2324 * uvm_fault_lower_done: queue lower center page. 2325 */ 2326 2327 void 2328 uvm_fault_lower_done( 2329 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 2330 struct uvm_object *uobj, struct vm_page *pg) 2331 { 2332 bool dropswap = false; 2333 2334 UVMHIST_FUNC("uvm_fault_lower_done"); UVMHIST_CALLED(maphist); 2335 2336 uvm_pagelock(pg); 2337 if (flt->wire_paging) { 2338 uvm_pagewire(pg); 2339 if (pg->flags & PG_AOBJ) { 2340 2341 /* 2342 * since the now-wired page cannot be paged out, 2343 * release its swap resources for others to use. 2344 * since an aobj page with no swap cannot be clean, 2345 * mark it dirty now. 2346 */ 2347 2348 KASSERT(uobj != NULL); 2349 uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY); 2350 dropswap = true; 2351 } 2352 } else { 2353 uvm_pageactivate(pg); 2354 } 2355 uvm_pageunlock(pg); 2356 2357 if (dropswap) { 2358 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT); 2359 } 2360 } 2361 2362 2363 /* 2364 * uvm_fault_wire: wire down a range of virtual addresses in a map. 2365 * 2366 * => map may be read-locked by caller, but MUST NOT be write-locked. 2367 * => if map is read-locked, any operations which may cause map to 2368 * be write-locked in uvm_fault() must be taken care of by 2369 * the caller. See uvm_map_pageable(). 2370 */ 2371 2372 int 2373 uvm_fault_wire(struct vm_map *map, vaddr_t start, vaddr_t end, 2374 vm_prot_t access_type, int maxprot) 2375 { 2376 vaddr_t va; 2377 int error; 2378 2379 /* 2380 * now fault it in a page at a time. if the fault fails then we have 2381 * to undo what we have done. note that in uvm_fault VM_PROT_NONE 2382 * is replaced with the max protection if fault_type is VM_FAULT_WIRE. 2383 */ 2384 2385 /* 2386 * XXX work around overflowing a vaddr_t. this prevents us from 2387 * wiring the last page in the address space, though. 2388 */ 2389 if (start > end) { 2390 return EFAULT; 2391 } 2392 2393 for (va = start; va < end; va += PAGE_SIZE) { 2394 error = uvm_fault_internal(map, va, access_type, 2395 (maxprot ? UVM_FAULT_MAXPROT : 0) | UVM_FAULT_WIRE); 2396 if (error) { 2397 if (va != start) { 2398 uvm_fault_unwire(map, start, va); 2399 } 2400 return error; 2401 } 2402 } 2403 return 0; 2404 } 2405 2406 /* 2407 * uvm_fault_unwire(): unwire range of virtual space. 2408 */ 2409 2410 void 2411 uvm_fault_unwire(struct vm_map *map, vaddr_t start, vaddr_t end) 2412 { 2413 vm_map_lock_read(map); 2414 uvm_fault_unwire_locked(map, start, end); 2415 vm_map_unlock_read(map); 2416 } 2417 2418 /* 2419 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire(). 2420 * 2421 * => map must be at least read-locked. 2422 */ 2423 2424 void 2425 uvm_fault_unwire_locked(struct vm_map *map, vaddr_t start, vaddr_t end) 2426 { 2427 struct vm_map_entry *entry, *oentry; 2428 pmap_t pmap = vm_map_pmap(map); 2429 vaddr_t va; 2430 paddr_t pa; 2431 struct vm_page *pg; 2432 2433 /* 2434 * we assume that the area we are unwiring has actually been wired 2435 * in the first place. this means that we should be able to extract 2436 * the PAs from the pmap. we also lock out the page daemon so that 2437 * we can call uvm_pageunwire. 2438 */ 2439 2440 /* 2441 * find the beginning map entry for the region. 2442 */ 2443 2444 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map)); 2445 if (uvm_map_lookup_entry(map, start, &entry) == false) 2446 panic("uvm_fault_unwire_locked: address not in map"); 2447 2448 oentry = NULL; 2449 for (va = start; va < end; va += PAGE_SIZE) { 2450 2451 /* 2452 * find the map entry for the current address. 2453 */ 2454 2455 KASSERT(va >= entry->start); 2456 while (va >= entry->end) { 2457 KASSERT(entry->next != &map->header && 2458 entry->next->start <= entry->end); 2459 entry = entry->next; 2460 } 2461 2462 /* 2463 * lock it. 2464 */ 2465 2466 if (entry != oentry) { 2467 if (oentry != NULL) { 2468 uvm_map_unlock_entry(oentry); 2469 } 2470 uvm_map_lock_entry(entry, RW_WRITER); 2471 oentry = entry; 2472 } 2473 2474 /* 2475 * if the entry is no longer wired, tell the pmap. 2476 */ 2477 2478 if (!pmap_extract(pmap, va, &pa)) 2479 continue; 2480 2481 if (VM_MAPENT_ISWIRED(entry) == 0) 2482 pmap_unwire(pmap, va); 2483 2484 pg = PHYS_TO_VM_PAGE(pa); 2485 if (pg) { 2486 uvm_pagelock(pg); 2487 uvm_pageunwire(pg); 2488 uvm_pageunlock(pg); 2489 } 2490 } 2491 2492 if (oentry != NULL) { 2493 uvm_map_unlock_entry(entry); 2494 } 2495 } 2496