1 /* $NetBSD: uvm_fault.c,v 1.228 2020/07/09 05:57:15 skrll Exp $ */ 2 3 /* 4 * Copyright (c) 1997 Charles D. Cranor and Washington University. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 * 27 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp 28 */ 29 30 /* 31 * uvm_fault.c: fault handler 32 */ 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: uvm_fault.c,v 1.228 2020/07/09 05:57:15 skrll Exp $"); 36 37 #include "opt_uvmhist.h" 38 39 #include <sys/param.h> 40 #include <sys/systm.h> 41 #include <sys/atomic.h> 42 #include <sys/kernel.h> 43 #include <sys/mman.h> 44 45 #include <uvm/uvm.h> 46 #include <uvm/uvm_pdpolicy.h> 47 48 /* 49 * 50 * a word on page faults: 51 * 52 * types of page faults we handle: 53 * 54 * CASE 1: upper layer faults CASE 2: lower layer faults 55 * 56 * CASE 1A CASE 1B CASE 2A CASE 2B 57 * read/write1 write>1 read/write +-cow_write/zero 58 * | | | | 59 * +--|--+ +--|--+ +-----+ + | + | +-----+ 60 * amap | V | | ---------> new | | | | ^ | 61 * +-----+ +-----+ +-----+ + | + | +--|--+ 62 * | | | 63 * +-----+ +-----+ +--|--+ | +--|--+ 64 * uobj | d/c | | d/c | | V | +----+ | 65 * +-----+ +-----+ +-----+ +-----+ 66 * 67 * d/c = don't care 68 * 69 * case [0]: layerless fault 70 * no amap or uobj is present. this is an error. 71 * 72 * case [1]: upper layer fault [anon active] 73 * 1A: [read] or [write with anon->an_ref == 1] 74 * I/O takes place in upper level anon and uobj is not touched. 75 * 1B: [write with anon->an_ref > 1] 76 * new anon is alloc'd and data is copied off ["COW"] 77 * 78 * case [2]: lower layer fault [uobj] 79 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area] 80 * I/O takes place directly in object. 81 * 2B: [write to copy_on_write] or [read on NULL uobj] 82 * data is "promoted" from uobj to a new anon. 83 * if uobj is null, then we zero fill. 84 * 85 * we follow the standard UVM locking protocol ordering: 86 * 87 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ) 88 * we hold a PG_BUSY page if we unlock for I/O 89 * 90 * 91 * the code is structured as follows: 92 * 93 * - init the "IN" params in the ufi structure 94 * ReFault: (ERESTART returned to the loop in uvm_fault_internal) 95 * - do lookups [locks maps], check protection, handle needs_copy 96 * - check for case 0 fault (error) 97 * - establish "range" of fault 98 * - if we have an amap lock it and extract the anons 99 * - if sequential advice deactivate pages behind us 100 * - at the same time check pmap for unmapped areas and anon for pages 101 * that we could map in (and do map it if found) 102 * - check object for resident pages that we could map in 103 * - if (case 2) goto Case2 104 * - >>> handle case 1 105 * - ensure source anon is resident in RAM 106 * - if case 1B alloc new anon and copy from source 107 * - map the correct page in 108 * Case2: 109 * - >>> handle case 2 110 * - ensure source page is resident (if uobj) 111 * - if case 2B alloc new anon and copy from source (could be zero 112 * fill if uobj == NULL) 113 * - map the correct page in 114 * - done! 115 * 116 * note on paging: 117 * if we have to do I/O we place a PG_BUSY page in the correct object, 118 * unlock everything, and do the I/O. when I/O is done we must reverify 119 * the state of the world before assuming that our data structures are 120 * valid. [because mappings could change while the map is unlocked] 121 * 122 * alternative 1: unbusy the page in question and restart the page fault 123 * from the top (ReFault). this is easy but does not take advantage 124 * of the information that we already have from our previous lookup, 125 * although it is possible that the "hints" in the vm_map will help here. 126 * 127 * alternative 2: the system already keeps track of a "version" number of 128 * a map. [i.e. every time you write-lock a map (e.g. to change a 129 * mapping) you bump the version number up by one...] so, we can save 130 * the version number of the map before we release the lock and start I/O. 131 * then when I/O is done we can relock and check the version numbers 132 * to see if anything changed. this might save us some over 1 because 133 * we don't have to unbusy the page and may be less compares(?). 134 * 135 * alternative 3: put in backpointers or a way to "hold" part of a map 136 * in place while I/O is in progress. this could be complex to 137 * implement (especially with structures like amap that can be referenced 138 * by multiple map entries, and figuring out what should wait could be 139 * complex as well...). 140 * 141 * we use alternative 2. given that we are multi-threaded now we may want 142 * to reconsider the choice. 143 */ 144 145 /* 146 * local data structures 147 */ 148 149 struct uvm_advice { 150 int advice; 151 int nback; 152 int nforw; 153 }; 154 155 /* 156 * page range array: 157 * note: index in array must match "advice" value 158 * XXX: borrowed numbers from freebsd. do they work well for us? 159 */ 160 161 static const struct uvm_advice uvmadvice[] = { 162 { UVM_ADV_NORMAL, 3, 4 }, 163 { UVM_ADV_RANDOM, 0, 0 }, 164 { UVM_ADV_SEQUENTIAL, 8, 7}, 165 }; 166 167 #define UVM_MAXRANGE 16 /* must be MAX() of nback+nforw+1 */ 168 169 /* 170 * private prototypes 171 */ 172 173 /* 174 * externs from other modules 175 */ 176 177 extern int start_init_exec; /* Is init_main() done / init running? */ 178 179 /* 180 * inline functions 181 */ 182 183 /* 184 * uvmfault_anonflush: try and deactivate pages in specified anons 185 * 186 * => does not have to deactivate page if it is busy 187 */ 188 189 static inline void 190 uvmfault_anonflush(struct vm_anon **anons, int n) 191 { 192 int lcv; 193 struct vm_page *pg; 194 195 for (lcv = 0; lcv < n; lcv++) { 196 if (anons[lcv] == NULL) 197 continue; 198 KASSERT(rw_lock_held(anons[lcv]->an_lock)); 199 pg = anons[lcv]->an_page; 200 if (pg && (pg->flags & PG_BUSY) == 0) { 201 uvm_pagelock(pg); 202 uvm_pagedeactivate(pg); 203 uvm_pageunlock(pg); 204 } 205 } 206 } 207 208 /* 209 * normal functions 210 */ 211 212 /* 213 * uvmfault_amapcopy: clear "needs_copy" in a map. 214 * 215 * => called with VM data structures unlocked (usually, see below) 216 * => we get a write lock on the maps and clear needs_copy for a VA 217 * => if we are out of RAM we sleep (waiting for more) 218 */ 219 220 static void 221 uvmfault_amapcopy(struct uvm_faultinfo *ufi) 222 { 223 for (;;) { 224 225 /* 226 * no mapping? give up. 227 */ 228 229 if (uvmfault_lookup(ufi, true) == false) 230 return; 231 232 /* 233 * copy if needed. 234 */ 235 236 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) 237 amap_copy(ufi->map, ufi->entry, AMAP_COPY_NOWAIT, 238 ufi->orig_rvaddr, ufi->orig_rvaddr + 1); 239 240 /* 241 * didn't work? must be out of RAM. unlock and sleep. 242 */ 243 244 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 245 uvmfault_unlockmaps(ufi, true); 246 uvm_wait("fltamapcopy"); 247 continue; 248 } 249 250 /* 251 * got it! unlock and return. 252 */ 253 254 uvmfault_unlockmaps(ufi, true); 255 return; 256 } 257 /*NOTREACHED*/ 258 } 259 260 /* 261 * uvmfault_anonget: get data in an anon into a non-busy, non-released 262 * page in that anon. 263 * 264 * => Map, amap and thus anon should be locked by caller. 265 * => If we fail, we unlock everything and error is returned. 266 * => If we are successful, return with everything still locked. 267 * => We do not move the page on the queues [gets moved later]. If we 268 * allocate a new page [we_own], it gets put on the queues. Either way, 269 * the result is that the page is on the queues at return time 270 * => For pages which are on loan from a uvm_object (and thus are not owned 271 * by the anon): if successful, return with the owning object locked. 272 * The caller must unlock this object when it unlocks everything else. 273 */ 274 275 int 276 uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap, 277 struct vm_anon *anon) 278 { 279 struct vm_page *pg; 280 krw_t lock_type; 281 int error; 282 283 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 284 KASSERT(rw_lock_held(anon->an_lock)); 285 KASSERT(anon->an_lock == amap->am_lock); 286 287 /* Increment the counters.*/ 288 cpu_count(CPU_COUNT_FLTANGET, 1); 289 if (anon->an_page) { 290 curlwp->l_ru.ru_minflt++; 291 } else { 292 curlwp->l_ru.ru_majflt++; 293 } 294 error = 0; 295 296 /* 297 * Loop until we get the anon data, or fail. 298 */ 299 300 for (;;) { 301 bool we_own, locked; 302 /* 303 * Note: 'we_own' will become true if we set PG_BUSY on a page. 304 */ 305 we_own = false; 306 pg = anon->an_page; 307 308 /* 309 * If there is a resident page and it is loaned, then anon 310 * may not own it. Call out to uvm_anon_lockloanpg() to 311 * identify and lock the real owner of the page. 312 */ 313 314 if (pg && pg->loan_count) 315 pg = uvm_anon_lockloanpg(anon); 316 317 /* 318 * Is page resident? Make sure it is not busy/released. 319 */ 320 321 lock_type = rw_lock_op(anon->an_lock); 322 if (pg) { 323 324 /* 325 * at this point, if the page has a uobject [meaning 326 * we have it on loan], then that uobject is locked 327 * by us! if the page is busy, we drop all the 328 * locks (including uobject) and try again. 329 */ 330 331 if ((pg->flags & PG_BUSY) == 0) { 332 UVMHIST_LOG(maphist, "<- OK",0,0,0,0); 333 return 0; 334 } 335 cpu_count(CPU_COUNT_FLTPGWAIT, 1); 336 337 /* 338 * The last unlock must be an atomic unlock and wait 339 * on the owner of page. 340 */ 341 342 if (pg->uobject) { 343 /* Owner of page is UVM object. */ 344 uvmfault_unlockall(ufi, amap, NULL); 345 UVMHIST_LOG(maphist, " unlock+wait on uobj",0, 346 0,0,0); 347 uvm_pagewait(pg, pg->uobject->vmobjlock, "anonget1"); 348 } else { 349 /* Owner of page is anon. */ 350 uvmfault_unlockall(ufi, NULL, NULL); 351 UVMHIST_LOG(maphist, " unlock+wait on anon",0, 352 0,0,0); 353 uvm_pagewait(pg, anon->an_lock, "anonget2"); 354 } 355 } else { 356 #if defined(VMSWAP) 357 /* 358 * No page, therefore allocate one. A write lock is 359 * required for this. If the caller didn't supply 360 * one, fail now and have them retry. 361 */ 362 363 if (lock_type == RW_READER) { 364 return ENOLCK; 365 } 366 pg = uvm_pagealloc(NULL, 367 ufi != NULL ? ufi->orig_rvaddr : 0, 368 anon, ufi != NULL ? UVM_FLAG_COLORMATCH : 0); 369 if (pg == NULL) { 370 /* Out of memory. Wait a little. */ 371 uvmfault_unlockall(ufi, amap, NULL); 372 cpu_count(CPU_COUNT_FLTNORAM, 1); 373 UVMHIST_LOG(maphist, " noram -- UVM_WAIT",0, 374 0,0,0); 375 if (!uvm_reclaimable()) { 376 return ENOMEM; 377 } 378 uvm_wait("flt_noram1"); 379 } else { 380 /* PG_BUSY bit is set. */ 381 we_own = true; 382 uvmfault_unlockall(ufi, amap, NULL); 383 384 /* 385 * Pass a PG_BUSY+PG_FAKE clean page into 386 * the uvm_swap_get() function with all data 387 * structures unlocked. Note that it is OK 388 * to read an_swslot here, because we hold 389 * PG_BUSY on the page. 390 */ 391 cpu_count(CPU_COUNT_PAGEINS, 1); 392 error = uvm_swap_get(pg, anon->an_swslot, 393 PGO_SYNCIO); 394 395 /* 396 * We clean up after the I/O below in the 397 * 'we_own' case. 398 */ 399 } 400 #else 401 panic("%s: no page", __func__); 402 #endif /* defined(VMSWAP) */ 403 } 404 405 /* 406 * Re-lock the map and anon. 407 */ 408 409 locked = uvmfault_relock(ufi); 410 if (locked || we_own) { 411 rw_enter(anon->an_lock, lock_type); 412 } 413 414 /* 415 * If we own the page (i.e. we set PG_BUSY), then we need 416 * to clean up after the I/O. There are three cases to 417 * consider: 418 * 419 * 1) Page was released during I/O: free anon and ReFault. 420 * 2) I/O not OK. Free the page and cause the fault to fail. 421 * 3) I/O OK! Activate the page and sync with the non-we_own 422 * case (i.e. drop anon lock if not locked). 423 */ 424 425 if (we_own) { 426 KASSERT(lock_type == RW_WRITER); 427 #if defined(VMSWAP) 428 if (error) { 429 430 /* 431 * Remove the swap slot from the anon and 432 * mark the anon as having no real slot. 433 * Do not free the swap slot, thus preventing 434 * it from being used again. 435 */ 436 437 if (anon->an_swslot > 0) { 438 uvm_swap_markbad(anon->an_swslot, 1); 439 } 440 anon->an_swslot = SWSLOT_BAD; 441 442 if ((pg->flags & PG_RELEASED) != 0) { 443 goto released; 444 } 445 446 /* 447 * Note: page was never !PG_BUSY, so it 448 * cannot be mapped and thus no need to 449 * pmap_page_protect() it. 450 */ 451 452 uvm_pagefree(pg); 453 454 if (locked) { 455 uvmfault_unlockall(ufi, NULL, NULL); 456 } 457 rw_exit(anon->an_lock); 458 UVMHIST_LOG(maphist, "<- ERROR", 0,0,0,0); 459 return error; 460 } 461 462 if ((pg->flags & PG_RELEASED) != 0) { 463 released: 464 KASSERT(anon->an_ref == 0); 465 466 /* 467 * Released while we had unlocked amap. 468 */ 469 470 if (locked) { 471 uvmfault_unlockall(ufi, NULL, NULL); 472 } 473 uvm_anon_release(anon); 474 475 if (error) { 476 UVMHIST_LOG(maphist, 477 "<- ERROR/RELEASED", 0,0,0,0); 478 return error; 479 } 480 481 UVMHIST_LOG(maphist, "<- RELEASED", 0,0,0,0); 482 return ERESTART; 483 } 484 485 /* 486 * We have successfully read the page, activate it. 487 */ 488 489 uvm_pagelock(pg); 490 uvm_pageactivate(pg); 491 uvm_pagewakeup(pg); 492 uvm_pageunlock(pg); 493 pg->flags &= ~(PG_BUSY|PG_FAKE); 494 uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_UNKNOWN); 495 UVM_PAGE_OWN(pg, NULL); 496 #else 497 panic("%s: we_own", __func__); 498 #endif /* defined(VMSWAP) */ 499 } 500 501 /* 502 * We were not able to re-lock the map - restart the fault. 503 */ 504 505 if (!locked) { 506 if (we_own) { 507 rw_exit(anon->an_lock); 508 } 509 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 510 return ERESTART; 511 } 512 513 /* 514 * Verify that no one has touched the amap and moved 515 * the anon on us. 516 */ 517 518 if (ufi != NULL && amap_lookup(&ufi->entry->aref, 519 ufi->orig_rvaddr - ufi->entry->start) != anon) { 520 521 uvmfault_unlockall(ufi, amap, NULL); 522 UVMHIST_LOG(maphist, "<- REFAULT", 0,0,0,0); 523 return ERESTART; 524 } 525 526 /* 527 * Retry.. 528 */ 529 530 cpu_count(CPU_COUNT_FLTANRETRY, 1); 531 continue; 532 } 533 /*NOTREACHED*/ 534 } 535 536 /* 537 * uvmfault_promote: promote data to a new anon. used for 1B and 2B. 538 * 539 * 1. allocate an anon and a page. 540 * 2. fill its contents. 541 * 3. put it into amap. 542 * 543 * => if we fail (result != 0) we unlock everything. 544 * => on success, return a new locked anon via 'nanon'. 545 * (*nanon)->an_page will be a resident, locked, dirty page. 546 * => it's caller's responsibility to put the promoted nanon->an_page to the 547 * page queue. 548 */ 549 550 static int 551 uvmfault_promote(struct uvm_faultinfo *ufi, 552 struct vm_anon *oanon, 553 struct vm_page *uobjpage, 554 struct vm_anon **nanon, /* OUT: allocated anon */ 555 struct vm_anon **spare) 556 { 557 struct vm_amap *amap = ufi->entry->aref.ar_amap; 558 struct uvm_object *uobj; 559 struct vm_anon *anon; 560 struct vm_page *pg; 561 struct vm_page *opg; 562 int error; 563 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 564 565 if (oanon) { 566 /* anon COW */ 567 opg = oanon->an_page; 568 KASSERT(opg != NULL); 569 KASSERT(opg->uobject == NULL || opg->loan_count > 0); 570 } else if (uobjpage != PGO_DONTCARE) { 571 /* object-backed COW */ 572 opg = uobjpage; 573 KASSERT(rw_lock_held(opg->uobject->vmobjlock)); 574 } else { 575 /* ZFOD */ 576 opg = NULL; 577 } 578 if (opg != NULL) { 579 uobj = opg->uobject; 580 } else { 581 uobj = NULL; 582 } 583 584 KASSERT(amap != NULL); 585 KASSERT(uobjpage != NULL); 586 KASSERT(rw_write_held(amap->am_lock)); 587 KASSERT(oanon == NULL || amap->am_lock == oanon->an_lock); 588 KASSERT(uobj == NULL || rw_lock_held(uobj->vmobjlock)); 589 590 if (*spare != NULL) { 591 anon = *spare; 592 *spare = NULL; 593 } else { 594 anon = uvm_analloc(); 595 } 596 if (anon) { 597 598 /* 599 * The new anon is locked. 600 * 601 * if opg == NULL, we want a zero'd, dirty page, 602 * so have uvm_pagealloc() do that for us. 603 */ 604 605 KASSERT(anon->an_lock == NULL); 606 anon->an_lock = amap->am_lock; 607 pg = uvm_pagealloc(NULL, ufi->orig_rvaddr, anon, 608 UVM_FLAG_COLORMATCH | (opg == NULL ? UVM_PGA_ZERO : 0)); 609 if (pg == NULL) { 610 anon->an_lock = NULL; 611 } 612 } else { 613 pg = NULL; 614 } 615 616 /* 617 * out of memory resources? 618 */ 619 620 if (pg == NULL) { 621 /* save anon for the next try. */ 622 if (anon != NULL) { 623 *spare = anon; 624 } 625 626 /* unlock and fail ... */ 627 uvmfault_unlockall(ufi, amap, uobj); 628 if (!uvm_reclaimable()) { 629 UVMHIST_LOG(maphist, "out of VM", 0,0,0,0); 630 cpu_count(CPU_COUNT_FLTNOANON, 1); 631 error = ENOMEM; 632 goto done; 633 } 634 635 UVMHIST_LOG(maphist, "out of RAM, waiting for more", 0,0,0,0); 636 cpu_count(CPU_COUNT_FLTNORAM, 1); 637 uvm_wait("flt_noram5"); 638 error = ERESTART; 639 goto done; 640 } 641 642 /* copy page [pg now dirty] */ 643 if (opg) { 644 uvm_pagecopy(opg, pg); 645 } 646 KASSERT(uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_DIRTY); 647 648 amap_add(&ufi->entry->aref, ufi->orig_rvaddr - ufi->entry->start, anon, 649 oanon != NULL); 650 651 /* 652 * from this point on am_lock won't be dropped until the page is 653 * entered, so it's safe to unbusy the page up front. 654 * 655 * uvm_fault_{upper,lower}_done will activate or enqueue the page. 656 */ 657 658 pg = anon->an_page; 659 pg->flags &= ~(PG_BUSY|PG_FAKE); 660 UVM_PAGE_OWN(pg, NULL); 661 662 *nanon = anon; 663 error = 0; 664 done: 665 return error; 666 } 667 668 /* 669 * Update statistics after fault resolution. 670 * - maxrss 671 */ 672 void 673 uvmfault_update_stats(struct uvm_faultinfo *ufi) 674 { 675 struct vm_map *map; 676 struct vmspace *vm; 677 struct proc *p; 678 vsize_t res; 679 680 map = ufi->orig_map; 681 682 p = curproc; 683 KASSERT(p != NULL); 684 vm = p->p_vmspace; 685 686 if (&vm->vm_map != map) 687 return; 688 689 res = pmap_resident_count(map->pmap); 690 if (vm->vm_rssmax < res) 691 vm->vm_rssmax = res; 692 } 693 694 /* 695 * F A U L T - m a i n e n t r y p o i n t 696 */ 697 698 /* 699 * uvm_fault: page fault handler 700 * 701 * => called from MD code to resolve a page fault 702 * => VM data structures usually should be unlocked. however, it is 703 * possible to call here with the main map locked if the caller 704 * gets a write lock, sets it recusive, and then calls us (c.f. 705 * uvm_map_pageable). this should be avoided because it keeps 706 * the map locked off during I/O. 707 * => MUST NEVER BE CALLED IN INTERRUPT CONTEXT 708 */ 709 710 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 711 ~VM_PROT_WRITE : VM_PROT_ALL) 712 713 /* fault_flag values passed from uvm_fault_wire to uvm_fault_internal */ 714 #define UVM_FAULT_WIRE (1 << 0) 715 #define UVM_FAULT_MAXPROT (1 << 1) 716 717 struct uvm_faultctx { 718 719 /* 720 * the following members are set up by uvm_fault_check() and 721 * read-only after that. 722 * 723 * note that narrow is used by uvm_fault_check() to change 724 * the behaviour after ERESTART. 725 * 726 * most of them might change after RESTART if the underlying 727 * map entry has been changed behind us. an exception is 728 * wire_paging, which does never change. 729 */ 730 vm_prot_t access_type; 731 vaddr_t startva; 732 int npages; 733 int centeridx; 734 bool narrow; /* work on a single requested page only */ 735 bool wire_mapping; /* request a PMAP_WIRED mapping 736 (UVM_FAULT_WIRE or VM_MAPENT_ISWIRED) */ 737 bool wire_paging; /* request uvm_pagewire 738 (true for UVM_FAULT_WIRE) */ 739 bool cow_now; /* VM_PROT_WRITE is actually requested 740 (ie. should break COW and page loaning) */ 741 742 /* 743 * enter_prot is set up by uvm_fault_check() and clamped 744 * (ie. drop the VM_PROT_WRITE bit) in various places in case 745 * of !cow_now. 746 */ 747 vm_prot_t enter_prot; /* prot at which we want to enter pages in */ 748 749 /* 750 * the following member is for uvmfault_promote() and ERESTART. 751 */ 752 struct vm_anon *anon_spare; 753 754 /* 755 * the folloing is actually a uvm_fault_lower() internal. 756 * it's here merely for debugging. 757 * (or due to the mechanical separation of the function?) 758 */ 759 bool promote; 760 761 /* 762 * type of lock to acquire on objects in both layers. 763 */ 764 krw_t lower_lock_type; 765 krw_t upper_lock_type; 766 }; 767 768 static inline int uvm_fault_check( 769 struct uvm_faultinfo *, struct uvm_faultctx *, 770 struct vm_anon ***, bool); 771 772 static int uvm_fault_upper( 773 struct uvm_faultinfo *, struct uvm_faultctx *, 774 struct vm_anon **); 775 static inline int uvm_fault_upper_lookup( 776 struct uvm_faultinfo *, const struct uvm_faultctx *, 777 struct vm_anon **, struct vm_page **); 778 static inline void uvm_fault_upper_neighbor( 779 struct uvm_faultinfo *, const struct uvm_faultctx *, 780 vaddr_t, struct vm_page *, bool); 781 static inline int uvm_fault_upper_loan( 782 struct uvm_faultinfo *, struct uvm_faultctx *, 783 struct vm_anon *, struct uvm_object **); 784 static inline int uvm_fault_upper_promote( 785 struct uvm_faultinfo *, struct uvm_faultctx *, 786 struct uvm_object *, struct vm_anon *); 787 static inline int uvm_fault_upper_direct( 788 struct uvm_faultinfo *, struct uvm_faultctx *, 789 struct uvm_object *, struct vm_anon *); 790 static int uvm_fault_upper_enter( 791 struct uvm_faultinfo *, const struct uvm_faultctx *, 792 struct uvm_object *, struct vm_anon *, 793 struct vm_page *, struct vm_anon *); 794 static inline void uvm_fault_upper_done( 795 struct uvm_faultinfo *, const struct uvm_faultctx *, 796 struct vm_anon *, struct vm_page *); 797 798 static int uvm_fault_lower( 799 struct uvm_faultinfo *, struct uvm_faultctx *, 800 struct vm_page **); 801 static inline void uvm_fault_lower_lookup( 802 struct uvm_faultinfo *, const struct uvm_faultctx *, 803 struct vm_page **); 804 static inline void uvm_fault_lower_neighbor( 805 struct uvm_faultinfo *, const struct uvm_faultctx *, 806 vaddr_t, struct vm_page *); 807 static inline int uvm_fault_lower_io( 808 struct uvm_faultinfo *, struct uvm_faultctx *, 809 struct uvm_object **, struct vm_page **); 810 static inline int uvm_fault_lower_direct( 811 struct uvm_faultinfo *, struct uvm_faultctx *, 812 struct uvm_object *, struct vm_page *); 813 static inline int uvm_fault_lower_direct_loan( 814 struct uvm_faultinfo *, struct uvm_faultctx *, 815 struct uvm_object *, struct vm_page **, 816 struct vm_page **); 817 static inline int uvm_fault_lower_promote( 818 struct uvm_faultinfo *, struct uvm_faultctx *, 819 struct uvm_object *, struct vm_page *); 820 static int uvm_fault_lower_enter( 821 struct uvm_faultinfo *, const struct uvm_faultctx *, 822 struct uvm_object *, 823 struct vm_anon *, struct vm_page *); 824 static inline void uvm_fault_lower_done( 825 struct uvm_faultinfo *, const struct uvm_faultctx *, 826 struct uvm_object *, struct vm_page *); 827 828 int 829 uvm_fault_internal(struct vm_map *orig_map, vaddr_t vaddr, 830 vm_prot_t access_type, int fault_flag) 831 { 832 struct uvm_faultinfo ufi; 833 struct uvm_faultctx flt = { 834 .access_type = access_type, 835 836 /* don't look for neighborhood * pages on "wire" fault */ 837 .narrow = (fault_flag & UVM_FAULT_WIRE) != 0, 838 839 /* "wire" fault causes wiring of both mapping and paging */ 840 .wire_mapping = (fault_flag & UVM_FAULT_WIRE) != 0, 841 .wire_paging = (fault_flag & UVM_FAULT_WIRE) != 0, 842 843 /* 844 * default lock type to acquire on upper & lower layer 845 * objects: reader. this can be upgraded at any point 846 * during the fault from read -> write and uvm_faultctx 847 * changed to match, but is never downgraded write -> read. 848 */ 849 #ifdef __HAVE_UNLOCKED_PMAP /* XXX temporary */ 850 .upper_lock_type = RW_WRITER, 851 .lower_lock_type = RW_WRITER, 852 #else 853 .upper_lock_type = RW_READER, 854 .lower_lock_type = RW_READER, 855 #endif 856 }; 857 const bool maxprot = (fault_flag & UVM_FAULT_MAXPROT) != 0; 858 struct vm_anon *anons_store[UVM_MAXRANGE], **anons; 859 struct vm_page *pages_store[UVM_MAXRANGE], **pages; 860 int error; 861 862 UVMHIST_FUNC(__func__); 863 UVMHIST_CALLARGS(maphist, "(map=%#jx, vaddr=%#jx, at=%jd, ff=%jd)", 864 (uintptr_t)orig_map, vaddr, access_type, fault_flag); 865 866 /* Don't count anything until user interaction is possible */ 867 kpreempt_disable(); 868 if (__predict_true(start_init_exec)) { 869 struct cpu_info *ci = curcpu(); 870 CPU_COUNT(CPU_COUNT_NFAULT, 1); 871 /* Don't flood RNG subsystem with samples. */ 872 if (++(ci->ci_faultrng) == 503) { 873 ci->ci_faultrng = 0; 874 rnd_add_uint32(&curcpu()->ci_data.cpu_uvm->rs, 875 sizeof(vaddr_t) == sizeof(uint32_t) ? 876 (uint32_t)vaddr : sizeof(vaddr_t) == 877 sizeof(uint64_t) ? 878 (uint32_t)vaddr : 879 (uint32_t)ci->ci_counts[CPU_COUNT_NFAULT]); 880 } 881 } 882 kpreempt_enable(); 883 884 /* 885 * init the IN parameters in the ufi 886 */ 887 888 ufi.orig_map = orig_map; 889 ufi.orig_rvaddr = trunc_page(vaddr); 890 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ 891 892 error = ERESTART; 893 while (error == ERESTART) { /* ReFault: */ 894 anons = anons_store; 895 pages = pages_store; 896 897 error = uvm_fault_check(&ufi, &flt, &anons, maxprot); 898 if (error != 0) 899 continue; 900 901 error = uvm_fault_upper_lookup(&ufi, &flt, anons, pages); 902 if (error != 0) 903 continue; 904 905 if (pages[flt.centeridx] == PGO_DONTCARE) 906 error = uvm_fault_upper(&ufi, &flt, anons); 907 else { 908 struct uvm_object * const uobj = 909 ufi.entry->object.uvm_obj; 910 911 if (uobj && uobj->pgops->pgo_fault != NULL) { 912 /* 913 * invoke "special" fault routine. 914 */ 915 rw_enter(uobj->vmobjlock, RW_WRITER); 916 /* locked: maps(read), amap(if there), uobj */ 917 error = uobj->pgops->pgo_fault(&ufi, 918 flt.startva, pages, flt.npages, 919 flt.centeridx, flt.access_type, 920 PGO_LOCKED|PGO_SYNCIO); 921 922 /* 923 * locked: nothing, pgo_fault has unlocked 924 * everything 925 */ 926 927 /* 928 * object fault routine responsible for 929 * pmap_update(). 930 */ 931 932 /* 933 * Wake up the pagedaemon if the fault method 934 * failed for lack of memory but some can be 935 * reclaimed. 936 */ 937 if (error == ENOMEM && uvm_reclaimable()) { 938 uvm_wait("pgo_fault"); 939 error = ERESTART; 940 } 941 } else { 942 error = uvm_fault_lower(&ufi, &flt, pages); 943 } 944 } 945 } 946 947 if (flt.anon_spare != NULL) { 948 flt.anon_spare->an_ref--; 949 KASSERT(flt.anon_spare->an_ref == 0); 950 KASSERT(flt.anon_spare->an_lock == NULL); 951 uvm_anfree(flt.anon_spare); 952 } 953 return error; 954 } 955 956 /* 957 * uvm_fault_check: check prot, handle needs-copy, etc. 958 * 959 * 1. lookup entry. 960 * 2. check protection. 961 * 3. adjust fault condition (mainly for simulated fault). 962 * 4. handle needs-copy (lazy amap copy). 963 * 5. establish range of interest for neighbor fault (aka pre-fault). 964 * 6. look up anons (if amap exists). 965 * 7. flush pages (if MADV_SEQUENTIAL) 966 * 967 * => called with nothing locked. 968 * => if we fail (result != 0) we unlock everything. 969 * => initialize/adjust many members of flt. 970 */ 971 972 static int 973 uvm_fault_check( 974 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 975 struct vm_anon ***ranons, bool maxprot) 976 { 977 struct vm_amap *amap; 978 struct uvm_object *uobj; 979 vm_prot_t check_prot; 980 int nback, nforw; 981 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 982 983 /* 984 * lookup and lock the maps 985 */ 986 987 if (uvmfault_lookup(ufi, false) == false) { 988 UVMHIST_LOG(maphist, "<- no mapping @ %#jx", ufi->orig_rvaddr, 989 0,0,0); 990 return EFAULT; 991 } 992 /* locked: maps(read) */ 993 994 #ifdef DIAGNOSTIC 995 if ((ufi->map->flags & VM_MAP_PAGEABLE) == 0) { 996 printf("Page fault on non-pageable map:\n"); 997 printf("ufi->map = %p\n", ufi->map); 998 printf("ufi->orig_map = %p\n", ufi->orig_map); 999 printf("ufi->orig_rvaddr = %#lx\n", (u_long) ufi->orig_rvaddr); 1000 panic("uvm_fault: (ufi->map->flags & VM_MAP_PAGEABLE) == 0"); 1001 } 1002 #endif 1003 1004 /* 1005 * check protection 1006 */ 1007 1008 check_prot = maxprot ? 1009 ufi->entry->max_protection : ufi->entry->protection; 1010 if ((check_prot & flt->access_type) != flt->access_type) { 1011 UVMHIST_LOG(maphist, 1012 "<- protection failure (prot=%#jx, access=%#jx)", 1013 ufi->entry->protection, flt->access_type, 0, 0); 1014 uvmfault_unlockmaps(ufi, false); 1015 return EFAULT; 1016 } 1017 1018 /* 1019 * "enter_prot" is the protection we want to enter the page in at. 1020 * for certain pages (e.g. copy-on-write pages) this protection can 1021 * be more strict than ufi->entry->protection. "wired" means either 1022 * the entry is wired or we are fault-wiring the pg. 1023 */ 1024 1025 flt->enter_prot = ufi->entry->protection; 1026 if (VM_MAPENT_ISWIRED(ufi->entry)) { 1027 flt->wire_mapping = true; 1028 flt->wire_paging = true; 1029 flt->narrow = true; 1030 } 1031 1032 if (flt->wire_mapping) { 1033 flt->access_type = flt->enter_prot; /* full access for wired */ 1034 flt->cow_now = (check_prot & VM_PROT_WRITE) != 0; 1035 } else { 1036 flt->cow_now = (flt->access_type & VM_PROT_WRITE) != 0; 1037 } 1038 1039 if (flt->wire_paging) { 1040 /* wiring pages requires a write lock. */ 1041 flt->upper_lock_type = RW_WRITER; 1042 flt->lower_lock_type = RW_WRITER; 1043 } 1044 1045 flt->promote = false; 1046 1047 /* 1048 * handle "needs_copy" case. if we need to copy the amap we will 1049 * have to drop our readlock and relock it with a write lock. (we 1050 * need a write lock to change anything in a map entry [e.g. 1051 * needs_copy]). 1052 */ 1053 1054 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 1055 if (flt->cow_now || (ufi->entry->object.uvm_obj == NULL)) { 1056 KASSERT(!maxprot); 1057 /* need to clear */ 1058 UVMHIST_LOG(maphist, 1059 " need to clear needs_copy and refault",0,0,0,0); 1060 uvmfault_unlockmaps(ufi, false); 1061 uvmfault_amapcopy(ufi); 1062 cpu_count(CPU_COUNT_FLTAMCOPY, 1); 1063 return ERESTART; 1064 1065 } else { 1066 1067 /* 1068 * ensure that we pmap_enter page R/O since 1069 * needs_copy is still true 1070 */ 1071 1072 flt->enter_prot &= ~VM_PROT_WRITE; 1073 } 1074 } 1075 1076 /* 1077 * identify the players 1078 */ 1079 1080 amap = ufi->entry->aref.ar_amap; /* upper layer */ 1081 uobj = ufi->entry->object.uvm_obj; /* lower layer */ 1082 1083 /* 1084 * check for a case 0 fault. if nothing backing the entry then 1085 * error now. 1086 */ 1087 1088 if (amap == NULL && uobj == NULL) { 1089 uvmfault_unlockmaps(ufi, false); 1090 UVMHIST_LOG(maphist,"<- no backing store, no overlay",0,0,0,0); 1091 return EFAULT; 1092 } 1093 1094 /* 1095 * for a case 2B fault waste no time on adjacent pages because 1096 * they are likely already entered. 1097 */ 1098 1099 if (uobj != NULL && amap != NULL && 1100 (flt->access_type & VM_PROT_WRITE) != 0) { 1101 /* wide fault (!narrow) */ 1102 flt->narrow = true; 1103 } 1104 1105 /* 1106 * establish range of interest based on advice from mapper 1107 * and then clip to fit map entry. note that we only want 1108 * to do this the first time through the fault. if we 1109 * ReFault we will disable this by setting "narrow" to true. 1110 */ 1111 1112 if (flt->narrow == false) { 1113 1114 /* wide fault (!narrow) */ 1115 KASSERT(uvmadvice[ufi->entry->advice].advice == 1116 ufi->entry->advice); 1117 nback = MIN(uvmadvice[ufi->entry->advice].nback, 1118 (ufi->orig_rvaddr - ufi->entry->start) >> PAGE_SHIFT); 1119 flt->startva = ufi->orig_rvaddr - (nback << PAGE_SHIFT); 1120 /* 1121 * note: "-1" because we don't want to count the 1122 * faulting page as forw 1123 */ 1124 nforw = MIN(uvmadvice[ufi->entry->advice].nforw, 1125 ((ufi->entry->end - ufi->orig_rvaddr) >> 1126 PAGE_SHIFT) - 1); 1127 flt->npages = nback + nforw + 1; 1128 flt->centeridx = nback; 1129 1130 flt->narrow = true; /* ensure only once per-fault */ 1131 1132 } else { 1133 1134 /* narrow fault! */ 1135 nback = nforw = 0; 1136 flt->startva = ufi->orig_rvaddr; 1137 flt->npages = 1; 1138 flt->centeridx = 0; 1139 1140 } 1141 /* offset from entry's start to pgs' start */ 1142 const voff_t eoff = flt->startva - ufi->entry->start; 1143 1144 /* locked: maps(read) */ 1145 UVMHIST_LOG(maphist, " narrow=%jd, back=%jd, forw=%jd, startva=%#jx", 1146 flt->narrow, nback, nforw, flt->startva); 1147 UVMHIST_LOG(maphist, " entry=%#jx, amap=%#jx, obj=%#jx", 1148 (uintptr_t)ufi->entry, (uintptr_t)amap, (uintptr_t)uobj, 0); 1149 1150 /* 1151 * guess at the most suitable lock types to acquire. 1152 * if we've got an amap then lock it and extract current anons. 1153 */ 1154 1155 if (amap) { 1156 if ((amap_flags(amap) & AMAP_SHARED) == 0) { 1157 /* 1158 * the amap isn't shared. get a writer lock to 1159 * avoid the cost of upgrading the lock later if 1160 * needed. 1161 * 1162 * XXX nice for PostgreSQL, but consider threads. 1163 */ 1164 flt->upper_lock_type = RW_WRITER; 1165 } else if ((flt->access_type & VM_PROT_WRITE) != 0) { 1166 /* 1167 * assume we're about to COW. 1168 */ 1169 flt->upper_lock_type = RW_WRITER; 1170 } 1171 amap_lock(amap, flt->upper_lock_type); 1172 amap_lookups(&ufi->entry->aref, eoff, *ranons, flt->npages); 1173 } else { 1174 if ((flt->access_type & VM_PROT_WRITE) != 0) { 1175 /* 1176 * we are about to dirty the object and that 1177 * requires a write lock. 1178 */ 1179 flt->lower_lock_type = RW_WRITER; 1180 } 1181 *ranons = NULL; /* to be safe */ 1182 } 1183 1184 /* locked: maps(read), amap(if there) */ 1185 KASSERT(amap == NULL || 1186 rw_lock_op(amap->am_lock) == flt->upper_lock_type); 1187 1188 /* 1189 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages 1190 * now and then forget about them (for the rest of the fault). 1191 */ 1192 1193 if (ufi->entry->advice == MADV_SEQUENTIAL && nback != 0) { 1194 1195 UVMHIST_LOG(maphist, " MADV_SEQUENTIAL: flushing backpages", 1196 0,0,0,0); 1197 /* flush back-page anons? */ 1198 if (amap) 1199 uvmfault_anonflush(*ranons, nback); 1200 1201 /* 1202 * flush object? change lock type to RW_WRITER, to avoid 1203 * excessive competition between read/write locks if many 1204 * threads doing "sequential access". 1205 */ 1206 if (uobj) { 1207 voff_t uoff; 1208 1209 flt->lower_lock_type = RW_WRITER; 1210 uoff = ufi->entry->offset + eoff; 1211 rw_enter(uobj->vmobjlock, RW_WRITER); 1212 (void) (uobj->pgops->pgo_put)(uobj, uoff, uoff + 1213 (nback << PAGE_SHIFT), PGO_DEACTIVATE); 1214 } 1215 1216 /* now forget about the backpages */ 1217 if (amap) 1218 *ranons += nback; 1219 flt->startva += (nback << PAGE_SHIFT); 1220 flt->npages -= nback; 1221 flt->centeridx = 0; 1222 } 1223 /* 1224 * => startva is fixed 1225 * => npages is fixed 1226 */ 1227 KASSERT(flt->startva <= ufi->orig_rvaddr); 1228 KASSERT(ufi->orig_rvaddr + ufi->orig_size <= 1229 flt->startva + (flt->npages << PAGE_SHIFT)); 1230 return 0; 1231 } 1232 1233 /* 1234 * uvm_fault_upper_upgrade: upgrade upper lock, reader -> writer 1235 */ 1236 1237 static inline int 1238 uvm_fault_upper_upgrade(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1239 struct vm_amap *amap, struct uvm_object *uobj) 1240 { 1241 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 1242 1243 KASSERT(amap != NULL); 1244 KASSERT(flt->upper_lock_type == rw_lock_op(amap->am_lock)); 1245 1246 /* 1247 * fast path. 1248 */ 1249 1250 if (__predict_true(flt->upper_lock_type == RW_WRITER)) { 1251 return 0; 1252 } 1253 1254 /* 1255 * otherwise try for the upgrade. if we don't get it, unlock 1256 * everything, restart the fault and next time around get a writer 1257 * lock. 1258 */ 1259 1260 flt->upper_lock_type = RW_WRITER; 1261 if (__predict_false(!rw_tryupgrade(amap->am_lock))) { 1262 uvmfault_unlockall(ufi, amap, uobj); 1263 cpu_count(CPU_COUNT_FLTNOUP, 1); 1264 UVMHIST_LOG(maphist, " !upgrade upper", 0, 0,0,0); 1265 return ERESTART; 1266 } 1267 cpu_count(CPU_COUNT_FLTUP, 1); 1268 KASSERT(flt->upper_lock_type == rw_lock_op(amap->am_lock)); 1269 return 0; 1270 } 1271 1272 /* 1273 * uvm_fault_upper_lookup: look up existing h/w mapping and amap. 1274 * 1275 * iterate range of interest: 1276 * 1. check if h/w mapping exists. if yes, we don't care 1277 * 2. check if anon exists. if not, page is lower. 1278 * 3. if anon exists, enter h/w mapping for neighbors. 1279 * 1280 * => called with amap locked (if exists). 1281 */ 1282 1283 static int 1284 uvm_fault_upper_lookup( 1285 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 1286 struct vm_anon **anons, struct vm_page **pages) 1287 { 1288 struct vm_amap *amap = ufi->entry->aref.ar_amap; 1289 int lcv; 1290 vaddr_t currva; 1291 bool shadowed __unused; 1292 bool entered; 1293 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 1294 1295 /* locked: maps(read), amap(if there) */ 1296 KASSERT(amap == NULL || 1297 rw_lock_op(amap->am_lock) == flt->upper_lock_type); 1298 1299 /* 1300 * map in the backpages and frontpages we found in the amap in hopes 1301 * of preventing future faults. we also init the pages[] array as 1302 * we go. 1303 */ 1304 1305 currva = flt->startva; 1306 shadowed = false; 1307 entered = false; 1308 for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) { 1309 /* 1310 * unmapped or center page. check if any anon at this level. 1311 */ 1312 if (amap == NULL || anons[lcv] == NULL) { 1313 pages[lcv] = NULL; 1314 continue; 1315 } 1316 1317 /* 1318 * check for present page and map if possible. 1319 */ 1320 1321 pages[lcv] = PGO_DONTCARE; 1322 if (lcv == flt->centeridx) { /* save center for later! */ 1323 shadowed = true; 1324 continue; 1325 } 1326 1327 struct vm_anon *anon = anons[lcv]; 1328 struct vm_page *pg = anon->an_page; 1329 1330 KASSERT(anon->an_lock == amap->am_lock); 1331 1332 /* 1333 * ignore loaned and busy pages. 1334 * don't play with VAs that are already mapped. 1335 */ 1336 1337 if (pg && pg->loan_count == 0 && (pg->flags & PG_BUSY) == 0 && 1338 !pmap_extract(ufi->orig_map->pmap, currva, NULL)) { 1339 uvm_fault_upper_neighbor(ufi, flt, currva, 1340 pg, anon->an_ref > 1); 1341 entered = true; 1342 } 1343 } 1344 if (entered) { 1345 pmap_update(ufi->orig_map->pmap); 1346 } 1347 1348 /* locked: maps(read), amap(if there) */ 1349 KASSERT(amap == NULL || 1350 rw_lock_op(amap->am_lock) == flt->upper_lock_type); 1351 /* (shadowed == true) if there is an anon at the faulting address */ 1352 UVMHIST_LOG(maphist, " shadowed=%jd, will_get=%jd", shadowed, 1353 (ufi->entry->object.uvm_obj && shadowed != false),0,0); 1354 1355 return 0; 1356 } 1357 1358 /* 1359 * uvm_fault_upper_neighbor: enter single upper neighbor page. 1360 * 1361 * => called with amap and anon locked. 1362 */ 1363 1364 static void 1365 uvm_fault_upper_neighbor( 1366 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 1367 vaddr_t currva, struct vm_page *pg, bool readonly) 1368 { 1369 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 1370 1371 /* locked: amap, anon */ 1372 1373 KASSERT(pg->uobject == NULL); 1374 KASSERT(pg->uanon != NULL); 1375 KASSERT(rw_lock_op(pg->uanon->an_lock) == flt->upper_lock_type); 1376 KASSERT(uvm_pagegetdirty(pg) != UVM_PAGE_STATUS_CLEAN); 1377 1378 /* 1379 * there wasn't a direct fault on the page, so avoid the cost of 1380 * activating it. 1381 */ 1382 1383 if (!uvmpdpol_pageisqueued_p(pg) && pg->wire_count == 0) { 1384 uvm_pagelock(pg); 1385 uvm_pageenqueue(pg); 1386 uvm_pageunlock(pg); 1387 } 1388 1389 UVMHIST_LOG(maphist, 1390 " MAPPING: n anon: pm=%#jx, va=%#jx, pg=%#jx", 1391 (uintptr_t)ufi->orig_map->pmap, currva, (uintptr_t)pg, 0); 1392 cpu_count(CPU_COUNT_FLTNAMAP, 1); 1393 1394 /* 1395 * Since this page isn't the page that's actually faulting, 1396 * ignore pmap_enter() failures; it's not critical that we 1397 * enter these right now. 1398 */ 1399 1400 (void) pmap_enter(ufi->orig_map->pmap, currva, 1401 VM_PAGE_TO_PHYS(pg), 1402 readonly ? (flt->enter_prot & ~VM_PROT_WRITE) : 1403 flt->enter_prot, 1404 PMAP_CANFAIL | (flt->wire_mapping ? PMAP_WIRED : 0)); 1405 } 1406 1407 /* 1408 * uvm_fault_upper: handle upper fault. 1409 * 1410 * 1. acquire anon lock. 1411 * 2. get anon. let uvmfault_anonget do the dirty work. 1412 * 3. handle loan. 1413 * 4. dispatch direct or promote handlers. 1414 */ 1415 1416 static int 1417 uvm_fault_upper( 1418 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1419 struct vm_anon **anons) 1420 { 1421 struct vm_amap * const amap = ufi->entry->aref.ar_amap; 1422 struct vm_anon * const anon = anons[flt->centeridx]; 1423 struct uvm_object *uobj; 1424 int error; 1425 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 1426 1427 /* locked: maps(read), amap, anon */ 1428 KASSERT(rw_lock_op(amap->am_lock) == flt->upper_lock_type); 1429 KASSERT(anon->an_lock == amap->am_lock); 1430 1431 /* 1432 * handle case 1: fault on an anon in our amap 1433 */ 1434 1435 UVMHIST_LOG(maphist, " case 1 fault: anon=%#jx", 1436 (uintptr_t)anon, 0, 0, 0); 1437 1438 /* 1439 * no matter if we have case 1A or case 1B we are going to need to 1440 * have the anon's memory resident. ensure that now. 1441 */ 1442 1443 /* 1444 * let uvmfault_anonget do the dirty work. 1445 * if it fails (!OK) it will unlock everything for us. 1446 * if it succeeds, locks are still valid and locked. 1447 * also, if it is OK, then the anon's page is on the queues. 1448 * if the page is on loan from a uvm_object, then anonget will 1449 * lock that object for us if it does not fail. 1450 */ 1451 retry: 1452 error = uvmfault_anonget(ufi, amap, anon); 1453 switch (error) { 1454 case 0: 1455 break; 1456 1457 case ERESTART: 1458 return ERESTART; 1459 1460 case EAGAIN: 1461 kpause("fltagain1", false, hz/2, NULL); 1462 return ERESTART; 1463 1464 case ENOLCK: 1465 /* it needs a write lock: retry */ 1466 error = uvm_fault_upper_upgrade(ufi, flt, amap, NULL); 1467 if (error != 0) { 1468 return error; 1469 } 1470 KASSERT(rw_write_held(amap->am_lock)); 1471 goto retry; 1472 1473 default: 1474 return error; 1475 } 1476 1477 /* 1478 * uobj is non null if the page is on loan from an object (i.e. uobj) 1479 */ 1480 1481 uobj = anon->an_page->uobject; /* locked by anonget if !NULL */ 1482 1483 /* locked: maps(read), amap, anon, uobj(if one) */ 1484 KASSERT(rw_lock_op(amap->am_lock) == flt->upper_lock_type); 1485 KASSERT(anon->an_lock == amap->am_lock); 1486 KASSERT(uobj == NULL || 1487 rw_lock_op(uobj->vmobjlock) == flt->lower_lock_type); 1488 1489 /* 1490 * special handling for loaned pages 1491 */ 1492 1493 if (anon->an_page->loan_count) { 1494 error = uvm_fault_upper_loan(ufi, flt, anon, &uobj); 1495 if (error != 0) 1496 return error; 1497 } 1498 1499 /* 1500 * if we are case 1B then we will need to allocate a new blank 1501 * anon to transfer the data into. note that we have a lock 1502 * on anon, so no one can busy or release the page until we are done. 1503 * also note that the ref count can't drop to zero here because 1504 * it is > 1 and we are only dropping one ref. 1505 * 1506 * in the (hopefully very rare) case that we are out of RAM we 1507 * will unlock, wait for more RAM, and refault. 1508 * 1509 * if we are out of anon VM we kill the process (XXX: could wait?). 1510 */ 1511 1512 if (flt->cow_now && anon->an_ref > 1) { 1513 flt->promote = true; 1514 error = uvm_fault_upper_promote(ufi, flt, uobj, anon); 1515 } else { 1516 error = uvm_fault_upper_direct(ufi, flt, uobj, anon); 1517 } 1518 return error; 1519 } 1520 1521 /* 1522 * uvm_fault_upper_loan: handle loaned upper page. 1523 * 1524 * 1. if not cow'ing now, simply adjust flt->enter_prot. 1525 * 2. if cow'ing now, and if ref count is 1, break loan. 1526 */ 1527 1528 static int 1529 uvm_fault_upper_loan( 1530 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1531 struct vm_anon *anon, struct uvm_object **ruobj) 1532 { 1533 struct vm_amap * const amap = ufi->entry->aref.ar_amap; 1534 int error = 0; 1535 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 1536 1537 if (!flt->cow_now) { 1538 1539 /* 1540 * for read faults on loaned pages we just cap the 1541 * protection at read-only. 1542 */ 1543 1544 flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE; 1545 1546 } else { 1547 /* 1548 * note that we can't allow writes into a loaned page! 1549 * 1550 * if we have a write fault on a loaned page in an 1551 * anon then we need to look at the anon's ref count. 1552 * if it is greater than one then we are going to do 1553 * a normal copy-on-write fault into a new anon (this 1554 * is not a problem). however, if the reference count 1555 * is one (a case where we would normally allow a 1556 * write directly to the page) then we need to kill 1557 * the loan before we continue. 1558 */ 1559 1560 /* >1 case is already ok */ 1561 if (anon->an_ref == 1) { 1562 /* breaking loan requires a write lock. */ 1563 error = uvm_fault_upper_upgrade(ufi, flt, amap, NULL); 1564 if (error != 0) { 1565 return error; 1566 } 1567 KASSERT(rw_write_held(amap->am_lock)); 1568 1569 error = uvm_loanbreak_anon(anon, *ruobj); 1570 if (error != 0) { 1571 uvmfault_unlockall(ufi, amap, *ruobj); 1572 uvm_wait("flt_noram2"); 1573 return ERESTART; 1574 } 1575 /* if we were a loan receiver uobj is gone */ 1576 if (*ruobj) 1577 *ruobj = NULL; 1578 } 1579 } 1580 return error; 1581 } 1582 1583 /* 1584 * uvm_fault_upper_promote: promote upper page. 1585 * 1586 * 1. call uvmfault_promote. 1587 * 2. enqueue page. 1588 * 3. deref. 1589 * 4. pass page to uvm_fault_upper_enter. 1590 */ 1591 1592 static int 1593 uvm_fault_upper_promote( 1594 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1595 struct uvm_object *uobj, struct vm_anon *anon) 1596 { 1597 struct vm_amap * const amap = ufi->entry->aref.ar_amap; 1598 struct vm_anon * const oanon = anon; 1599 struct vm_page *pg; 1600 int error; 1601 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 1602 1603 UVMHIST_LOG(maphist, " case 1B: COW fault",0,0,0,0); 1604 cpu_count(CPU_COUNT_FLT_ACOW, 1); 1605 1606 /* promoting requires a write lock. */ 1607 error = uvm_fault_upper_upgrade(ufi, flt, amap, NULL); 1608 if (error != 0) { 1609 return error; 1610 } 1611 KASSERT(rw_write_held(amap->am_lock)); 1612 1613 error = uvmfault_promote(ufi, oanon, PGO_DONTCARE, &anon, 1614 &flt->anon_spare); 1615 switch (error) { 1616 case 0: 1617 break; 1618 case ERESTART: 1619 return ERESTART; 1620 default: 1621 return error; 1622 } 1623 pg = anon->an_page; 1624 1625 KASSERT(anon->an_lock == oanon->an_lock); 1626 KASSERT((pg->flags & (PG_BUSY | PG_FAKE)) == 0); 1627 1628 /* deref: can not drop to zero here by defn! */ 1629 KASSERT(oanon->an_ref > 1); 1630 oanon->an_ref--; 1631 1632 /* 1633 * note: oanon is still locked, as is the new anon. we 1634 * need to check for this later when we unlock oanon; if 1635 * oanon != anon, we'll have to unlock anon, too. 1636 */ 1637 1638 return uvm_fault_upper_enter(ufi, flt, uobj, anon, pg, oanon); 1639 } 1640 1641 /* 1642 * uvm_fault_upper_direct: handle direct fault. 1643 */ 1644 1645 static int 1646 uvm_fault_upper_direct( 1647 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1648 struct uvm_object *uobj, struct vm_anon *anon) 1649 { 1650 struct vm_anon * const oanon = anon; 1651 struct vm_page *pg; 1652 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 1653 1654 cpu_count(CPU_COUNT_FLT_ANON, 1); 1655 pg = anon->an_page; 1656 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */ 1657 flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE; 1658 1659 return uvm_fault_upper_enter(ufi, flt, uobj, anon, pg, oanon); 1660 } 1661 1662 /* 1663 * uvm_fault_upper_enter: enter h/w mapping of upper page. 1664 */ 1665 1666 static int 1667 uvm_fault_upper_enter( 1668 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 1669 struct uvm_object *uobj, struct vm_anon *anon, struct vm_page *pg, 1670 struct vm_anon *oanon) 1671 { 1672 struct pmap *pmap = ufi->orig_map->pmap; 1673 vaddr_t va = ufi->orig_rvaddr; 1674 struct vm_amap * const amap = ufi->entry->aref.ar_amap; 1675 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 1676 1677 /* locked: maps(read), amap, oanon, anon(if different from oanon) */ 1678 KASSERT(rw_lock_op(amap->am_lock) == flt->upper_lock_type); 1679 KASSERT(anon->an_lock == amap->am_lock); 1680 KASSERT(oanon->an_lock == amap->am_lock); 1681 KASSERT(uobj == NULL || 1682 rw_lock_op(uobj->vmobjlock) == flt->lower_lock_type); 1683 KASSERT(uvm_pagegetdirty(pg) != UVM_PAGE_STATUS_CLEAN); 1684 1685 /* 1686 * now map the page in. 1687 */ 1688 1689 UVMHIST_LOG(maphist, 1690 " MAPPING: anon: pm=%#jx, va=%#jx, pg=%#jx, promote=%jd", 1691 (uintptr_t)pmap, va, (uintptr_t)pg, flt->promote); 1692 if (pmap_enter(pmap, va, VM_PAGE_TO_PHYS(pg), 1693 flt->enter_prot, flt->access_type | PMAP_CANFAIL | 1694 (flt->wire_mapping ? PMAP_WIRED : 0)) != 0) { 1695 1696 /* 1697 * If pmap_enter() fails, it must not leave behind an existing 1698 * pmap entry. In particular, a now-stale entry for a different 1699 * page would leave the pmap inconsistent with the vm_map. 1700 * This is not to imply that pmap_enter() should remove an 1701 * existing mapping in such a situation (since that could create 1702 * different problems, eg. if the existing mapping is wired), 1703 * but rather that the pmap should be designed such that it 1704 * never needs to fail when the new mapping is replacing an 1705 * existing mapping and the new page has no existing mappings. 1706 * 1707 * XXX This can't be asserted safely any more because many 1708 * LWPs and/or many processes could simultaneously fault on 1709 * the same VA and some might succeed. 1710 */ 1711 1712 /* KASSERT(!pmap_extract(pmap, va, NULL)); */ 1713 1714 /* 1715 * ensure that the page is queued in the case that 1716 * we just promoted. 1717 */ 1718 1719 uvm_pagelock(pg); 1720 uvm_pageenqueue(pg); 1721 uvm_pageunlock(pg); 1722 1723 /* 1724 * No need to undo what we did; we can simply think of 1725 * this as the pmap throwing away the mapping information. 1726 * 1727 * We do, however, have to go through the ReFault path, 1728 * as the map may change while we're asleep. 1729 */ 1730 1731 uvmfault_unlockall(ufi, amap, uobj); 1732 if (!uvm_reclaimable()) { 1733 UVMHIST_LOG(maphist, 1734 "<- failed. out of VM",0,0,0,0); 1735 /* XXX instrumentation */ 1736 return ENOMEM; 1737 } 1738 /* XXX instrumentation */ 1739 uvm_wait("flt_pmfail1"); 1740 return ERESTART; 1741 } 1742 1743 uvm_fault_upper_done(ufi, flt, anon, pg); 1744 1745 /* 1746 * done case 1! finish up by unlocking everything and returning success 1747 */ 1748 1749 pmap_update(pmap); 1750 uvmfault_unlockall(ufi, amap, uobj); 1751 return 0; 1752 } 1753 1754 /* 1755 * uvm_fault_upper_done: queue upper center page. 1756 */ 1757 1758 static void 1759 uvm_fault_upper_done( 1760 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 1761 struct vm_anon *anon, struct vm_page *pg) 1762 { 1763 const bool wire_paging = flt->wire_paging; 1764 1765 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 1766 1767 /* 1768 * ... update the page queues. 1769 */ 1770 1771 if (wire_paging) { 1772 uvm_pagelock(pg); 1773 uvm_pagewire(pg); 1774 uvm_pageunlock(pg); 1775 1776 /* 1777 * since the now-wired page cannot be paged out, 1778 * release its swap resources for others to use. 1779 * and since an anon with no swap cannot be clean, 1780 * mark it dirty now. 1781 */ 1782 1783 uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY); 1784 uvm_anon_dropswap(anon); 1785 } else if (uvmpdpol_pageactivate_p(pg)) { 1786 /* 1787 * avoid re-activating the page unless needed, 1788 * to avoid false sharing on multiprocessor. 1789 */ 1790 1791 uvm_pagelock(pg); 1792 uvm_pageactivate(pg); 1793 uvm_pageunlock(pg); 1794 } 1795 } 1796 1797 /* 1798 * uvm_fault_lower_upgrade: upgrade lower lock, reader -> writer 1799 */ 1800 1801 static inline int 1802 uvm_fault_lower_upgrade(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1803 struct vm_amap *amap, struct uvm_object *uobj, struct vm_page *uobjpage) 1804 { 1805 1806 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 1807 1808 KASSERT(uobj != NULL); 1809 KASSERT(flt->lower_lock_type == rw_lock_op(uobj->vmobjlock)); 1810 1811 /* 1812 * fast path. 1813 */ 1814 1815 if (__predict_true(flt->lower_lock_type == RW_WRITER)) { 1816 return 0; 1817 } 1818 1819 /* 1820 * otherwise try for the upgrade. if we don't get it, unlock 1821 * everything, restart the fault and next time around get a writer 1822 * lock. 1823 */ 1824 1825 flt->lower_lock_type = RW_WRITER; 1826 if (__predict_false(!rw_tryupgrade(uobj->vmobjlock))) { 1827 uvmfault_unlockall(ufi, amap, uobj); 1828 cpu_count(CPU_COUNT_FLTNOUP, 1); 1829 UVMHIST_LOG(maphist, " !upgrade lower", 0, 0,0,0); 1830 return ERESTART; 1831 } 1832 cpu_count(CPU_COUNT_FLTUP, 1); 1833 KASSERT(flt->lower_lock_type == rw_lock_op(uobj->vmobjlock)); 1834 return 0; 1835 } 1836 1837 /* 1838 * uvm_fault_lower: handle lower fault. 1839 * 1840 * 1. check uobj 1841 * 1.1. if null, ZFOD. 1842 * 1.2. if not null, look up unnmapped neighbor pages. 1843 * 2. for center page, check if promote. 1844 * 2.1. ZFOD always needs promotion. 1845 * 2.2. other uobjs, when entry is marked COW (usually MAP_PRIVATE vnode). 1846 * 3. if uobj is not ZFOD and page is not found, do i/o. 1847 * 4. dispatch either direct / promote fault. 1848 */ 1849 1850 static int 1851 uvm_fault_lower( 1852 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1853 struct vm_page **pages) 1854 { 1855 struct vm_amap *amap __diagused = ufi->entry->aref.ar_amap; 1856 struct uvm_object *uobj = ufi->entry->object.uvm_obj; 1857 struct vm_page *uobjpage; 1858 int error; 1859 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 1860 1861 /* 1862 * now, if the desired page is not shadowed by the amap and we have 1863 * a backing object that does not have a special fault routine, then 1864 * we ask (with pgo_get) the object for resident pages that we care 1865 * about and attempt to map them in. we do not let pgo_get block 1866 * (PGO_LOCKED). 1867 */ 1868 1869 if (uobj == NULL) { 1870 /* zero fill; don't care neighbor pages */ 1871 uobjpage = NULL; 1872 } else { 1873 uvm_fault_lower_lookup(ufi, flt, pages); 1874 uobjpage = pages[flt->centeridx]; 1875 } 1876 1877 /* 1878 * note that at this point we are done with any front or back pages. 1879 * we are now going to focus on the center page (i.e. the one we've 1880 * faulted on). if we have faulted on the upper (anon) layer 1881 * [i.e. case 1], then the anon we want is anons[centeridx] (we have 1882 * not touched it yet). if we have faulted on the bottom (uobj) 1883 * layer [i.e. case 2] and the page was both present and available, 1884 * then we've got a pointer to it as "uobjpage" and we've already 1885 * made it BUSY. 1886 */ 1887 1888 /* 1889 * locked: 1890 * maps(read), amap(if there), uobj(if !null), uobjpage(if !null) 1891 */ 1892 KASSERT(amap == NULL || 1893 rw_lock_op(amap->am_lock) == flt->upper_lock_type); 1894 KASSERT(uobj == NULL || 1895 rw_lock_op(uobj->vmobjlock) == flt->lower_lock_type); 1896 1897 /* 1898 * note that uobjpage can not be PGO_DONTCARE at this point. we now 1899 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we 1900 * have a backing object, check and see if we are going to promote 1901 * the data up to an anon during the fault. 1902 */ 1903 1904 if (uobj == NULL) { 1905 uobjpage = PGO_DONTCARE; 1906 flt->promote = true; /* always need anon here */ 1907 } else { 1908 KASSERT(uobjpage != PGO_DONTCARE); 1909 flt->promote = flt->cow_now && UVM_ET_ISCOPYONWRITE(ufi->entry); 1910 } 1911 UVMHIST_LOG(maphist, " case 2 fault: promote=%jd, zfill=%jd", 1912 flt->promote, (uobj == NULL), 0,0); 1913 1914 /* 1915 * if uobjpage is not null then we do not need to do I/O to get the 1916 * uobjpage. 1917 * 1918 * if uobjpage is null, then we need to unlock and ask the pager to 1919 * get the data for us. once we have the data, we need to reverify 1920 * the state the world. we are currently not holding any resources. 1921 */ 1922 1923 if (uobjpage) { 1924 /* update rusage counters */ 1925 curlwp->l_ru.ru_minflt++; 1926 } else { 1927 error = uvm_fault_lower_io(ufi, flt, &uobj, &uobjpage); 1928 if (error != 0) 1929 return error; 1930 } 1931 1932 /* 1933 * locked: 1934 * maps(read), amap(if !null), uobj(if !null), uobjpage(if uobj) 1935 */ 1936 KASSERT(amap == NULL || 1937 rw_lock_op(amap->am_lock) == flt->upper_lock_type); 1938 KASSERT(uobj == NULL || 1939 rw_lock_op(uobj->vmobjlock) == flt->lower_lock_type); 1940 1941 /* 1942 * notes: 1943 * - at this point uobjpage can not be NULL 1944 * - at this point uobjpage can not be PG_RELEASED (since we checked 1945 * for it above) 1946 * - at this point uobjpage could be waited on (handle later) 1947 * - uobjpage can be from a different object if tmpfs (vnode vs UAO) 1948 */ 1949 1950 KASSERT(uobjpage != NULL); 1951 KASSERT(uobj == NULL || 1952 uobjpage->uobject->vmobjlock == uobj->vmobjlock); 1953 KASSERT(uobj == NULL || !UVM_OBJ_IS_CLEAN(uobjpage->uobject) || 1954 uvm_pagegetdirty(uobjpage) == UVM_PAGE_STATUS_CLEAN); 1955 1956 if (!flt->promote) { 1957 error = uvm_fault_lower_direct(ufi, flt, uobj, uobjpage); 1958 } else { 1959 error = uvm_fault_lower_promote(ufi, flt, uobj, uobjpage); 1960 } 1961 return error; 1962 } 1963 1964 /* 1965 * uvm_fault_lower_lookup: look up on-memory uobj pages. 1966 * 1967 * 1. get on-memory pages. 1968 * 2. if failed, give up (get only center page later). 1969 * 3. if succeeded, enter h/w mapping of neighbor pages. 1970 */ 1971 1972 static void 1973 uvm_fault_lower_lookup( 1974 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 1975 struct vm_page **pages) 1976 { 1977 struct uvm_object *uobj = ufi->entry->object.uvm_obj; 1978 int lcv, gotpages; 1979 vaddr_t currva; 1980 bool entered; 1981 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 1982 1983 rw_enter(uobj->vmobjlock, flt->lower_lock_type); 1984 1985 /* 1986 * Locked: maps(read), amap(if there), uobj 1987 */ 1988 1989 cpu_count(CPU_COUNT_FLTLGET, 1); 1990 gotpages = flt->npages; 1991 (void) uobj->pgops->pgo_get(uobj, 1992 ufi->entry->offset + flt->startva - ufi->entry->start, 1993 pages, &gotpages, flt->centeridx, 1994 flt->access_type & MASK(ufi->entry), ufi->entry->advice, 1995 PGO_LOCKED); 1996 1997 KASSERT(rw_lock_op(uobj->vmobjlock) == flt->lower_lock_type); 1998 1999 /* 2000 * check for pages to map, if we got any 2001 */ 2002 2003 if (gotpages == 0) { 2004 pages[flt->centeridx] = NULL; 2005 return; 2006 } 2007 2008 entered = false; 2009 currva = flt->startva; 2010 for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) { 2011 struct vm_page *curpg; 2012 2013 curpg = pages[lcv]; 2014 if (curpg == NULL || curpg == PGO_DONTCARE) { 2015 continue; 2016 } 2017 2018 /* 2019 * in the case of tmpfs, the pages might be from a different 2020 * uvm_object. just make sure that they have the same lock. 2021 */ 2022 2023 KASSERT(curpg->uobject->vmobjlock == uobj->vmobjlock); 2024 KASSERT((curpg->flags & PG_BUSY) == 0); 2025 2026 /* 2027 * leave the centre page for later. don't screw with 2028 * existing mappings (needless & expensive). 2029 */ 2030 2031 if (lcv == flt->centeridx) { 2032 UVMHIST_LOG(maphist, " got uobjpage (%#jx) " 2033 "with locked get", (uintptr_t)curpg, 0, 0, 0); 2034 } else if (!pmap_extract(ufi->orig_map->pmap, currva, NULL)) { 2035 uvm_fault_lower_neighbor(ufi, flt, currva, curpg); 2036 entered = true; 2037 } 2038 } 2039 if (entered) { 2040 pmap_update(ufi->orig_map->pmap); 2041 } 2042 } 2043 2044 /* 2045 * uvm_fault_lower_neighbor: enter h/w mapping of lower neighbor page. 2046 */ 2047 2048 static void 2049 uvm_fault_lower_neighbor( 2050 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 2051 vaddr_t currva, struct vm_page *pg) 2052 { 2053 const bool readonly = uvm_pagereadonly_p(pg) || pg->loan_count > 0; 2054 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 2055 2056 /* locked: maps(read), amap(if there), uobj */ 2057 2058 /* 2059 * calling pgo_get with PGO_LOCKED returns us pages which 2060 * are neither busy nor released, so we don't need to check 2061 * for this. we can just directly enter the pages. 2062 * 2063 * there wasn't a direct fault on the page, so avoid the cost of 2064 * activating it. 2065 */ 2066 2067 if (!uvmpdpol_pageisqueued_p(pg) && pg->wire_count == 0) { 2068 uvm_pagelock(pg); 2069 uvm_pageenqueue(pg); 2070 uvm_pageunlock(pg); 2071 } 2072 2073 UVMHIST_LOG(maphist, 2074 " MAPPING: n obj: pm=%#jx, va=%#jx, pg=%#jx", 2075 (uintptr_t)ufi->orig_map->pmap, currva, (uintptr_t)pg, 0); 2076 cpu_count(CPU_COUNT_FLTNOMAP, 1); 2077 2078 /* 2079 * Since this page isn't the page that's actually faulting, 2080 * ignore pmap_enter() failures; it's not critical that we 2081 * enter these right now. 2082 * NOTE: page can't be waited on or PG_RELEASED because we've 2083 * held the lock the whole time we've had the handle. 2084 */ 2085 KASSERT((pg->flags & PG_PAGEOUT) == 0); 2086 KASSERT((pg->flags & PG_RELEASED) == 0); 2087 KASSERT(!UVM_OBJ_IS_CLEAN(pg->uobject) || 2088 uvm_pagegetdirty(pg) == UVM_PAGE_STATUS_CLEAN); 2089 KASSERT((pg->flags & PG_BUSY) == 0); 2090 KASSERT(rw_lock_op(pg->uobject->vmobjlock) == flt->lower_lock_type); 2091 2092 const vm_prot_t mapprot = 2093 readonly ? (flt->enter_prot & ~VM_PROT_WRITE) : 2094 flt->enter_prot & MASK(ufi->entry); 2095 const u_int mapflags = 2096 PMAP_CANFAIL | (flt->wire_mapping ? (mapprot | PMAP_WIRED) : 0); 2097 (void) pmap_enter(ufi->orig_map->pmap, currva, 2098 VM_PAGE_TO_PHYS(pg), mapprot, mapflags); 2099 } 2100 2101 /* 2102 * uvm_fault_lower_io: get lower page from backing store. 2103 * 2104 * 1. unlock everything, because i/o will block. 2105 * 2. call pgo_get. 2106 * 3. if failed, recover. 2107 * 4. if succeeded, relock everything and verify things. 2108 */ 2109 2110 static int 2111 uvm_fault_lower_io( 2112 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 2113 struct uvm_object **ruobj, struct vm_page **ruobjpage) 2114 { 2115 struct vm_amap * const amap = ufi->entry->aref.ar_amap; 2116 struct uvm_object *uobj = *ruobj; 2117 struct vm_page *pg; 2118 bool locked; 2119 int gotpages; 2120 int error; 2121 voff_t uoff; 2122 vm_prot_t access_type; 2123 int advice; 2124 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 2125 2126 /* update rusage counters */ 2127 curlwp->l_ru.ru_majflt++; 2128 2129 /* grab everything we need from the entry before we unlock */ 2130 uoff = (ufi->orig_rvaddr - ufi->entry->start) + ufi->entry->offset; 2131 access_type = flt->access_type & MASK(ufi->entry); 2132 advice = ufi->entry->advice; 2133 2134 /* Locked: maps(read), amap(if there), uobj */ 2135 KASSERT(rw_lock_op(uobj->vmobjlock) == flt->lower_lock_type); 2136 2137 /* Upgrade to a write lock if needed. */ 2138 error = uvm_fault_lower_upgrade(ufi, flt, amap, uobj, NULL); 2139 if (error != 0) { 2140 return error; 2141 } 2142 uvmfault_unlockall(ufi, amap, NULL); 2143 2144 /* Locked: uobj(write) */ 2145 KASSERT(rw_write_held(uobj->vmobjlock)); 2146 2147 cpu_count(CPU_COUNT_FLTGET, 1); 2148 gotpages = 1; 2149 pg = NULL; 2150 error = uobj->pgops->pgo_get(uobj, uoff, &pg, &gotpages, 2151 0, access_type, advice, PGO_SYNCIO); 2152 /* locked: pg(if no error) */ 2153 2154 /* 2155 * recover from I/O 2156 */ 2157 2158 if (error) { 2159 if (error == EAGAIN) { 2160 UVMHIST_LOG(maphist, 2161 " pgo_get says TRY AGAIN!",0,0,0,0); 2162 kpause("fltagain2", false, hz/2, NULL); 2163 return ERESTART; 2164 } 2165 2166 #if 0 2167 KASSERT(error != ERESTART); 2168 #else 2169 /* XXXUEBS don't re-fault? */ 2170 if (error == ERESTART) 2171 error = EIO; 2172 #endif 2173 2174 UVMHIST_LOG(maphist, "<- pgo_get failed (code %jd)", 2175 error, 0,0,0); 2176 return error; 2177 } 2178 2179 /* 2180 * re-verify the state of the world by first trying to relock 2181 * the maps. always relock the object. 2182 */ 2183 2184 locked = uvmfault_relock(ufi); 2185 if (locked && amap) 2186 amap_lock(amap, flt->upper_lock_type); 2187 2188 /* might be changed */ 2189 uobj = pg->uobject; 2190 2191 rw_enter(uobj->vmobjlock, flt->lower_lock_type); 2192 KASSERT((pg->flags & PG_BUSY) != 0); 2193 KASSERT(flt->lower_lock_type == RW_WRITER); 2194 2195 uvm_pagelock(pg); 2196 uvm_pageactivate(pg); 2197 uvm_pageunlock(pg); 2198 2199 /* locked(locked): maps(read), amap(if !null), uobj, pg */ 2200 /* locked(!locked): uobj, pg */ 2201 2202 /* 2203 * verify that the page has not be released and re-verify 2204 * that amap slot is still free. if there is a problem, 2205 * we unlock and clean up. 2206 */ 2207 2208 if ((pg->flags & PG_RELEASED) != 0 || 2209 (locked && amap && amap_lookup(&ufi->entry->aref, 2210 ufi->orig_rvaddr - ufi->entry->start))) { 2211 if (locked) 2212 uvmfault_unlockall(ufi, amap, NULL); 2213 locked = false; 2214 } 2215 2216 /* 2217 * unbusy/release the page. 2218 */ 2219 2220 if ((pg->flags & PG_RELEASED) == 0) { 2221 pg->flags &= ~PG_BUSY; 2222 uvm_pagelock(pg); 2223 uvm_pagewakeup(pg); 2224 uvm_pageunlock(pg); 2225 UVM_PAGE_OWN(pg, NULL); 2226 } else { 2227 cpu_count(CPU_COUNT_FLTPGRELE, 1); 2228 uvm_pagefree(pg); 2229 } 2230 2231 /* 2232 * didn't get the lock? retry. 2233 */ 2234 2235 if (locked == false) { 2236 UVMHIST_LOG(maphist, 2237 " wasn't able to relock after fault: retry", 2238 0,0,0,0); 2239 rw_exit(uobj->vmobjlock); 2240 return ERESTART; 2241 } 2242 2243 /* 2244 * we have the data in pg. we are holding object lock (so the page 2245 * can't be released on us). 2246 */ 2247 2248 /* locked: maps(read), amap(if !null), uobj */ 2249 2250 *ruobj = uobj; 2251 *ruobjpage = pg; 2252 return 0; 2253 } 2254 2255 /* 2256 * uvm_fault_lower_direct: fault lower center page 2257 * 2258 * 1. adjust flt->enter_prot. 2259 * 2. if page is loaned, resolve. 2260 */ 2261 2262 int 2263 uvm_fault_lower_direct( 2264 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 2265 struct uvm_object *uobj, struct vm_page *uobjpage) 2266 { 2267 struct vm_page *pg; 2268 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 2269 2270 /* 2271 * we are not promoting. if the mapping is COW ensure that we 2272 * don't give more access than we should (e.g. when doing a read 2273 * fault on a COPYONWRITE mapping we want to map the COW page in 2274 * R/O even though the entry protection could be R/W). 2275 * 2276 * set "pg" to the page we want to map in (uobjpage, usually) 2277 */ 2278 2279 cpu_count(CPU_COUNT_FLT_OBJ, 1); 2280 if (UVM_ET_ISCOPYONWRITE(ufi->entry) || 2281 UVM_OBJ_NEEDS_WRITEFAULT(uobjpage->uobject)) 2282 flt->enter_prot &= ~VM_PROT_WRITE; 2283 pg = uobjpage; /* map in the actual object */ 2284 2285 KASSERT(uobjpage != PGO_DONTCARE); 2286 2287 /* 2288 * we are faulting directly on the page. be careful 2289 * about writing to loaned pages... 2290 */ 2291 2292 if (uobjpage->loan_count) { 2293 uvm_fault_lower_direct_loan(ufi, flt, uobj, &pg, &uobjpage); 2294 } 2295 KASSERT(pg == uobjpage); 2296 KASSERT((pg->flags & PG_BUSY) == 0); 2297 return uvm_fault_lower_enter(ufi, flt, uobj, NULL, pg); 2298 } 2299 2300 /* 2301 * uvm_fault_lower_direct_loan: resolve loaned page. 2302 * 2303 * 1. if not cow'ing, adjust flt->enter_prot. 2304 * 2. if cow'ing, break loan. 2305 */ 2306 2307 static int 2308 uvm_fault_lower_direct_loan( 2309 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 2310 struct uvm_object *uobj, struct vm_page **rpg, 2311 struct vm_page **ruobjpage) 2312 { 2313 struct vm_amap * const amap = ufi->entry->aref.ar_amap; 2314 struct vm_page *pg; 2315 struct vm_page *uobjpage = *ruobjpage; 2316 int error; 2317 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 2318 2319 if (!flt->cow_now) { 2320 /* read fault: cap the protection at readonly */ 2321 /* cap! */ 2322 flt->enter_prot = flt->enter_prot & ~VM_PROT_WRITE; 2323 } else { 2324 /* 2325 * write fault: must break the loan here. to do this 2326 * we need a write lock on the object. 2327 */ 2328 2329 error = uvm_fault_lower_upgrade(ufi, flt, amap, uobj, uobjpage); 2330 if (error != 0) { 2331 return error; 2332 } 2333 KASSERT(rw_write_held(uobj->vmobjlock)); 2334 2335 pg = uvm_loanbreak(uobjpage); 2336 if (pg == NULL) { 2337 2338 uvmfault_unlockall(ufi, amap, uobj); 2339 UVMHIST_LOG(maphist, 2340 " out of RAM breaking loan, waiting", 2341 0,0,0,0); 2342 cpu_count(CPU_COUNT_FLTNORAM, 1); 2343 uvm_wait("flt_noram4"); 2344 return ERESTART; 2345 } 2346 *rpg = pg; 2347 *ruobjpage = pg; 2348 2349 /* 2350 * drop ownership of page while still holding object lock, 2351 * which won't be dropped until the page is entered. 2352 */ 2353 2354 uvm_pagelock(pg); 2355 uvm_pagewakeup(pg); 2356 uvm_pageunlock(pg); 2357 pg->flags &= ~PG_BUSY; 2358 UVM_PAGE_OWN(pg, NULL); 2359 } 2360 return 0; 2361 } 2362 2363 /* 2364 * uvm_fault_lower_promote: promote lower page. 2365 * 2366 * 1. call uvmfault_promote. 2367 * 2. fill in data. 2368 * 3. if not ZFOD, dispose old page. 2369 */ 2370 2371 int 2372 uvm_fault_lower_promote( 2373 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 2374 struct uvm_object *uobj, struct vm_page *uobjpage) 2375 { 2376 struct vm_amap * const amap = ufi->entry->aref.ar_amap; 2377 struct vm_anon *anon; 2378 struct vm_page *pg; 2379 int error; 2380 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 2381 2382 KASSERT(amap != NULL); 2383 2384 /* promoting requires a write lock. */ 2385 error = uvm_fault_upper_upgrade(ufi, flt, amap, uobj); 2386 if (error != 0) { 2387 return error; 2388 } 2389 KASSERT(rw_write_held(amap->am_lock)); 2390 KASSERT(uobj == NULL || 2391 rw_lock_op(uobj->vmobjlock) == flt->lower_lock_type); 2392 2393 /* 2394 * If we are going to promote the data to an anon we 2395 * allocate a blank anon here and plug it into our amap. 2396 */ 2397 error = uvmfault_promote(ufi, NULL, uobjpage, &anon, &flt->anon_spare); 2398 switch (error) { 2399 case 0: 2400 break; 2401 case ERESTART: 2402 return ERESTART; 2403 default: 2404 return error; 2405 } 2406 2407 pg = anon->an_page; 2408 2409 /* 2410 * Fill in the data. 2411 */ 2412 2413 if (uobjpage != PGO_DONTCARE) { 2414 cpu_count(CPU_COUNT_FLT_PRCOPY, 1); 2415 2416 /* 2417 * promote to shared amap? make sure all sharing 2418 * procs see it 2419 */ 2420 2421 if ((amap_flags(amap) & AMAP_SHARED) != 0) { 2422 pmap_page_protect(uobjpage, VM_PROT_NONE); 2423 /* 2424 * XXX: PAGE MIGHT BE WIRED! 2425 */ 2426 } 2427 2428 UVMHIST_LOG(maphist, 2429 " promote uobjpage %#jx to anon/page %#jx/%#jx", 2430 (uintptr_t)uobjpage, (uintptr_t)anon, (uintptr_t)pg, 0); 2431 2432 } else { 2433 cpu_count(CPU_COUNT_FLT_PRZERO, 1); 2434 2435 /* 2436 * Page is zero'd and marked dirty by 2437 * uvmfault_promote(). 2438 */ 2439 2440 UVMHIST_LOG(maphist," zero fill anon/page %#jx/%#jx", 2441 (uintptr_t)anon, (uintptr_t)pg, 0, 0); 2442 } 2443 2444 return uvm_fault_lower_enter(ufi, flt, uobj, anon, pg); 2445 } 2446 2447 /* 2448 * uvm_fault_lower_enter: enter h/w mapping of lower page or anon page promoted 2449 * from the lower page. 2450 */ 2451 2452 int 2453 uvm_fault_lower_enter( 2454 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 2455 struct uvm_object *uobj, 2456 struct vm_anon *anon, struct vm_page *pg) 2457 { 2458 struct vm_amap * const amap = ufi->entry->aref.ar_amap; 2459 const bool readonly = uvm_pagereadonly_p(pg); 2460 int error; 2461 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 2462 2463 /* 2464 * Locked: 2465 * 2466 * maps(read), amap(if !null), uobj(if !null), 2467 * anon(if !null), pg(if anon), unlock_uobj(if !null) 2468 * 2469 * anon must be write locked (promotion). uobj can be either. 2470 * 2471 * Note: pg is either the uobjpage or the new page in the new anon. 2472 */ 2473 2474 KASSERT(amap == NULL || 2475 rw_lock_op(amap->am_lock) == flt->upper_lock_type); 2476 KASSERT(uobj == NULL || 2477 rw_lock_op(uobj->vmobjlock) == flt->lower_lock_type); 2478 KASSERT(anon == NULL || anon->an_lock == amap->am_lock); 2479 2480 /* 2481 * note that pg can't be PG_RELEASED or PG_BUSY since we did 2482 * not drop the object lock since the last time we checked. 2483 */ 2484 2485 KASSERT((pg->flags & PG_RELEASED) == 0); 2486 KASSERT((pg->flags & PG_BUSY) == 0); 2487 2488 /* 2489 * all resources are present. we can now map it in and free our 2490 * resources. 2491 */ 2492 2493 UVMHIST_LOG(maphist, 2494 " MAPPING: case2: pm=%#jx, va=%#jx, pg=%#jx, promote=%jd", 2495 (uintptr_t)ufi->orig_map->pmap, ufi->orig_rvaddr, 2496 (uintptr_t)pg, flt->promote); 2497 KASSERTMSG((flt->access_type & VM_PROT_WRITE) == 0 || !readonly, 2498 "promote=%u cow_now=%u access_type=%x enter_prot=%x cow=%u " 2499 "entry=%p map=%p orig_rvaddr=%p pg=%p", 2500 flt->promote, flt->cow_now, flt->access_type, flt->enter_prot, 2501 UVM_ET_ISCOPYONWRITE(ufi->entry), ufi->entry, ufi->orig_map, 2502 (void *)ufi->orig_rvaddr, pg); 2503 KASSERT((flt->access_type & VM_PROT_WRITE) == 0 || !readonly); 2504 if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr, 2505 VM_PAGE_TO_PHYS(pg), 2506 readonly ? flt->enter_prot & ~VM_PROT_WRITE : flt->enter_prot, 2507 flt->access_type | PMAP_CANFAIL | 2508 (flt->wire_mapping ? PMAP_WIRED : 0)) != 0) { 2509 2510 /* 2511 * No need to undo what we did; we can simply think of 2512 * this as the pmap throwing away the mapping information. 2513 * 2514 * We do, however, have to go through the ReFault path, 2515 * as the map may change while we're asleep. 2516 */ 2517 2518 /* 2519 * ensure that the page is queued in the case that 2520 * we just promoted the page. 2521 */ 2522 2523 if (anon != NULL) { 2524 uvm_pagelock(pg); 2525 uvm_pageenqueue(pg); 2526 uvm_pagewakeup(pg); 2527 uvm_pageunlock(pg); 2528 } 2529 2530 uvmfault_unlockall(ufi, amap, uobj); 2531 if (!uvm_reclaimable()) { 2532 UVMHIST_LOG(maphist, 2533 "<- failed. out of VM",0,0,0,0); 2534 /* XXX instrumentation */ 2535 error = ENOMEM; 2536 return error; 2537 } 2538 /* XXX instrumentation */ 2539 uvm_wait("flt_pmfail2"); 2540 return ERESTART; 2541 } 2542 2543 uvm_fault_lower_done(ufi, flt, uobj, pg); 2544 pmap_update(ufi->orig_map->pmap); 2545 uvmfault_unlockall(ufi, amap, uobj); 2546 2547 UVMHIST_LOG(maphist, "<- done (SUCCESS!)",0,0,0,0); 2548 return 0; 2549 } 2550 2551 /* 2552 * uvm_fault_lower_done: queue lower center page. 2553 */ 2554 2555 void 2556 uvm_fault_lower_done( 2557 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 2558 struct uvm_object *uobj, struct vm_page *pg) 2559 { 2560 2561 UVMHIST_FUNC(__func__); UVMHIST_CALLED(maphist); 2562 2563 if (flt->wire_paging) { 2564 uvm_pagelock(pg); 2565 uvm_pagewire(pg); 2566 uvm_pageunlock(pg); 2567 if (pg->flags & PG_AOBJ) { 2568 2569 /* 2570 * since the now-wired page cannot be paged out, 2571 * release its swap resources for others to use. 2572 * since an aobj page with no swap cannot be clean, 2573 * mark it dirty now. 2574 * 2575 * use pg->uobject here. if the page is from a 2576 * tmpfs vnode, the pages are backed by its UAO and 2577 * not the vnode. 2578 */ 2579 2580 KASSERT(uobj != NULL); 2581 KASSERT(uobj->vmobjlock == pg->uobject->vmobjlock); 2582 uvm_pagemarkdirty(pg, UVM_PAGE_STATUS_DIRTY); 2583 uao_dropswap(pg->uobject, pg->offset >> PAGE_SHIFT); 2584 } 2585 } else if (uvmpdpol_pageactivate_p(pg)) { 2586 /* 2587 * avoid re-activating the page unless needed, 2588 * to avoid false sharing on multiprocessor. 2589 */ 2590 2591 uvm_pagelock(pg); 2592 uvm_pageactivate(pg); 2593 uvm_pageunlock(pg); 2594 } 2595 } 2596 2597 2598 /* 2599 * uvm_fault_wire: wire down a range of virtual addresses in a map. 2600 * 2601 * => map may be read-locked by caller, but MUST NOT be write-locked. 2602 * => if map is read-locked, any operations which may cause map to 2603 * be write-locked in uvm_fault() must be taken care of by 2604 * the caller. See uvm_map_pageable(). 2605 */ 2606 2607 int 2608 uvm_fault_wire(struct vm_map *map, vaddr_t start, vaddr_t end, 2609 vm_prot_t access_type, int maxprot) 2610 { 2611 vaddr_t va; 2612 int error; 2613 2614 /* 2615 * now fault it in a page at a time. if the fault fails then we have 2616 * to undo what we have done. note that in uvm_fault VM_PROT_NONE 2617 * is replaced with the max protection if fault_type is VM_FAULT_WIRE. 2618 */ 2619 2620 /* 2621 * XXX work around overflowing a vaddr_t. this prevents us from 2622 * wiring the last page in the address space, though. 2623 */ 2624 if (start > end) { 2625 return EFAULT; 2626 } 2627 2628 for (va = start; va < end; va += PAGE_SIZE) { 2629 error = uvm_fault_internal(map, va, access_type, 2630 (maxprot ? UVM_FAULT_MAXPROT : 0) | UVM_FAULT_WIRE); 2631 if (error) { 2632 if (va != start) { 2633 uvm_fault_unwire(map, start, va); 2634 } 2635 return error; 2636 } 2637 } 2638 return 0; 2639 } 2640 2641 /* 2642 * uvm_fault_unwire(): unwire range of virtual space. 2643 */ 2644 2645 void 2646 uvm_fault_unwire(struct vm_map *map, vaddr_t start, vaddr_t end) 2647 { 2648 vm_map_lock_read(map); 2649 uvm_fault_unwire_locked(map, start, end); 2650 vm_map_unlock_read(map); 2651 } 2652 2653 /* 2654 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire(). 2655 * 2656 * => map must be at least read-locked. 2657 */ 2658 2659 void 2660 uvm_fault_unwire_locked(struct vm_map *map, vaddr_t start, vaddr_t end) 2661 { 2662 struct vm_map_entry *entry, *oentry; 2663 pmap_t pmap = vm_map_pmap(map); 2664 vaddr_t va; 2665 paddr_t pa; 2666 struct vm_page *pg; 2667 2668 /* 2669 * we assume that the area we are unwiring has actually been wired 2670 * in the first place. this means that we should be able to extract 2671 * the PAs from the pmap. we also lock out the page daemon so that 2672 * we can call uvm_pageunwire. 2673 */ 2674 2675 /* 2676 * find the beginning map entry for the region. 2677 */ 2678 2679 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map)); 2680 if (uvm_map_lookup_entry(map, start, &entry) == false) 2681 panic("uvm_fault_unwire_locked: address not in map"); 2682 2683 oentry = NULL; 2684 for (va = start; va < end; va += PAGE_SIZE) { 2685 2686 /* 2687 * find the map entry for the current address. 2688 */ 2689 2690 KASSERT(va >= entry->start); 2691 while (va >= entry->end) { 2692 KASSERT(entry->next != &map->header && 2693 entry->next->start <= entry->end); 2694 entry = entry->next; 2695 } 2696 2697 /* 2698 * lock it. 2699 */ 2700 2701 if (entry != oentry) { 2702 if (oentry != NULL) { 2703 uvm_map_unlock_entry(oentry); 2704 } 2705 uvm_map_lock_entry(entry, RW_WRITER); 2706 oentry = entry; 2707 } 2708 2709 /* 2710 * if the entry is no longer wired, tell the pmap. 2711 */ 2712 2713 if (!pmap_extract(pmap, va, &pa)) 2714 continue; 2715 2716 if (VM_MAPENT_ISWIRED(entry) == 0) 2717 pmap_unwire(pmap, va); 2718 2719 pg = PHYS_TO_VM_PAGE(pa); 2720 if (pg) { 2721 uvm_pagelock(pg); 2722 uvm_pageunwire(pg); 2723 uvm_pageunlock(pg); 2724 } 2725 } 2726 2727 if (oentry != NULL) { 2728 uvm_map_unlock_entry(entry); 2729 } 2730 } 2731