1 /* $OpenBSD: uvm_fault.c,v 1.151 2024/12/04 09:19:11 mpi Exp $ */ 2 /* $NetBSD: uvm_fault.c,v 1.51 2000/08/06 00:22:53 thorpej Exp $ */ 3 4 /* 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp 29 */ 30 31 /* 32 * uvm_fault.c: fault handler 33 */ 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/percpu.h> 39 #include <sys/proc.h> 40 #include <sys/malloc.h> 41 #include <sys/mman.h> 42 #include <sys/tracepoint.h> 43 44 #include <uvm/uvm.h> 45 46 /* 47 * 48 * a word on page faults: 49 * 50 * types of page faults we handle: 51 * 52 * CASE 1: upper layer faults CASE 2: lower layer faults 53 * 54 * CASE 1A CASE 1B CASE 2A CASE 2B 55 * read/write1 write>1 read/write +-cow_write/zero 56 * | | | | 57 * +--|--+ +--|--+ +-----+ + | + | +-----+ 58 * amap | V | | ---------> new | | | | ^ | 59 * +-----+ +-----+ +-----+ + | + | +--|--+ 60 * | | | 61 * +-----+ +-----+ +--|--+ | +--|--+ 62 * uobj | d/c | | d/c | | V | +----+ | 63 * +-----+ +-----+ +-----+ +-----+ 64 * 65 * d/c = don't care 66 * 67 * case [0]: layerless fault 68 * no amap or uobj is present. this is an error. 69 * 70 * case [1]: upper layer fault [anon active] 71 * 1A: [read] or [write with anon->an_ref == 1] 72 * I/O takes place in upper level anon and uobj is not touched. 73 * 1B: [write with anon->an_ref > 1] 74 * new anon is alloc'd and data is copied off ["COW"] 75 * 76 * case [2]: lower layer fault [uobj] 77 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area] 78 * I/O takes place directly in object. 79 * 2B: [write to copy_on_write] or [read on NULL uobj] 80 * data is "promoted" from uobj to a new anon. 81 * if uobj is null, then we zero fill. 82 * 83 * we follow the standard UVM locking protocol ordering: 84 * 85 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ) 86 * we hold a PG_BUSY page if we unlock for I/O 87 * 88 * 89 * the code is structured as follows: 90 * 91 * - init the "IN" params in the ufi structure 92 * ReFault: (ERESTART returned to the loop in uvm_fault) 93 * - do lookups [locks maps], check protection, handle needs_copy 94 * - check for case 0 fault (error) 95 * - establish "range" of fault 96 * - if we have an amap lock it and extract the anons 97 * - if sequential advice deactivate pages behind us 98 * - at the same time check pmap for unmapped areas and anon for pages 99 * that we could map in (and do map it if found) 100 * - check object for resident pages that we could map in 101 * - if (case 2) goto Case2 102 * - >>> handle case 1 103 * - ensure source anon is resident in RAM 104 * - if case 1B alloc new anon and copy from source 105 * - map the correct page in 106 * Case2: 107 * - >>> handle case 2 108 * - ensure source page is resident (if uobj) 109 * - if case 2B alloc new anon and copy from source (could be zero 110 * fill if uobj == NULL) 111 * - map the correct page in 112 * - done! 113 * 114 * note on paging: 115 * if we have to do I/O we place a PG_BUSY page in the correct object, 116 * unlock everything, and do the I/O. when I/O is done we must reverify 117 * the state of the world before assuming that our data structures are 118 * valid. [because mappings could change while the map is unlocked] 119 * 120 * alternative 1: unbusy the page in question and restart the page fault 121 * from the top (ReFault). this is easy but does not take advantage 122 * of the information that we already have from our previous lookup, 123 * although it is possible that the "hints" in the vm_map will help here. 124 * 125 * alternative 2: the system already keeps track of a "version" number of 126 * a map. [i.e. every time you write-lock a map (e.g. to change a 127 * mapping) you bump the version number up by one...] so, we can save 128 * the version number of the map before we release the lock and start I/O. 129 * then when I/O is done we can relock and check the version numbers 130 * to see if anything changed. this might save us some over 1 because 131 * we don't have to unbusy the page and may be less compares(?). 132 * 133 * alternative 3: put in backpointers or a way to "hold" part of a map 134 * in place while I/O is in progress. this could be complex to 135 * implement (especially with structures like amap that can be referenced 136 * by multiple map entries, and figuring out what should wait could be 137 * complex as well...). 138 * 139 * we use alternative 2. given that we are multi-threaded now we may want 140 * to reconsider the choice. 141 */ 142 143 /* 144 * local data structures 145 */ 146 struct uvm_advice { 147 int nback; 148 int nforw; 149 }; 150 151 /* 152 * page range array: set up in uvmfault_init(). 153 */ 154 static struct uvm_advice uvmadvice[MADV_MASK + 1]; 155 156 #define UVM_MAXRANGE 16 /* must be max() of nback+nforw+1 */ 157 158 /* 159 * private prototypes 160 */ 161 static void uvmfault_amapcopy(struct uvm_faultinfo *); 162 static inline void uvmfault_anonflush(struct vm_anon **, int); 163 void uvmfault_unlockmaps(struct uvm_faultinfo *, boolean_t); 164 void uvmfault_update_stats(struct uvm_faultinfo *); 165 166 /* 167 * inline functions 168 */ 169 /* 170 * uvmfault_anonflush: try and deactivate pages in specified anons 171 * 172 * => does not have to deactivate page if it is busy 173 */ 174 static inline void 175 uvmfault_anonflush(struct vm_anon **anons, int n) 176 { 177 int lcv; 178 struct vm_page *pg; 179 180 for (lcv = 0; lcv < n; lcv++) { 181 if (anons[lcv] == NULL) 182 continue; 183 KASSERT(rw_lock_held(anons[lcv]->an_lock)); 184 pg = anons[lcv]->an_page; 185 if (pg && (pg->pg_flags & PG_BUSY) == 0) { 186 uvm_lock_pageq(); 187 if (pg->wire_count == 0) { 188 pmap_page_protect(pg, PROT_NONE); 189 uvm_pagedeactivate(pg); 190 } 191 uvm_unlock_pageq(); 192 } 193 } 194 } 195 196 /* 197 * normal functions 198 */ 199 /* 200 * uvmfault_init: compute proper values for the uvmadvice[] array. 201 */ 202 void 203 uvmfault_init(void) 204 { 205 int npages; 206 207 npages = atop(16384); 208 if (npages > 0) { 209 KASSERT(npages <= UVM_MAXRANGE / 2); 210 uvmadvice[MADV_NORMAL].nforw = npages; 211 uvmadvice[MADV_NORMAL].nback = npages - 1; 212 } 213 214 npages = atop(32768); 215 if (npages > 0) { 216 KASSERT(npages <= UVM_MAXRANGE / 2); 217 uvmadvice[MADV_SEQUENTIAL].nforw = npages - 1; 218 uvmadvice[MADV_SEQUENTIAL].nback = npages; 219 } 220 } 221 222 /* 223 * uvmfault_amapcopy: clear "needs_copy" in a map. 224 * 225 * => called with VM data structures unlocked (usually, see below) 226 * => we get a write lock on the maps and clear needs_copy for a VA 227 * => if we are out of RAM we sleep (waiting for more) 228 */ 229 static void 230 uvmfault_amapcopy(struct uvm_faultinfo *ufi) 231 { 232 for (;;) { 233 /* 234 * no mapping? give up. 235 */ 236 if (uvmfault_lookup(ufi, TRUE) == FALSE) 237 return; 238 239 /* 240 * copy if needed. 241 */ 242 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) 243 amap_copy(ufi->map, ufi->entry, M_NOWAIT, 244 UVM_ET_ISSTACK(ufi->entry) ? FALSE : TRUE, 245 ufi->orig_rvaddr, ufi->orig_rvaddr + 1); 246 247 /* 248 * didn't work? must be out of RAM. unlock and sleep. 249 */ 250 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 251 uvmfault_unlockmaps(ufi, TRUE); 252 uvm_wait("fltamapcopy"); 253 continue; 254 } 255 256 /* 257 * got it! unlock and return. 258 */ 259 uvmfault_unlockmaps(ufi, TRUE); 260 return; 261 } 262 /*NOTREACHED*/ 263 } 264 265 /* 266 * uvmfault_anonget: get data in an anon into a non-busy, non-released 267 * page in that anon. 268 * 269 * => Map, amap and thus anon should be locked by caller. 270 * => If we fail, we unlock everything and error is returned. 271 * => If we are successful, return with everything still locked. 272 * => We do not move the page on the queues [gets moved later]. If we 273 * allocate a new page [we_own], it gets put on the queues. Either way, 274 * the result is that the page is on the queues at return time 275 */ 276 int 277 uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap, 278 struct vm_anon *anon) 279 { 280 struct vm_page *pg; 281 int error; 282 283 KASSERT(rw_lock_held(anon->an_lock)); 284 KASSERT(anon->an_lock == amap->am_lock); 285 286 /* Increment the counters.*/ 287 counters_inc(uvmexp_counters, flt_anget); 288 if (anon->an_page) { 289 curproc->p_ru.ru_minflt++; 290 } else { 291 curproc->p_ru.ru_majflt++; 292 } 293 error = 0; 294 295 /* 296 * Loop until we get the anon data, or fail. 297 */ 298 for (;;) { 299 boolean_t we_own, locked; 300 /* 301 * Note: 'we_own' will become true if we set PG_BUSY on a page. 302 */ 303 we_own = FALSE; 304 pg = anon->an_page; 305 306 /* 307 * Is page resident? Make sure it is not busy/released. 308 */ 309 if (pg) { 310 KASSERT(pg->pg_flags & PQ_ANON); 311 KASSERT(pg->uanon == anon); 312 313 /* 314 * if the page is busy, we drop all the locks and 315 * try again. 316 */ 317 if ((pg->pg_flags & (PG_BUSY|PG_RELEASED)) == 0) 318 return 0; 319 counters_inc(uvmexp_counters, flt_pgwait); 320 321 /* 322 * The last unlock must be an atomic unlock and wait 323 * on the owner of page. 324 */ 325 if (pg->uobject) { 326 /* Owner of page is UVM object. */ 327 uvmfault_unlockall(ufi, amap, NULL); 328 uvm_pagewait(pg, pg->uobject->vmobjlock, 329 "anonget1"); 330 } else { 331 /* Owner of page is anon. */ 332 uvmfault_unlockall(ufi, NULL, NULL); 333 uvm_pagewait(pg, anon->an_lock, "anonget2"); 334 } 335 } else { 336 /* 337 * No page, therefore allocate one. 338 */ 339 pg = uvm_pagealloc(NULL, 0, anon, 0); 340 if (pg == NULL) { 341 /* Out of memory. Wait a little. */ 342 uvmfault_unlockall(ufi, amap, NULL); 343 counters_inc(uvmexp_counters, flt_noram); 344 uvm_wait("flt_noram1"); 345 } else { 346 /* PG_BUSY bit is set. */ 347 we_own = TRUE; 348 uvmfault_unlockall(ufi, amap, NULL); 349 350 /* 351 * Pass a PG_BUSY+PG_FAKE+PG_CLEAN page into 352 * the uvm_swap_get() function with all data 353 * structures unlocked. Note that it is OK 354 * to read an_swslot here, because we hold 355 * PG_BUSY on the page. 356 */ 357 counters_inc(uvmexp_counters, pageins); 358 error = uvm_swap_get(pg, anon->an_swslot, 359 PGO_SYNCIO); 360 361 /* 362 * We clean up after the I/O below in the 363 * 'we_own' case. 364 */ 365 } 366 } 367 368 /* 369 * Re-lock the map and anon. 370 */ 371 locked = uvmfault_relock(ufi); 372 if (locked || we_own) { 373 rw_enter(anon->an_lock, RW_WRITE); 374 } 375 376 /* 377 * If we own the page (i.e. we set PG_BUSY), then we need 378 * to clean up after the I/O. There are three cases to 379 * consider: 380 * 381 * 1) Page was released during I/O: free anon and ReFault. 382 * 2) I/O not OK. Free the page and cause the fault to fail. 383 * 3) I/O OK! Activate the page and sync with the non-we_own 384 * case (i.e. drop anon lock if not locked). 385 */ 386 if (we_own) { 387 if (pg->pg_flags & PG_WANTED) { 388 wakeup(pg); 389 } 390 391 /* 392 * if we were RELEASED during I/O, then our anon is 393 * no longer part of an amap. we need to free the 394 * anon and try again. 395 */ 396 if (pg->pg_flags & PG_RELEASED) { 397 KASSERT(anon->an_ref == 0); 398 /* 399 * Released while we had unlocked amap. 400 */ 401 if (locked) 402 uvmfault_unlockall(ufi, NULL, NULL); 403 uvm_anon_release(anon); /* frees page for us */ 404 counters_inc(uvmexp_counters, flt_pgrele); 405 return ERESTART; /* refault! */ 406 } 407 408 if (error != VM_PAGER_OK) { 409 KASSERT(error != VM_PAGER_PEND); 410 411 /* remove page from anon */ 412 anon->an_page = NULL; 413 414 /* 415 * Remove the swap slot from the anon and 416 * mark the anon as having no real slot. 417 * Do not free the swap slot, thus preventing 418 * it from being used again. 419 */ 420 uvm_swap_markbad(anon->an_swslot, 1); 421 anon->an_swslot = SWSLOT_BAD; 422 423 /* 424 * Note: page was never !PG_BUSY, so it 425 * cannot be mapped and thus no need to 426 * pmap_page_protect() it. 427 */ 428 uvm_lock_pageq(); 429 uvm_pagefree(pg); 430 uvm_unlock_pageq(); 431 432 if (locked) { 433 uvmfault_unlockall(ufi, NULL, NULL); 434 } 435 rw_exit(anon->an_lock); 436 /* 437 * An error occurred while trying to bring 438 * in the page -- this is the only error we 439 * return right now. 440 */ 441 return EACCES; /* XXX */ 442 } 443 444 /* 445 * We have successfully read the page, activate it. 446 */ 447 pmap_clear_modify(pg); 448 uvm_lock_pageq(); 449 uvm_pageactivate(pg); 450 uvm_unlock_pageq(); 451 atomic_clearbits_int(&pg->pg_flags, 452 PG_WANTED|PG_BUSY|PG_FAKE); 453 UVM_PAGE_OWN(pg, NULL); 454 } 455 456 /* 457 * We were not able to re-lock the map - restart the fault. 458 */ 459 if (!locked) { 460 if (we_own) { 461 rw_exit(anon->an_lock); 462 } 463 return ERESTART; 464 } 465 466 /* 467 * Verify that no one has touched the amap and moved 468 * the anon on us. 469 */ 470 if (ufi != NULL && amap_lookup(&ufi->entry->aref, 471 ufi->orig_rvaddr - ufi->entry->start) != anon) { 472 473 uvmfault_unlockall(ufi, amap, NULL); 474 return ERESTART; 475 } 476 477 /* 478 * Retry.. 479 */ 480 counters_inc(uvmexp_counters, flt_anretry); 481 continue; 482 483 } 484 /*NOTREACHED*/ 485 } 486 487 /* 488 * Update statistics after fault resolution. 489 * - maxrss 490 */ 491 void 492 uvmfault_update_stats(struct uvm_faultinfo *ufi) 493 { 494 struct vm_map *map; 495 struct proc *p; 496 vsize_t res; 497 498 map = ufi->orig_map; 499 500 /* 501 * If this is a nested pmap (eg, a virtual machine pmap managed 502 * by vmm(4) on amd64/i386), don't do any updating, just return. 503 * 504 * pmap_nested() on other archs is #defined to 0, so this is a 505 * no-op. 506 */ 507 if (pmap_nested(map->pmap)) 508 return; 509 510 /* Update the maxrss for the process. */ 511 if (map->flags & VM_MAP_ISVMSPACE) { 512 p = curproc; 513 KASSERT(p != NULL && &p->p_vmspace->vm_map == map); 514 515 res = pmap_resident_count(map->pmap); 516 /* Convert res from pages to kilobytes. */ 517 res <<= (PAGE_SHIFT - 10); 518 519 if (p->p_ru.ru_maxrss < res) 520 p->p_ru.ru_maxrss = res; 521 } 522 } 523 524 /* 525 * F A U L T - m a i n e n t r y p o i n t 526 */ 527 528 /* 529 * uvm_fault: page fault handler 530 * 531 * => called from MD code to resolve a page fault 532 * => VM data structures usually should be unlocked. however, it is 533 * possible to call here with the main map locked if the caller 534 * gets a write lock, sets it recursive, and then calls us (c.f. 535 * uvm_map_pageable). this should be avoided because it keeps 536 * the map locked off during I/O. 537 * => MUST NEVER BE CALLED IN INTERRUPT CONTEXT 538 */ 539 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 540 ~PROT_WRITE : PROT_MASK) 541 struct uvm_faultctx { 542 /* 543 * the following members are set up by uvm_fault_check() and 544 * read-only after that. 545 */ 546 vm_prot_t enter_prot; 547 vm_prot_t access_type; 548 vaddr_t startva; 549 int npages; 550 int centeridx; 551 boolean_t narrow; 552 boolean_t wired; 553 paddr_t pa_flags; 554 boolean_t promote; 555 }; 556 557 int uvm_fault_check( 558 struct uvm_faultinfo *, struct uvm_faultctx *, 559 struct vm_anon ***, vm_fault_t); 560 561 int uvm_fault_upper( 562 struct uvm_faultinfo *, struct uvm_faultctx *, 563 struct vm_anon **); 564 boolean_t uvm_fault_upper_lookup( 565 struct uvm_faultinfo *, const struct uvm_faultctx *, 566 struct vm_anon **, struct vm_page **); 567 568 int uvm_fault_lower( 569 struct uvm_faultinfo *, struct uvm_faultctx *, 570 struct vm_page **); 571 int uvm_fault_lower_io( 572 struct uvm_faultinfo *, struct uvm_faultctx *, 573 struct uvm_object **, struct vm_page **); 574 575 int 576 uvm_fault(vm_map_t orig_map, vaddr_t vaddr, vm_fault_t fault_type, 577 vm_prot_t access_type) 578 { 579 struct uvm_faultinfo ufi; 580 struct uvm_faultctx flt; 581 boolean_t shadowed; 582 struct vm_anon *anons_store[UVM_MAXRANGE], **anons; 583 struct vm_page *pages[UVM_MAXRANGE]; 584 int error; 585 586 counters_inc(uvmexp_counters, faults); 587 TRACEPOINT(uvm, fault, vaddr, fault_type, access_type, NULL); 588 589 /* 590 * init the IN parameters in the ufi 591 */ 592 ufi.orig_map = orig_map; 593 ufi.orig_rvaddr = trunc_page(vaddr); 594 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ 595 flt.access_type = access_type; 596 flt.narrow = FALSE; /* assume normal fault for now */ 597 flt.wired = FALSE; /* assume non-wired fault for now */ 598 599 error = ERESTART; 600 while (error == ERESTART) { /* ReFault: */ 601 anons = anons_store; 602 603 error = uvm_fault_check(&ufi, &flt, &anons, fault_type); 604 if (error != 0) 605 continue; 606 607 /* True if there is an anon at the faulting address */ 608 shadowed = uvm_fault_upper_lookup(&ufi, &flt, anons, pages); 609 if (shadowed == TRUE) { 610 /* case 1: fault on an anon in our amap */ 611 error = uvm_fault_upper(&ufi, &flt, anons); 612 } else { 613 struct uvm_object *uobj = ufi.entry->object.uvm_obj; 614 615 /* 616 * if the desired page is not shadowed by the amap and 617 * we have a backing object, then we check to see if 618 * the backing object would prefer to handle the fault 619 * itself (rather than letting us do it with the usual 620 * pgo_get hook). the backing object signals this by 621 * providing a pgo_fault routine. 622 */ 623 if (uobj != NULL && uobj->pgops->pgo_fault != NULL) { 624 KERNEL_LOCK(); 625 rw_enter(uobj->vmobjlock, RW_WRITE); 626 error = uobj->pgops->pgo_fault(&ufi, 627 flt.startva, pages, flt.npages, 628 flt.centeridx, fault_type, flt.access_type, 629 PGO_LOCKED); 630 KERNEL_UNLOCK(); 631 632 if (error == VM_PAGER_OK) 633 error = 0; 634 else if (error == VM_PAGER_REFAULT) 635 error = ERESTART; 636 else 637 error = EACCES; 638 } else { 639 /* case 2: fault on backing obj or zero fill */ 640 error = uvm_fault_lower(&ufi, &flt, pages); 641 } 642 } 643 } 644 645 return error; 646 } 647 648 /* 649 * uvm_fault_check: check prot, handle needs-copy, etc. 650 * 651 * 1. lookup entry. 652 * 2. check protection. 653 * 3. adjust fault condition (mainly for simulated fault). 654 * 4. handle needs-copy (lazy amap copy). 655 * 5. establish range of interest for neighbor fault (aka pre-fault). 656 * 6. look up anons (if amap exists). 657 * 7. flush pages (if MADV_SEQUENTIAL) 658 * 659 * => called with nothing locked. 660 * => if we fail (result != 0) we unlock everything. 661 * => initialize/adjust many members of flt. 662 */ 663 int 664 uvm_fault_check(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 665 struct vm_anon ***ranons, vm_fault_t fault_type) 666 { 667 struct vm_amap *amap; 668 struct uvm_object *uobj; 669 int nback, nforw; 670 671 /* 672 * lookup and lock the maps 673 */ 674 if (uvmfault_lookup(ufi, FALSE) == FALSE) { 675 return EFAULT; 676 } 677 /* locked: maps(read) */ 678 679 #ifdef DIAGNOSTIC 680 if ((ufi->map->flags & VM_MAP_PAGEABLE) == 0) 681 panic("uvm_fault: fault on non-pageable map (%p, 0x%lx)", 682 ufi->map, ufi->orig_rvaddr); 683 #endif 684 685 /* 686 * check protection 687 */ 688 if ((ufi->entry->protection & flt->access_type) != flt->access_type) { 689 uvmfault_unlockmaps(ufi, FALSE); 690 return EACCES; 691 } 692 693 /* 694 * "enter_prot" is the protection we want to enter the page in at. 695 * for certain pages (e.g. copy-on-write pages) this protection can 696 * be more strict than ufi->entry->protection. "wired" means either 697 * the entry is wired or we are fault-wiring the pg. 698 */ 699 flt->enter_prot = ufi->entry->protection; 700 flt->pa_flags = UVM_ET_ISWC(ufi->entry) ? PMAP_WC : 0; 701 if (VM_MAPENT_ISWIRED(ufi->entry) || (fault_type == VM_FAULT_WIRE)) { 702 flt->wired = TRUE; 703 flt->access_type = flt->enter_prot; /* full access for wired */ 704 /* don't look for neighborhood * pages on "wire" fault */ 705 flt->narrow = TRUE; 706 } 707 708 /* handle "needs_copy" case. */ 709 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 710 if ((flt->access_type & PROT_WRITE) || 711 (ufi->entry->object.uvm_obj == NULL)) { 712 /* need to clear */ 713 uvmfault_unlockmaps(ufi, FALSE); 714 uvmfault_amapcopy(ufi); 715 counters_inc(uvmexp_counters, flt_amcopy); 716 return ERESTART; 717 } else { 718 /* 719 * ensure that we pmap_enter page R/O since 720 * needs_copy is still true 721 */ 722 flt->enter_prot &= ~PROT_WRITE; 723 } 724 } 725 726 /* 727 * identify the players 728 */ 729 amap = ufi->entry->aref.ar_amap; /* upper layer */ 730 uobj = ufi->entry->object.uvm_obj; /* lower layer */ 731 732 /* 733 * check for a case 0 fault. if nothing backing the entry then 734 * error now. 735 */ 736 if (amap == NULL && uobj == NULL) { 737 uvmfault_unlockmaps(ufi, FALSE); 738 return EFAULT; 739 } 740 741 /* 742 * for a case 2B fault waste no time on adjacent pages because 743 * they are likely already entered. 744 */ 745 if (uobj != NULL && amap != NULL && 746 (flt->access_type & PROT_WRITE) != 0) { 747 /* wide fault (!narrow) */ 748 flt->narrow = TRUE; 749 } 750 751 /* 752 * establish range of interest based on advice from mapper 753 * and then clip to fit map entry. note that we only want 754 * to do this the first time through the fault. if we 755 * ReFault we will disable this by setting "narrow" to true. 756 */ 757 if (flt->narrow == FALSE) { 758 759 /* wide fault (!narrow) */ 760 nback = min(uvmadvice[ufi->entry->advice].nback, 761 (ufi->orig_rvaddr - ufi->entry->start) >> PAGE_SHIFT); 762 flt->startva = ufi->orig_rvaddr - ((vsize_t)nback << PAGE_SHIFT); 763 nforw = min(uvmadvice[ufi->entry->advice].nforw, 764 ((ufi->entry->end - ufi->orig_rvaddr) >> PAGE_SHIFT) - 1); 765 /* 766 * note: "-1" because we don't want to count the 767 * faulting page as forw 768 */ 769 flt->npages = nback + nforw + 1; 770 flt->centeridx = nback; 771 772 flt->narrow = TRUE; /* ensure only once per-fault */ 773 } else { 774 /* narrow fault! */ 775 nback = nforw = 0; 776 flt->startva = ufi->orig_rvaddr; 777 flt->npages = 1; 778 flt->centeridx = 0; 779 } 780 781 /* 782 * if we've got an amap then lock it and extract current anons. 783 */ 784 if (amap) { 785 amap_lock(amap, RW_WRITE); 786 amap_lookups(&ufi->entry->aref, 787 flt->startva - ufi->entry->start, *ranons, flt->npages); 788 } else { 789 *ranons = NULL; /* to be safe */ 790 } 791 792 /* 793 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages 794 * now and then forget about them (for the rest of the fault). 795 */ 796 if (ufi->entry->advice == MADV_SEQUENTIAL && nback != 0) { 797 /* flush back-page anons? */ 798 if (amap) 799 uvmfault_anonflush(*ranons, nback); 800 801 /* 802 * flush object? 803 */ 804 if (uobj) { 805 voff_t uoff; 806 807 uoff = (flt->startva - ufi->entry->start) + ufi->entry->offset; 808 rw_enter(uobj->vmobjlock, RW_WRITE); 809 (void) uobj->pgops->pgo_flush(uobj, uoff, uoff + 810 ((vsize_t)nback << PAGE_SHIFT), PGO_DEACTIVATE); 811 rw_exit(uobj->vmobjlock); 812 } 813 814 /* now forget about the backpages */ 815 if (amap) 816 *ranons += nback; 817 flt->startva += ((vsize_t)nback << PAGE_SHIFT); 818 flt->npages -= nback; 819 flt->centeridx = 0; 820 } 821 822 return 0; 823 } 824 825 /* 826 * uvm_fault_upper_lookup: look up existing h/w mapping and amap. 827 * 828 * iterate range of interest: 829 * 1. check if h/w mapping exists. if yes, we don't care 830 * 2. check if anon exists. if not, page is lower. 831 * 3. if anon exists, enter h/w mapping for neighbors. 832 * 833 * => called with amap locked (if exists). 834 */ 835 boolean_t 836 uvm_fault_upper_lookup(struct uvm_faultinfo *ufi, 837 const struct uvm_faultctx *flt, struct vm_anon **anons, 838 struct vm_page **pages) 839 { 840 struct vm_amap *amap = ufi->entry->aref.ar_amap; 841 struct vm_anon *anon; 842 struct vm_page *pg; 843 boolean_t shadowed; 844 vaddr_t currva; 845 paddr_t pa; 846 int lcv, entered = 0; 847 848 /* locked: maps(read), amap(if there) */ 849 KASSERT(amap == NULL || 850 rw_write_held(amap->am_lock)); 851 852 /* 853 * map in the backpages and frontpages we found in the amap in hopes 854 * of preventing future faults. we also init the pages[] array as 855 * we go. 856 */ 857 currva = flt->startva; 858 shadowed = FALSE; 859 for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) { 860 /* 861 * unmapped or center page. check if any anon at this level. 862 */ 863 if (amap == NULL || anons[lcv] == NULL) { 864 pages[lcv] = NULL; 865 continue; 866 } 867 868 /* 869 * check for present page and map if possible. 870 */ 871 pages[lcv] = PGO_DONTCARE; 872 if (lcv == flt->centeridx) { /* save center for later! */ 873 shadowed = TRUE; 874 continue; 875 } 876 877 anon = anons[lcv]; 878 pg = anon->an_page; 879 880 KASSERT(anon->an_lock == amap->am_lock); 881 882 /* 883 * ignore busy pages. 884 * don't play with VAs that are already mapped. 885 */ 886 if (pg && (pg->pg_flags & (PG_RELEASED|PG_BUSY)) == 0 && 887 !pmap_extract(ufi->orig_map->pmap, currva, &pa)) { 888 uvm_lock_pageq(); 889 uvm_pageactivate(pg); /* reactivate */ 890 uvm_unlock_pageq(); 891 counters_inc(uvmexp_counters, flt_namap); 892 893 /* No fault-ahead when wired. */ 894 KASSERT(flt->wired == FALSE); 895 896 /* 897 * Since this isn't the page that's actually faulting, 898 * ignore pmap_enter() failures; it's not critical 899 * that we enter these right now. 900 */ 901 (void) pmap_enter(ufi->orig_map->pmap, currva, 902 VM_PAGE_TO_PHYS(pg) | flt->pa_flags, 903 (anon->an_ref > 1) ? 904 (flt->enter_prot & ~PROT_WRITE) : flt->enter_prot, 905 PMAP_CANFAIL); 906 entered++; 907 } 908 } 909 if (entered > 0) 910 pmap_update(ufi->orig_map->pmap); 911 912 return shadowed; 913 } 914 915 /* 916 * uvm_fault_upper: handle upper fault. 917 * 918 * 1. acquire anon lock. 919 * 2. get anon. let uvmfault_anonget do the dirty work. 920 * 3. if COW, promote data to new anon 921 * 4. enter h/w mapping 922 */ 923 int 924 uvm_fault_upper(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 925 struct vm_anon **anons) 926 { 927 struct vm_amap *amap = ufi->entry->aref.ar_amap; 928 struct vm_anon *oanon, *anon = anons[flt->centeridx]; 929 struct vm_page *pg = NULL; 930 int error, ret; 931 932 /* locked: maps(read), amap, anon */ 933 KASSERT(rw_write_held(amap->am_lock)); 934 KASSERT(anon->an_lock == amap->am_lock); 935 936 /* 937 * no matter if we have case 1A or case 1B we are going to need to 938 * have the anon's memory resident. ensure that now. 939 */ 940 /* 941 * let uvmfault_anonget do the dirty work. 942 * if it fails (!OK) it will unlock everything for us. 943 * if it succeeds, locks are still valid and locked. 944 * also, if it is OK, then the anon's page is on the queues. 945 * if the page is on loan from a uvm_object, then anonget will 946 * lock that object for us if it does not fail. 947 */ 948 error = uvmfault_anonget(ufi, amap, anon); 949 switch (error) { 950 case 0: 951 break; 952 953 case ERESTART: 954 return ERESTART; 955 956 default: 957 return error; 958 } 959 960 KASSERT(rw_write_held(amap->am_lock)); 961 KASSERT(anon->an_lock == amap->am_lock); 962 963 /* 964 * if we are case 1B then we will need to allocate a new blank 965 * anon to transfer the data into. note that we have a lock 966 * on anon, so no one can busy or release the page until we are done. 967 * also note that the ref count can't drop to zero here because 968 * it is > 1 and we are only dropping one ref. 969 * 970 * in the (hopefully very rare) case that we are out of RAM we 971 * will unlock, wait for more RAM, and refault. 972 * 973 * if we are out of anon VM we wait for RAM to become available. 974 */ 975 976 if ((flt->access_type & PROT_WRITE) != 0 && anon->an_ref > 1) { 977 /* promoting requires a write lock. */ 978 KASSERT(rw_write_held(amap->am_lock)); 979 980 counters_inc(uvmexp_counters, flt_acow); 981 oanon = anon; /* oanon = old */ 982 anon = uvm_analloc(); 983 if (anon) { 984 anon->an_lock = amap->am_lock; 985 pg = uvm_pagealloc(NULL, 0, anon, 0); 986 } 987 988 /* check for out of RAM */ 989 if (anon == NULL || pg == NULL) { 990 uvmfault_unlockall(ufi, amap, NULL); 991 if (anon == NULL) 992 counters_inc(uvmexp_counters, flt_noanon); 993 else { 994 anon->an_lock = NULL; 995 anon->an_ref--; 996 uvm_anfree(anon); 997 counters_inc(uvmexp_counters, flt_noram); 998 } 999 1000 if (uvm_swapisfull()) 1001 return ENOMEM; 1002 1003 /* out of RAM, wait for more */ 1004 if (anon == NULL) 1005 uvm_anwait(); 1006 else 1007 uvm_wait("flt_noram3"); 1008 return ERESTART; 1009 } 1010 1011 /* got all resources, replace anon with nanon */ 1012 uvm_pagecopy(oanon->an_page, pg); /* pg now !PG_CLEAN */ 1013 /* un-busy! new page */ 1014 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE); 1015 UVM_PAGE_OWN(pg, NULL); 1016 ret = amap_add(&ufi->entry->aref, 1017 ufi->orig_rvaddr - ufi->entry->start, anon, 1); 1018 KASSERT(ret == 0); 1019 1020 KASSERT(anon->an_lock == oanon->an_lock); 1021 1022 /* deref: can not drop to zero here by defn! */ 1023 KASSERT(oanon->an_ref > 1); 1024 oanon->an_ref--; 1025 1026 #if defined(MULTIPROCESSOR) && !defined(__HAVE_PMAP_MPSAFE_ENTER_COW) 1027 /* 1028 * If there are multiple threads, either uvm or the 1029 * pmap has to make sure no threads see the old RO 1030 * mapping once any have seen the new RW mapping. 1031 * uvm does it by inserting the new mapping RO and 1032 * letting it fault again. 1033 * This is only a problem on MP systems. 1034 */ 1035 if (P_HASSIBLING(curproc)) { 1036 flt->enter_prot &= ~PROT_WRITE; 1037 flt->access_type &= ~PROT_WRITE; 1038 } 1039 #endif 1040 1041 /* 1042 * note: anon is _not_ locked, but we have the sole references 1043 * to in from amap. 1044 * thus, no one can get at it until we are done with it. 1045 */ 1046 } else { 1047 counters_inc(uvmexp_counters, flt_anon); 1048 oanon = anon; 1049 pg = anon->an_page; 1050 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */ 1051 flt->enter_prot = flt->enter_prot & ~PROT_WRITE; 1052 } 1053 1054 /* 1055 * now map the page in . 1056 */ 1057 if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr, 1058 VM_PAGE_TO_PHYS(pg) | flt->pa_flags, flt->enter_prot, 1059 flt->access_type | PMAP_CANFAIL | (flt->wired ? PMAP_WIRED : 0)) != 0) { 1060 /* 1061 * No need to undo what we did; we can simply think of 1062 * this as the pmap throwing away the mapping information. 1063 * 1064 * We do, however, have to go through the ReFault path, 1065 * as the map may change while we're asleep. 1066 */ 1067 uvmfault_unlockall(ufi, amap, NULL); 1068 if (uvm_swapisfull()) { 1069 /* XXX instrumentation */ 1070 return ENOMEM; 1071 } 1072 /* XXX instrumentation */ 1073 uvm_wait("flt_pmfail1"); 1074 return ERESTART; 1075 } 1076 1077 /* 1078 * ... update the page queues. 1079 */ 1080 uvm_lock_pageq(); 1081 if (flt->wired) { 1082 uvm_pagewire(pg); 1083 } else { 1084 uvm_pageactivate(pg); 1085 } 1086 uvm_unlock_pageq(); 1087 1088 if (flt->wired) { 1089 /* 1090 * since the now-wired page cannot be paged out, 1091 * release its swap resources for others to use. 1092 * since an anon with no swap cannot be PG_CLEAN, 1093 * clear its clean flag now. 1094 */ 1095 atomic_clearbits_int(&pg->pg_flags, PG_CLEAN); 1096 uvm_anon_dropswap(anon); 1097 } 1098 1099 /* 1100 * done case 1! finish up by unlocking everything and returning success 1101 */ 1102 uvmfault_unlockall(ufi, amap, NULL); 1103 pmap_update(ufi->orig_map->pmap); 1104 return 0; 1105 } 1106 1107 /* 1108 * uvm_fault_lower_lookup: look up on-memory uobj pages. 1109 * 1110 * 1. get on-memory pages. 1111 * 2. if failed, give up (get only center page later). 1112 * 3. if succeeded, enter h/w mapping of neighbor pages. 1113 */ 1114 1115 struct vm_page * 1116 uvm_fault_lower_lookup( 1117 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 1118 struct vm_page **pages) 1119 { 1120 struct uvm_object *uobj = ufi->entry->object.uvm_obj; 1121 struct vm_page *uobjpage = NULL; 1122 int lcv, gotpages, entered; 1123 vaddr_t currva; 1124 paddr_t pa; 1125 1126 rw_enter(uobj->vmobjlock, RW_WRITE); 1127 1128 counters_inc(uvmexp_counters, flt_lget); 1129 gotpages = flt->npages; 1130 (void) uobj->pgops->pgo_get(uobj, 1131 ufi->entry->offset + (flt->startva - ufi->entry->start), 1132 pages, &gotpages, flt->centeridx, 1133 flt->access_type & MASK(ufi->entry), ufi->entry->advice, 1134 PGO_LOCKED); 1135 1136 /* 1137 * check for pages to map, if we got any 1138 */ 1139 if (gotpages == 0) { 1140 return NULL; 1141 } 1142 1143 entered = 0; 1144 currva = flt->startva; 1145 for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) { 1146 if (pages[lcv] == NULL || 1147 pages[lcv] == PGO_DONTCARE) 1148 continue; 1149 1150 KASSERT((pages[lcv]->pg_flags & PG_RELEASED) == 0); 1151 1152 /* 1153 * if center page is resident and not 1154 * PG_BUSY, then pgo_get made it PG_BUSY 1155 * for us and gave us a handle to it. 1156 * remember this page as "uobjpage." 1157 * (for later use). 1158 */ 1159 if (lcv == flt->centeridx) { 1160 uobjpage = pages[lcv]; 1161 continue; 1162 } 1163 1164 if (pmap_extract(ufi->orig_map->pmap, currva, &pa)) 1165 goto next; 1166 1167 if (pages[lcv]->wire_count == 0) { 1168 uvm_lock_pageq(); 1169 uvm_pageactivate(pages[lcv]); 1170 uvm_unlock_pageq(); 1171 } 1172 counters_inc(uvmexp_counters, flt_nomap); 1173 1174 /* No fault-ahead when wired. */ 1175 KASSERT(flt->wired == FALSE); 1176 1177 /* 1178 * Since this page isn't the page that's 1179 * actually faulting, ignore pmap_enter() 1180 * failures; it's not critical that we 1181 * enter these right now. 1182 */ 1183 (void) pmap_enter(ufi->orig_map->pmap, currva, 1184 VM_PAGE_TO_PHYS(pages[lcv]) | flt->pa_flags, 1185 flt->enter_prot & MASK(ufi->entry), PMAP_CANFAIL); 1186 entered++; 1187 1188 /* 1189 * NOTE: page can't be PG_WANTED because 1190 * we've held the lock the whole time 1191 * we've had the handle. 1192 */ 1193 next: 1194 atomic_clearbits_int(&pages[lcv]->pg_flags, PG_BUSY); 1195 UVM_PAGE_OWN(pages[lcv], NULL); 1196 } 1197 if (entered > 0) 1198 pmap_update(ufi->orig_map->pmap); 1199 1200 return uobjpage; 1201 } 1202 1203 /* 1204 * uvm_fault_lower: handle lower fault. 1205 * 1206 * 1. check uobj 1207 * 1.1. if null, ZFOD. 1208 * 1.2. if not null, look up unnmapped neighbor pages. 1209 * 2. for center page, check if promote. 1210 * 2.1. ZFOD always needs promotion. 1211 * 2.2. other uobjs, when entry is marked COW (usually MAP_PRIVATE vnode). 1212 * 3. if uobj is not ZFOD and page is not found, do i/o. 1213 * 4. dispatch either direct / promote fault. 1214 */ 1215 int 1216 uvm_fault_lower(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1217 struct vm_page **pages) 1218 { 1219 struct vm_amap *amap = ufi->entry->aref.ar_amap; 1220 struct uvm_object *uobj = ufi->entry->object.uvm_obj; 1221 int dropswap = 0; 1222 struct vm_page *uobjpage, *pg = NULL; 1223 struct vm_anon *anon = NULL; 1224 int error; 1225 1226 /* 1227 * now, if the desired page is not shadowed by the amap and we have 1228 * a backing object that does not have a special fault routine, then 1229 * we ask (with pgo_get) the object for resident pages that we care 1230 * about and attempt to map them in. we do not let pgo_get block 1231 * (PGO_LOCKED). 1232 */ 1233 if (uobj == NULL) { 1234 /* zero fill; don't care neighbor pages */ 1235 uobjpage = NULL; 1236 } else { 1237 uobjpage = uvm_fault_lower_lookup(ufi, flt, pages); 1238 } 1239 1240 /* 1241 * note that at this point we are done with any front or back pages. 1242 * we are now going to focus on the center page (i.e. the one we've 1243 * faulted on). if we have faulted on the bottom (uobj) 1244 * layer [i.e. case 2] and the page was both present and available, 1245 * then we've got a pointer to it as "uobjpage" and we've already 1246 * made it BUSY. 1247 */ 1248 1249 /* 1250 * locked: 1251 */ 1252 KASSERT(amap == NULL || 1253 rw_write_held(amap->am_lock)); 1254 KASSERT(uobj == NULL || 1255 rw_write_held(uobj->vmobjlock)); 1256 1257 /* 1258 * note that uobjpage can not be PGO_DONTCARE at this point. we now 1259 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we 1260 * have a backing object, check and see if we are going to promote 1261 * the data up to an anon during the fault. 1262 */ 1263 if (uobj == NULL) { 1264 uobjpage = PGO_DONTCARE; 1265 flt->promote = TRUE; /* always need anon here */ 1266 } else { 1267 KASSERT(uobjpage != PGO_DONTCARE); 1268 flt->promote = (flt->access_type & PROT_WRITE) && 1269 UVM_ET_ISCOPYONWRITE(ufi->entry); 1270 } 1271 1272 /* 1273 * if uobjpage is not null then we do not need to do I/O to get the 1274 * uobjpage. 1275 * 1276 * if uobjpage is null, then we need to ask the pager to 1277 * get the data for us. once we have the data, we need to reverify 1278 * the state the world. we are currently not holding any resources. 1279 */ 1280 if (uobjpage) { 1281 /* update rusage counters */ 1282 curproc->p_ru.ru_minflt++; 1283 } else { 1284 error = uvm_fault_lower_io(ufi, flt, &uobj, &uobjpage); 1285 if (error != 0) 1286 return error; 1287 } 1288 1289 /* 1290 * notes: 1291 * - at this point uobjpage can not be NULL 1292 * - at this point uobjpage could be PG_WANTED (handle later) 1293 */ 1294 if (flt->promote == FALSE) { 1295 /* 1296 * we are not promoting. if the mapping is COW ensure that we 1297 * don't give more access than we should (e.g. when doing a read 1298 * fault on a COPYONWRITE mapping we want to map the COW page in 1299 * R/O even though the entry protection could be R/W). 1300 * 1301 * set "pg" to the page we want to map in (uobjpage, usually) 1302 */ 1303 counters_inc(uvmexp_counters, flt_obj); 1304 if (UVM_ET_ISCOPYONWRITE(ufi->entry)) 1305 flt->enter_prot &= ~PROT_WRITE; 1306 pg = uobjpage; /* map in the actual object */ 1307 1308 /* assert(uobjpage != PGO_DONTCARE) */ 1309 1310 /* 1311 * we are faulting directly on the page. 1312 */ 1313 } else { 1314 /* 1315 * if we are going to promote the data to an anon we 1316 * allocate a blank anon here and plug it into our amap. 1317 */ 1318 #ifdef DIAGNOSTIC 1319 if (amap == NULL) 1320 panic("uvm_fault: want to promote data, but no anon"); 1321 #endif 1322 1323 anon = uvm_analloc(); 1324 if (anon) { 1325 /* 1326 * In `Fill in data...' below, if 1327 * uobjpage == PGO_DONTCARE, we want 1328 * a zero'd, dirty page, so have 1329 * uvm_pagealloc() do that for us. 1330 */ 1331 anon->an_lock = amap->am_lock; 1332 pg = uvm_pagealloc(NULL, 0, anon, 1333 (uobjpage == PGO_DONTCARE) ? UVM_PGA_ZERO : 0); 1334 } 1335 1336 /* 1337 * out of memory resources? 1338 */ 1339 if (anon == NULL || pg == NULL) { 1340 /* 1341 * arg! must unbusy our page and fail or sleep. 1342 */ 1343 if (uobjpage != PGO_DONTCARE) { 1344 uvm_lock_pageq(); 1345 uvm_pageactivate(uobjpage); 1346 uvm_unlock_pageq(); 1347 1348 if (uobjpage->pg_flags & PG_WANTED) 1349 wakeup(uobjpage); 1350 atomic_clearbits_int(&uobjpage->pg_flags, 1351 PG_BUSY|PG_WANTED); 1352 UVM_PAGE_OWN(uobjpage, NULL); 1353 } 1354 1355 /* unlock and fail ... */ 1356 uvmfault_unlockall(ufi, amap, uobj); 1357 if (anon == NULL) 1358 counters_inc(uvmexp_counters, flt_noanon); 1359 else { 1360 anon->an_lock = NULL; 1361 anon->an_ref--; 1362 uvm_anfree(anon); 1363 counters_inc(uvmexp_counters, flt_noram); 1364 } 1365 1366 if (uvm_swapisfull()) 1367 return (ENOMEM); 1368 1369 /* out of RAM, wait for more */ 1370 if (anon == NULL) 1371 uvm_anwait(); 1372 else 1373 uvm_wait("flt_noram5"); 1374 return ERESTART; 1375 } 1376 1377 /* 1378 * fill in the data 1379 */ 1380 if (uobjpage != PGO_DONTCARE) { 1381 counters_inc(uvmexp_counters, flt_prcopy); 1382 /* copy page [pg now dirty] */ 1383 uvm_pagecopy(uobjpage, pg); 1384 1385 /* 1386 * promote to shared amap? make sure all sharing 1387 * procs see it 1388 */ 1389 if ((amap_flags(amap) & AMAP_SHARED) != 0) { 1390 pmap_page_protect(uobjpage, PROT_NONE); 1391 } 1392 #if defined(MULTIPROCESSOR) && !defined(__HAVE_PMAP_MPSAFE_ENTER_COW) 1393 /* 1394 * Otherwise: 1395 * If there are multiple threads, either uvm or the 1396 * pmap has to make sure no threads see the old RO 1397 * mapping once any have seen the new RW mapping. 1398 * uvm does it here by forcing it to PROT_NONE before 1399 * inserting the new mapping. 1400 */ 1401 else if (P_HASSIBLING(curproc)) { 1402 pmap_page_protect(uobjpage, PROT_NONE); 1403 } 1404 #endif 1405 1406 /* dispose of uobjpage. drop handle to uobj as well. */ 1407 if (uobjpage->pg_flags & PG_WANTED) 1408 wakeup(uobjpage); 1409 atomic_clearbits_int(&uobjpage->pg_flags, 1410 PG_BUSY|PG_WANTED); 1411 UVM_PAGE_OWN(uobjpage, NULL); 1412 uvm_lock_pageq(); 1413 uvm_pageactivate(uobjpage); 1414 uvm_unlock_pageq(); 1415 rw_exit(uobj->vmobjlock); 1416 uobj = NULL; 1417 } else { 1418 counters_inc(uvmexp_counters, flt_przero); 1419 /* 1420 * Page is zero'd and marked dirty by uvm_pagealloc() 1421 * above. 1422 */ 1423 } 1424 1425 if (amap_add(&ufi->entry->aref, 1426 ufi->orig_rvaddr - ufi->entry->start, anon, 0)) { 1427 if (pg->pg_flags & PG_WANTED) 1428 wakeup(pg); 1429 1430 atomic_clearbits_int(&pg->pg_flags, 1431 PG_BUSY|PG_FAKE|PG_WANTED); 1432 UVM_PAGE_OWN(pg, NULL); 1433 uvmfault_unlockall(ufi, amap, uobj); 1434 uvm_anfree(anon); 1435 counters_inc(uvmexp_counters, flt_noamap); 1436 1437 if (uvm_swapisfull()) 1438 return (ENOMEM); 1439 1440 amap_populate(&ufi->entry->aref, 1441 ufi->orig_rvaddr - ufi->entry->start); 1442 return ERESTART; 1443 } 1444 } 1445 1446 /* note: pg is either the uobjpage or the new page in the new anon */ 1447 /* 1448 * all resources are present. we can now map it in and free our 1449 * resources. 1450 */ 1451 if (amap == NULL) 1452 KASSERT(anon == NULL); 1453 else { 1454 KASSERT(rw_write_held(amap->am_lock)); 1455 KASSERT(anon == NULL || anon->an_lock == amap->am_lock); 1456 } 1457 if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr, 1458 VM_PAGE_TO_PHYS(pg) | flt->pa_flags, flt->enter_prot, 1459 flt->access_type | PMAP_CANFAIL | (flt->wired ? PMAP_WIRED : 0)) != 0) { 1460 /* 1461 * No need to undo what we did; we can simply think of 1462 * this as the pmap throwing away the mapping information. 1463 * 1464 * We do, however, have to go through the ReFault path, 1465 * as the map may change while we're asleep. 1466 */ 1467 if (pg->pg_flags & PG_WANTED) 1468 wakeup(pg); 1469 1470 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED); 1471 UVM_PAGE_OWN(pg, NULL); 1472 uvmfault_unlockall(ufi, amap, uobj); 1473 if (uvm_swapisfull()) { 1474 /* XXX instrumentation */ 1475 return (ENOMEM); 1476 } 1477 /* XXX instrumentation */ 1478 uvm_wait("flt_pmfail2"); 1479 return ERESTART; 1480 } 1481 1482 uvm_lock_pageq(); 1483 if (flt->wired) { 1484 uvm_pagewire(pg); 1485 if (pg->pg_flags & PQ_AOBJ) { 1486 /* 1487 * since the now-wired page cannot be paged out, 1488 * release its swap resources for others to use. 1489 * since an aobj page with no swap cannot be clean, 1490 * mark it dirty now. 1491 * 1492 * use pg->uobject here. if the page is from a 1493 * tmpfs vnode, the pages are backed by its UAO and 1494 * not the vnode. 1495 */ 1496 KASSERT(uobj != NULL); 1497 KASSERT(uobj->vmobjlock == pg->uobject->vmobjlock); 1498 atomic_clearbits_int(&pg->pg_flags, PG_CLEAN); 1499 dropswap = 1; 1500 } 1501 } else { 1502 uvm_pageactivate(pg); 1503 } 1504 uvm_unlock_pageq(); 1505 1506 if (dropswap) 1507 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT); 1508 1509 if (pg->pg_flags & PG_WANTED) 1510 wakeup(pg); 1511 1512 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED); 1513 UVM_PAGE_OWN(pg, NULL); 1514 uvmfault_unlockall(ufi, amap, uobj); 1515 pmap_update(ufi->orig_map->pmap); 1516 1517 return (0); 1518 } 1519 1520 /* 1521 * uvm_fault_lower_io: get lower page from backing store. 1522 * 1523 * 1. unlock everything, because i/o will block. 1524 * 2. call pgo_get. 1525 * 3. if failed, recover. 1526 * 4. if succeeded, relock everything and verify things. 1527 */ 1528 int 1529 uvm_fault_lower_io( 1530 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1531 struct uvm_object **ruobj, struct vm_page **ruobjpage) 1532 { 1533 struct vm_amap * const amap = ufi->entry->aref.ar_amap; 1534 struct uvm_object *uobj = *ruobj; 1535 struct vm_page *pg; 1536 boolean_t locked; 1537 int gotpages; 1538 int result; 1539 voff_t uoff; 1540 1541 /* update rusage counters */ 1542 curproc->p_ru.ru_majflt++; 1543 1544 uvmfault_unlockall(ufi, amap, NULL); 1545 1546 counters_inc(uvmexp_counters, flt_get); 1547 gotpages = 1; 1548 pg = NULL; 1549 uoff = (ufi->orig_rvaddr - ufi->entry->start) + ufi->entry->offset; 1550 result = uobj->pgops->pgo_get(uobj, uoff, &pg, &gotpages, 1551 0, flt->access_type & MASK(ufi->entry), ufi->entry->advice, 1552 PGO_SYNCIO); 1553 1554 /* 1555 * recover from I/O 1556 */ 1557 if (result != VM_PAGER_OK) { 1558 KASSERT(result != VM_PAGER_PEND); 1559 1560 if (result == VM_PAGER_AGAIN) { 1561 tsleep_nsec(&nowake, PVM, "fltagain2", MSEC_TO_NSEC(5)); 1562 return ERESTART; 1563 } 1564 1565 if (!UVM_ET_ISNOFAULT(ufi->entry)) 1566 return (EIO); 1567 1568 pg = PGO_DONTCARE; 1569 uobj = NULL; 1570 flt->promote = TRUE; 1571 } 1572 1573 /* re-verify the state of the world. */ 1574 locked = uvmfault_relock(ufi); 1575 if (locked && amap != NULL) 1576 amap_lock(amap, RW_WRITE); 1577 1578 /* might be changed */ 1579 if (pg != PGO_DONTCARE) { 1580 uobj = pg->uobject; 1581 rw_enter(uobj->vmobjlock, RW_WRITE); 1582 } 1583 1584 /* 1585 * Re-verify that amap slot is still free. if there is 1586 * a problem, we clean up. 1587 */ 1588 if (locked && amap && amap_lookup(&ufi->entry->aref, 1589 ufi->orig_rvaddr - ufi->entry->start)) { 1590 if (locked) 1591 uvmfault_unlockall(ufi, amap, NULL); 1592 locked = FALSE; 1593 } 1594 1595 /* didn't get the lock? release the page and retry. */ 1596 if (locked == FALSE && pg != PGO_DONTCARE) { 1597 uvm_lock_pageq(); 1598 /* make sure it is in queues */ 1599 uvm_pageactivate(pg); 1600 uvm_unlock_pageq(); 1601 1602 if (pg->pg_flags & PG_WANTED) 1603 /* still holding object lock */ 1604 wakeup(pg); 1605 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_WANTED); 1606 UVM_PAGE_OWN(pg, NULL); 1607 } 1608 1609 if (locked == FALSE) { 1610 if (pg != PGO_DONTCARE) 1611 rw_exit(uobj->vmobjlock); 1612 return ERESTART; 1613 } 1614 1615 /* 1616 * we have the data in pg which is PG_BUSY 1617 */ 1618 *ruobj = uobj; 1619 *ruobjpage = pg; 1620 return 0; 1621 } 1622 1623 /* 1624 * uvm_fault_wire: wire down a range of virtual addresses in a map. 1625 * 1626 * => map may be read-locked by caller, but MUST NOT be write-locked. 1627 * => if map is read-locked, any operations which may cause map to 1628 * be write-locked in uvm_fault() must be taken care of by 1629 * the caller. See uvm_map_pageable(). 1630 */ 1631 int 1632 uvm_fault_wire(vm_map_t map, vaddr_t start, vaddr_t end, vm_prot_t access_type) 1633 { 1634 vaddr_t va; 1635 int rv; 1636 1637 /* 1638 * now fault it in a page at a time. if the fault fails then we have 1639 * to undo what we have done. note that in uvm_fault PROT_NONE 1640 * is replaced with the max protection if fault_type is VM_FAULT_WIRE. 1641 */ 1642 for (va = start ; va < end ; va += PAGE_SIZE) { 1643 rv = uvm_fault(map, va, VM_FAULT_WIRE, access_type); 1644 if (rv) { 1645 if (va != start) { 1646 uvm_fault_unwire(map, start, va); 1647 } 1648 return (rv); 1649 } 1650 } 1651 1652 return (0); 1653 } 1654 1655 /* 1656 * uvm_fault_unwire(): unwire range of virtual space. 1657 */ 1658 void 1659 uvm_fault_unwire(vm_map_t map, vaddr_t start, vaddr_t end) 1660 { 1661 1662 vm_map_lock_read(map); 1663 uvm_fault_unwire_locked(map, start, end); 1664 vm_map_unlock_read(map); 1665 } 1666 1667 /* 1668 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire(). 1669 * 1670 * => map must be at least read-locked. 1671 */ 1672 void 1673 uvm_fault_unwire_locked(vm_map_t map, vaddr_t start, vaddr_t end) 1674 { 1675 vm_map_entry_t entry, oentry = NULL, next; 1676 pmap_t pmap = vm_map_pmap(map); 1677 vaddr_t va; 1678 paddr_t pa; 1679 struct vm_page *pg; 1680 1681 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0); 1682 vm_map_assert_anylock(map); 1683 1684 /* 1685 * we assume that the area we are unwiring has actually been wired 1686 * in the first place. this means that we should be able to extract 1687 * the PAs from the pmap. 1688 */ 1689 1690 /* 1691 * find the beginning map entry for the region. 1692 */ 1693 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map)); 1694 if (uvm_map_lookup_entry(map, start, &entry) == FALSE) 1695 panic("uvm_fault_unwire_locked: address not in map"); 1696 1697 for (va = start; va < end ; va += PAGE_SIZE) { 1698 /* 1699 * find the map entry for the current address. 1700 */ 1701 KASSERT(va >= entry->start); 1702 while (entry && va >= entry->end) { 1703 next = RBT_NEXT(uvm_map_addr, entry); 1704 entry = next; 1705 } 1706 1707 if (entry == NULL) 1708 return; 1709 if (va < entry->start) 1710 continue; 1711 1712 /* 1713 * lock it. 1714 */ 1715 if (entry != oentry) { 1716 if (oentry != NULL) { 1717 uvm_map_unlock_entry(oentry); 1718 } 1719 uvm_map_lock_entry(entry); 1720 oentry = entry; 1721 } 1722 1723 if (!pmap_extract(pmap, va, &pa)) 1724 continue; 1725 1726 /* 1727 * if the entry is no longer wired, tell the pmap. 1728 */ 1729 if (VM_MAPENT_ISWIRED(entry) == 0) 1730 pmap_unwire(pmap, va); 1731 1732 pg = PHYS_TO_VM_PAGE(pa); 1733 if (pg) { 1734 uvm_lock_pageq(); 1735 uvm_pageunwire(pg); 1736 uvm_unlock_pageq(); 1737 } 1738 } 1739 1740 if (oentry != NULL) { 1741 uvm_map_unlock_entry(oentry); 1742 } 1743 } 1744 1745 /* 1746 * uvmfault_unlockmaps: unlock the maps 1747 */ 1748 void 1749 uvmfault_unlockmaps(struct uvm_faultinfo *ufi, boolean_t write_locked) 1750 { 1751 /* 1752 * ufi can be NULL when this isn't really a fault, 1753 * but merely paging in anon data. 1754 */ 1755 if (ufi == NULL) { 1756 return; 1757 } 1758 1759 uvmfault_update_stats(ufi); 1760 if (write_locked) { 1761 vm_map_unlock(ufi->map); 1762 } else { 1763 vm_map_unlock_read(ufi->map); 1764 } 1765 } 1766 1767 /* 1768 * uvmfault_unlockall: unlock everything passed in. 1769 * 1770 * => maps must be read-locked (not write-locked). 1771 */ 1772 void 1773 uvmfault_unlockall(struct uvm_faultinfo *ufi, struct vm_amap *amap, 1774 struct uvm_object *uobj) 1775 { 1776 if (uobj) 1777 rw_exit(uobj->vmobjlock); 1778 if (amap != NULL) 1779 amap_unlock(amap); 1780 uvmfault_unlockmaps(ufi, FALSE); 1781 } 1782 1783 /* 1784 * uvmfault_lookup: lookup a virtual address in a map 1785 * 1786 * => caller must provide a uvm_faultinfo structure with the IN 1787 * params properly filled in 1788 * => we will lookup the map entry (handling submaps) as we go 1789 * => if the lookup is a success we will return with the maps locked 1790 * => if "write_lock" is TRUE, we write_lock the map, otherwise we only 1791 * get a read lock. 1792 * => note that submaps can only appear in the kernel and they are 1793 * required to use the same virtual addresses as the map they 1794 * are referenced by (thus address translation between the main 1795 * map and the submap is unnecessary). 1796 */ 1797 1798 boolean_t 1799 uvmfault_lookup(struct uvm_faultinfo *ufi, boolean_t write_lock) 1800 { 1801 vm_map_t tmpmap; 1802 1803 /* 1804 * init ufi values for lookup. 1805 */ 1806 ufi->map = ufi->orig_map; 1807 ufi->size = ufi->orig_size; 1808 1809 /* 1810 * keep going down levels until we are done. note that there can 1811 * only be two levels so we won't loop very long. 1812 */ 1813 while (1) { 1814 if (ufi->orig_rvaddr < ufi->map->min_offset || 1815 ufi->orig_rvaddr >= ufi->map->max_offset) 1816 return FALSE; 1817 1818 /* lock map */ 1819 if (write_lock) { 1820 vm_map_lock(ufi->map); 1821 } else { 1822 vm_map_lock_read(ufi->map); 1823 } 1824 1825 /* lookup */ 1826 if (!uvm_map_lookup_entry(ufi->map, ufi->orig_rvaddr, 1827 &ufi->entry)) { 1828 uvmfault_unlockmaps(ufi, write_lock); 1829 return FALSE; 1830 } 1831 1832 /* reduce size if necessary */ 1833 if (ufi->entry->end - ufi->orig_rvaddr < ufi->size) 1834 ufi->size = ufi->entry->end - ufi->orig_rvaddr; 1835 1836 /* 1837 * submap? replace map with the submap and lookup again. 1838 * note: VAs in submaps must match VAs in main map. 1839 */ 1840 if (UVM_ET_ISSUBMAP(ufi->entry)) { 1841 tmpmap = ufi->entry->object.sub_map; 1842 uvmfault_unlockmaps(ufi, write_lock); 1843 ufi->map = tmpmap; 1844 continue; 1845 } 1846 1847 /* 1848 * got it! 1849 */ 1850 ufi->mapv = ufi->map->timestamp; 1851 return TRUE; 1852 1853 } /* while loop */ 1854 1855 /*NOTREACHED*/ 1856 } 1857 1858 /* 1859 * uvmfault_relock: attempt to relock the same version of the map 1860 * 1861 * => fault data structures should be unlocked before calling. 1862 * => if a success (TRUE) maps will be locked after call. 1863 */ 1864 boolean_t 1865 uvmfault_relock(struct uvm_faultinfo *ufi) 1866 { 1867 /* 1868 * ufi can be NULL when this isn't really a fault, 1869 * but merely paging in anon data. 1870 */ 1871 if (ufi == NULL) { 1872 return TRUE; 1873 } 1874 1875 counters_inc(uvmexp_counters, flt_relck); 1876 1877 /* 1878 * relock map. fail if version mismatch (in which case nothing 1879 * gets locked). 1880 */ 1881 vm_map_lock_read(ufi->map); 1882 if (ufi->mapv != ufi->map->timestamp) { 1883 vm_map_unlock_read(ufi->map); 1884 return FALSE; 1885 } 1886 1887 counters_inc(uvmexp_counters, flt_relckok); 1888 return TRUE; /* got it! */ 1889 } 1890