1 /* $OpenBSD: uvm_fault.c,v 1.140 2024/11/05 08:13:41 mpi Exp $ */ 2 /* $NetBSD: uvm_fault.c,v 1.51 2000/08/06 00:22:53 thorpej Exp $ */ 3 4 /* 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp 29 */ 30 31 /* 32 * uvm_fault.c: fault handler 33 */ 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/percpu.h> 39 #include <sys/proc.h> 40 #include <sys/malloc.h> 41 #include <sys/mman.h> 42 #include <sys/tracepoint.h> 43 44 #include <uvm/uvm.h> 45 46 /* 47 * 48 * a word on page faults: 49 * 50 * types of page faults we handle: 51 * 52 * CASE 1: upper layer faults CASE 2: lower layer faults 53 * 54 * CASE 1A CASE 1B CASE 2A CASE 2B 55 * read/write1 write>1 read/write +-cow_write/zero 56 * | | | | 57 * +--|--+ +--|--+ +-----+ + | + | +-----+ 58 * amap | V | | ---------> new | | | | ^ | 59 * +-----+ +-----+ +-----+ + | + | +--|--+ 60 * | | | 61 * +-----+ +-----+ +--|--+ | +--|--+ 62 * uobj | d/c | | d/c | | V | +----+ | 63 * +-----+ +-----+ +-----+ +-----+ 64 * 65 * d/c = don't care 66 * 67 * case [0]: layerless fault 68 * no amap or uobj is present. this is an error. 69 * 70 * case [1]: upper layer fault [anon active] 71 * 1A: [read] or [write with anon->an_ref == 1] 72 * I/O takes place in upper level anon and uobj is not touched. 73 * 1B: [write with anon->an_ref > 1] 74 * new anon is alloc'd and data is copied off ["COW"] 75 * 76 * case [2]: lower layer fault [uobj] 77 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area] 78 * I/O takes place directly in object. 79 * 2B: [write to copy_on_write] or [read on NULL uobj] 80 * data is "promoted" from uobj to a new anon. 81 * if uobj is null, then we zero fill. 82 * 83 * we follow the standard UVM locking protocol ordering: 84 * 85 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ) 86 * we hold a PG_BUSY page if we unlock for I/O 87 * 88 * 89 * the code is structured as follows: 90 * 91 * - init the "IN" params in the ufi structure 92 * ReFault: (ERESTART returned to the loop in uvm_fault) 93 * - do lookups [locks maps], check protection, handle needs_copy 94 * - check for case 0 fault (error) 95 * - establish "range" of fault 96 * - if we have an amap lock it and extract the anons 97 * - if sequential advice deactivate pages behind us 98 * - at the same time check pmap for unmapped areas and anon for pages 99 * that we could map in (and do map it if found) 100 * - check object for resident pages that we could map in 101 * - if (case 2) goto Case2 102 * - >>> handle case 1 103 * - ensure source anon is resident in RAM 104 * - if case 1B alloc new anon and copy from source 105 * - map the correct page in 106 * Case2: 107 * - >>> handle case 2 108 * - ensure source page is resident (if uobj) 109 * - if case 2B alloc new anon and copy from source (could be zero 110 * fill if uobj == NULL) 111 * - map the correct page in 112 * - done! 113 * 114 * note on paging: 115 * if we have to do I/O we place a PG_BUSY page in the correct object, 116 * unlock everything, and do the I/O. when I/O is done we must reverify 117 * the state of the world before assuming that our data structures are 118 * valid. [because mappings could change while the map is unlocked] 119 * 120 * alternative 1: unbusy the page in question and restart the page fault 121 * from the top (ReFault). this is easy but does not take advantage 122 * of the information that we already have from our previous lookup, 123 * although it is possible that the "hints" in the vm_map will help here. 124 * 125 * alternative 2: the system already keeps track of a "version" number of 126 * a map. [i.e. every time you write-lock a map (e.g. to change a 127 * mapping) you bump the version number up by one...] so, we can save 128 * the version number of the map before we release the lock and start I/O. 129 * then when I/O is done we can relock and check the version numbers 130 * to see if anything changed. this might save us some over 1 because 131 * we don't have to unbusy the page and may be less compares(?). 132 * 133 * alternative 3: put in backpointers or a way to "hold" part of a map 134 * in place while I/O is in progress. this could be complex to 135 * implement (especially with structures like amap that can be referenced 136 * by multiple map entries, and figuring out what should wait could be 137 * complex as well...). 138 * 139 * we use alternative 2. given that we are multi-threaded now we may want 140 * to reconsider the choice. 141 */ 142 143 /* 144 * local data structures 145 */ 146 struct uvm_advice { 147 int nback; 148 int nforw; 149 }; 150 151 /* 152 * page range array: set up in uvmfault_init(). 153 */ 154 static struct uvm_advice uvmadvice[MADV_MASK + 1]; 155 156 #define UVM_MAXRANGE 16 /* must be max() of nback+nforw+1 */ 157 158 /* 159 * private prototypes 160 */ 161 static void uvmfault_amapcopy(struct uvm_faultinfo *); 162 static inline void uvmfault_anonflush(struct vm_anon **, int); 163 void uvmfault_unlockmaps(struct uvm_faultinfo *, boolean_t); 164 void uvmfault_update_stats(struct uvm_faultinfo *); 165 166 /* 167 * inline functions 168 */ 169 /* 170 * uvmfault_anonflush: try and deactivate pages in specified anons 171 * 172 * => does not have to deactivate page if it is busy 173 */ 174 static inline void 175 uvmfault_anonflush(struct vm_anon **anons, int n) 176 { 177 int lcv; 178 struct vm_page *pg; 179 180 for (lcv = 0; lcv < n; lcv++) { 181 if (anons[lcv] == NULL) 182 continue; 183 KASSERT(rw_lock_held(anons[lcv]->an_lock)); 184 pg = anons[lcv]->an_page; 185 if (pg && (pg->pg_flags & PG_BUSY) == 0) { 186 uvm_lock_pageq(); 187 if (pg->wire_count == 0) { 188 pmap_page_protect(pg, PROT_NONE); 189 uvm_pagedeactivate(pg); 190 } 191 uvm_unlock_pageq(); 192 } 193 } 194 } 195 196 /* 197 * normal functions 198 */ 199 /* 200 * uvmfault_init: compute proper values for the uvmadvice[] array. 201 */ 202 void 203 uvmfault_init(void) 204 { 205 int npages; 206 207 npages = atop(16384); 208 if (npages > 0) { 209 KASSERT(npages <= UVM_MAXRANGE / 2); 210 uvmadvice[MADV_NORMAL].nforw = npages; 211 uvmadvice[MADV_NORMAL].nback = npages - 1; 212 } 213 214 npages = atop(32768); 215 if (npages > 0) { 216 KASSERT(npages <= UVM_MAXRANGE / 2); 217 uvmadvice[MADV_SEQUENTIAL].nforw = npages - 1; 218 uvmadvice[MADV_SEQUENTIAL].nback = npages; 219 } 220 } 221 222 /* 223 * uvmfault_amapcopy: clear "needs_copy" in a map. 224 * 225 * => called with VM data structures unlocked (usually, see below) 226 * => we get a write lock on the maps and clear needs_copy for a VA 227 * => if we are out of RAM we sleep (waiting for more) 228 */ 229 static void 230 uvmfault_amapcopy(struct uvm_faultinfo *ufi) 231 { 232 for (;;) { 233 /* 234 * no mapping? give up. 235 */ 236 if (uvmfault_lookup(ufi, TRUE) == FALSE) 237 return; 238 239 /* 240 * copy if needed. 241 */ 242 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) 243 amap_copy(ufi->map, ufi->entry, M_NOWAIT, 244 UVM_ET_ISSTACK(ufi->entry) ? FALSE : TRUE, 245 ufi->orig_rvaddr, ufi->orig_rvaddr + 1); 246 247 /* 248 * didn't work? must be out of RAM. unlock and sleep. 249 */ 250 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 251 uvmfault_unlockmaps(ufi, TRUE); 252 uvm_wait("fltamapcopy"); 253 continue; 254 } 255 256 /* 257 * got it! unlock and return. 258 */ 259 uvmfault_unlockmaps(ufi, TRUE); 260 return; 261 } 262 /*NOTREACHED*/ 263 } 264 265 /* 266 * uvmfault_anonget: get data in an anon into a non-busy, non-released 267 * page in that anon. 268 * 269 * => Map, amap and thus anon should be locked by caller. 270 * => If we fail, we unlock everything and error is returned. 271 * => If we are successful, return with everything still locked. 272 * => We do not move the page on the queues [gets moved later]. If we 273 * allocate a new page [we_own], it gets put on the queues. Either way, 274 * the result is that the page is on the queues at return time 275 */ 276 int 277 uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap, 278 struct vm_anon *anon) 279 { 280 struct vm_page *pg; 281 int error; 282 283 KASSERT(rw_lock_held(anon->an_lock)); 284 KASSERT(anon->an_lock == amap->am_lock); 285 286 /* Increment the counters.*/ 287 counters_inc(uvmexp_counters, flt_anget); 288 if (anon->an_page) { 289 curproc->p_ru.ru_minflt++; 290 } else { 291 curproc->p_ru.ru_majflt++; 292 } 293 error = 0; 294 295 /* 296 * Loop until we get the anon data, or fail. 297 */ 298 for (;;) { 299 boolean_t we_own, locked; 300 /* 301 * Note: 'we_own' will become true if we set PG_BUSY on a page. 302 */ 303 we_own = FALSE; 304 pg = anon->an_page; 305 306 /* 307 * Is page resident? Make sure it is not busy/released. 308 */ 309 if (pg) { 310 KASSERT(pg->pg_flags & PQ_ANON); 311 KASSERT(pg->uanon == anon); 312 313 /* 314 * if the page is busy, we drop all the locks and 315 * try again. 316 */ 317 if ((pg->pg_flags & (PG_BUSY|PG_RELEASED)) == 0) 318 return (VM_PAGER_OK); 319 atomic_setbits_int(&pg->pg_flags, PG_WANTED); 320 counters_inc(uvmexp_counters, flt_pgwait); 321 322 /* 323 * The last unlock must be an atomic unlock and wait 324 * on the owner of page. 325 */ 326 if (pg->uobject) { 327 /* Owner of page is UVM object. */ 328 uvmfault_unlockall(ufi, amap, NULL); 329 rwsleep_nsec(pg, pg->uobject->vmobjlock, 330 PVM | PNORELOCK, "anonget1", INFSLP); 331 } else { 332 /* Owner of page is anon. */ 333 uvmfault_unlockall(ufi, NULL, NULL); 334 rwsleep_nsec(pg, anon->an_lock, PVM | PNORELOCK, 335 "anonget2", INFSLP); 336 } 337 } else { 338 /* 339 * No page, therefore allocate one. 340 */ 341 pg = uvm_pagealloc(NULL, 0, anon, 0); 342 if (pg == NULL) { 343 /* Out of memory. Wait a little. */ 344 uvmfault_unlockall(ufi, amap, NULL); 345 counters_inc(uvmexp_counters, flt_noram); 346 uvm_wait("flt_noram1"); 347 } else { 348 /* PG_BUSY bit is set. */ 349 we_own = TRUE; 350 uvmfault_unlockall(ufi, amap, NULL); 351 352 /* 353 * Pass a PG_BUSY+PG_FAKE+PG_CLEAN page into 354 * the uvm_swap_get() function with all data 355 * structures unlocked. Note that it is OK 356 * to read an_swslot here, because we hold 357 * PG_BUSY on the page. 358 */ 359 counters_inc(uvmexp_counters, pageins); 360 error = uvm_swap_get(pg, anon->an_swslot, 361 PGO_SYNCIO); 362 363 /* 364 * We clean up after the I/O below in the 365 * 'we_own' case. 366 */ 367 } 368 } 369 370 /* 371 * Re-lock the map and anon. 372 */ 373 locked = uvmfault_relock(ufi); 374 if (locked || we_own) { 375 rw_enter(anon->an_lock, RW_WRITE); 376 } 377 378 /* 379 * If we own the page (i.e. we set PG_BUSY), then we need 380 * to clean up after the I/O. There are three cases to 381 * consider: 382 * 383 * 1) Page was released during I/O: free anon and ReFault. 384 * 2) I/O not OK. Free the page and cause the fault to fail. 385 * 3) I/O OK! Activate the page and sync with the non-we_own 386 * case (i.e. drop anon lock if not locked). 387 */ 388 if (we_own) { 389 if (pg->pg_flags & PG_WANTED) { 390 wakeup(pg); 391 } 392 393 /* 394 * if we were RELEASED during I/O, then our anon is 395 * no longer part of an amap. we need to free the 396 * anon and try again. 397 */ 398 if (pg->pg_flags & PG_RELEASED) { 399 KASSERT(anon->an_ref == 0); 400 /* 401 * Released while we had unlocked amap. 402 */ 403 if (locked) 404 uvmfault_unlockall(ufi, NULL, NULL); 405 uvm_anon_release(anon); /* frees page for us */ 406 counters_inc(uvmexp_counters, flt_pgrele); 407 return (VM_PAGER_REFAULT); /* refault! */ 408 } 409 410 if (error != VM_PAGER_OK) { 411 KASSERT(error != VM_PAGER_PEND); 412 413 /* remove page from anon */ 414 anon->an_page = NULL; 415 416 /* 417 * Remove the swap slot from the anon and 418 * mark the anon as having no real slot. 419 * Do not free the swap slot, thus preventing 420 * it from being used again. 421 */ 422 uvm_swap_markbad(anon->an_swslot, 1); 423 anon->an_swslot = SWSLOT_BAD; 424 425 /* 426 * Note: page was never !PG_BUSY, so it 427 * cannot be mapped and thus no need to 428 * pmap_page_protect() it. 429 */ 430 uvm_lock_pageq(); 431 uvm_pagefree(pg); 432 uvm_unlock_pageq(); 433 434 if (locked) { 435 uvmfault_unlockall(ufi, NULL, NULL); 436 } 437 rw_exit(anon->an_lock); 438 return (VM_PAGER_ERROR); 439 } 440 441 /* 442 * We have successfully read the page, activate it. 443 */ 444 pmap_clear_modify(pg); 445 uvm_lock_pageq(); 446 uvm_pageactivate(pg); 447 uvm_unlock_pageq(); 448 atomic_clearbits_int(&pg->pg_flags, 449 PG_WANTED|PG_BUSY|PG_FAKE); 450 UVM_PAGE_OWN(pg, NULL); 451 } 452 453 /* 454 * We were not able to re-lock the map - restart the fault. 455 */ 456 if (!locked) { 457 if (we_own) { 458 rw_exit(anon->an_lock); 459 } 460 return (VM_PAGER_REFAULT); 461 } 462 463 /* 464 * Verify that no one has touched the amap and moved 465 * the anon on us. 466 */ 467 if (ufi != NULL && amap_lookup(&ufi->entry->aref, 468 ufi->orig_rvaddr - ufi->entry->start) != anon) { 469 470 uvmfault_unlockall(ufi, amap, NULL); 471 return (VM_PAGER_REFAULT); 472 } 473 474 /* 475 * Retry.. 476 */ 477 counters_inc(uvmexp_counters, flt_anretry); 478 continue; 479 480 } 481 /*NOTREACHED*/ 482 } 483 484 /* 485 * Update statistics after fault resolution. 486 * - maxrss 487 */ 488 void 489 uvmfault_update_stats(struct uvm_faultinfo *ufi) 490 { 491 struct vm_map *map; 492 struct proc *p; 493 vsize_t res; 494 495 map = ufi->orig_map; 496 497 /* 498 * If this is a nested pmap (eg, a virtual machine pmap managed 499 * by vmm(4) on amd64/i386), don't do any updating, just return. 500 * 501 * pmap_nested() on other archs is #defined to 0, so this is a 502 * no-op. 503 */ 504 if (pmap_nested(map->pmap)) 505 return; 506 507 /* Update the maxrss for the process. */ 508 if (map->flags & VM_MAP_ISVMSPACE) { 509 p = curproc; 510 KASSERT(p != NULL && &p->p_vmspace->vm_map == map); 511 512 res = pmap_resident_count(map->pmap); 513 /* Convert res from pages to kilobytes. */ 514 res <<= (PAGE_SHIFT - 10); 515 516 if (p->p_ru.ru_maxrss < res) 517 p->p_ru.ru_maxrss = res; 518 } 519 } 520 521 /* 522 * F A U L T - m a i n e n t r y p o i n t 523 */ 524 525 /* 526 * uvm_fault: page fault handler 527 * 528 * => called from MD code to resolve a page fault 529 * => VM data structures usually should be unlocked. however, it is 530 * possible to call here with the main map locked if the caller 531 * gets a write lock, sets it recursive, and then calls us (c.f. 532 * uvm_map_pageable). this should be avoided because it keeps 533 * the map locked off during I/O. 534 * => MUST NEVER BE CALLED IN INTERRUPT CONTEXT 535 */ 536 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 537 ~PROT_WRITE : PROT_MASK) 538 struct uvm_faultctx { 539 /* 540 * the following members are set up by uvm_fault_check() and 541 * read-only after that. 542 */ 543 vm_prot_t enter_prot; 544 vm_prot_t access_type; 545 vaddr_t startva; 546 int npages; 547 int centeridx; 548 boolean_t narrow; 549 boolean_t wired; 550 paddr_t pa_flags; 551 }; 552 553 int uvm_fault_check( 554 struct uvm_faultinfo *, struct uvm_faultctx *, 555 struct vm_anon ***, vm_fault_t); 556 557 int uvm_fault_upper( 558 struct uvm_faultinfo *, struct uvm_faultctx *, 559 struct vm_anon **, vm_fault_t); 560 boolean_t uvm_fault_upper_lookup( 561 struct uvm_faultinfo *, const struct uvm_faultctx *, 562 struct vm_anon **, struct vm_page **); 563 564 int uvm_fault_lower( 565 struct uvm_faultinfo *, struct uvm_faultctx *, 566 struct vm_page **, vm_fault_t); 567 568 int 569 uvm_fault(vm_map_t orig_map, vaddr_t vaddr, vm_fault_t fault_type, 570 vm_prot_t access_type) 571 { 572 struct uvm_faultinfo ufi; 573 struct uvm_faultctx flt; 574 boolean_t shadowed; 575 struct vm_anon *anons_store[UVM_MAXRANGE], **anons; 576 struct vm_page *pages[UVM_MAXRANGE]; 577 int error; 578 579 counters_inc(uvmexp_counters, faults); 580 TRACEPOINT(uvm, fault, vaddr, fault_type, access_type, NULL); 581 582 /* 583 * init the IN parameters in the ufi 584 */ 585 ufi.orig_map = orig_map; 586 ufi.orig_rvaddr = trunc_page(vaddr); 587 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ 588 flt.access_type = access_type; 589 flt.narrow = FALSE; /* assume normal fault for now */ 590 flt.wired = FALSE; /* assume non-wired fault for now */ 591 592 error = ERESTART; 593 while (error == ERESTART) { /* ReFault: */ 594 anons = anons_store; 595 596 error = uvm_fault_check(&ufi, &flt, &anons, fault_type); 597 if (error != 0) 598 continue; 599 600 /* True if there is an anon at the faulting address */ 601 shadowed = uvm_fault_upper_lookup(&ufi, &flt, anons, pages); 602 if (shadowed == TRUE) { 603 /* case 1: fault on an anon in our amap */ 604 error = uvm_fault_upper(&ufi, &flt, anons, fault_type); 605 } else { 606 struct uvm_object *uobj = ufi.entry->object.uvm_obj; 607 608 /* 609 * if the desired page is not shadowed by the amap and 610 * we have a backing object, then we check to see if 611 * the backing object would prefer to handle the fault 612 * itself (rather than letting us do it with the usual 613 * pgo_get hook). the backing object signals this by 614 * providing a pgo_fault routine. 615 */ 616 if (uobj != NULL && uobj->pgops->pgo_fault != NULL) { 617 KERNEL_LOCK(); 618 rw_enter(uobj->vmobjlock, RW_WRITE); 619 error = uobj->pgops->pgo_fault(&ufi, 620 flt.startva, pages, flt.npages, 621 flt.centeridx, fault_type, flt.access_type, 622 PGO_LOCKED); 623 KERNEL_UNLOCK(); 624 625 if (error == VM_PAGER_OK) 626 error = 0; 627 else if (error == VM_PAGER_REFAULT) 628 error = ERESTART; 629 else 630 error = EACCES; 631 } else { 632 /* case 2: fault on backing obj or zero fill */ 633 error = uvm_fault_lower(&ufi, &flt, pages, 634 fault_type); 635 } 636 } 637 } 638 639 return error; 640 } 641 642 /* 643 * uvm_fault_check: check prot, handle needs-copy, etc. 644 * 645 * 1. lookup entry. 646 * 2. check protection. 647 * 3. adjust fault condition (mainly for simulated fault). 648 * 4. handle needs-copy (lazy amap copy). 649 * 5. establish range of interest for neighbor fault (aka pre-fault). 650 * 6. look up anons (if amap exists). 651 * 7. flush pages (if MADV_SEQUENTIAL) 652 * 653 * => called with nothing locked. 654 * => if we fail (result != 0) we unlock everything. 655 * => initialize/adjust many members of flt. 656 */ 657 int 658 uvm_fault_check(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 659 struct vm_anon ***ranons, vm_fault_t fault_type) 660 { 661 struct vm_amap *amap; 662 struct uvm_object *uobj; 663 int nback, nforw; 664 665 /* 666 * lookup and lock the maps 667 */ 668 if (uvmfault_lookup(ufi, FALSE) == FALSE) { 669 return EFAULT; 670 } 671 /* locked: maps(read) */ 672 673 #ifdef DIAGNOSTIC 674 if ((ufi->map->flags & VM_MAP_PAGEABLE) == 0) 675 panic("uvm_fault: fault on non-pageable map (%p, 0x%lx)", 676 ufi->map, ufi->orig_rvaddr); 677 #endif 678 679 /* 680 * check protection 681 */ 682 if ((ufi->entry->protection & flt->access_type) != flt->access_type) { 683 uvmfault_unlockmaps(ufi, FALSE); 684 return EACCES; 685 } 686 687 /* 688 * "enter_prot" is the protection we want to enter the page in at. 689 * for certain pages (e.g. copy-on-write pages) this protection can 690 * be more strict than ufi->entry->protection. "wired" means either 691 * the entry is wired or we are fault-wiring the pg. 692 */ 693 flt->enter_prot = ufi->entry->protection; 694 flt->pa_flags = UVM_ET_ISWC(ufi->entry) ? PMAP_WC : 0; 695 if (VM_MAPENT_ISWIRED(ufi->entry) || (fault_type == VM_FAULT_WIRE)) { 696 flt->wired = TRUE; 697 flt->access_type = flt->enter_prot; /* full access for wired */ 698 /* don't look for neighborhood * pages on "wire" fault */ 699 flt->narrow = TRUE; 700 } 701 702 /* handle "needs_copy" case. */ 703 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 704 if ((flt->access_type & PROT_WRITE) || 705 (ufi->entry->object.uvm_obj == NULL)) { 706 /* need to clear */ 707 uvmfault_unlockmaps(ufi, FALSE); 708 uvmfault_amapcopy(ufi); 709 counters_inc(uvmexp_counters, flt_amcopy); 710 return ERESTART; 711 } else { 712 /* 713 * ensure that we pmap_enter page R/O since 714 * needs_copy is still true 715 */ 716 flt->enter_prot &= ~PROT_WRITE; 717 } 718 } 719 720 /* 721 * identify the players 722 */ 723 amap = ufi->entry->aref.ar_amap; /* upper layer */ 724 uobj = ufi->entry->object.uvm_obj; /* lower layer */ 725 726 /* 727 * check for a case 0 fault. if nothing backing the entry then 728 * error now. 729 */ 730 if (amap == NULL && uobj == NULL) { 731 uvmfault_unlockmaps(ufi, FALSE); 732 return EFAULT; 733 } 734 735 /* 736 * for a case 2B fault waste no time on adjacent pages because 737 * they are likely already entered. 738 */ 739 if (uobj != NULL && amap != NULL && 740 (flt->access_type & PROT_WRITE) != 0) { 741 /* wide fault (!narrow) */ 742 flt->narrow = TRUE; 743 } 744 745 /* 746 * establish range of interest based on advice from mapper 747 * and then clip to fit map entry. note that we only want 748 * to do this the first time through the fault. if we 749 * ReFault we will disable this by setting "narrow" to true. 750 */ 751 if (flt->narrow == FALSE) { 752 753 /* wide fault (!narrow) */ 754 nback = min(uvmadvice[ufi->entry->advice].nback, 755 (ufi->orig_rvaddr - ufi->entry->start) >> PAGE_SHIFT); 756 flt->startva = ufi->orig_rvaddr - ((vsize_t)nback << PAGE_SHIFT); 757 nforw = min(uvmadvice[ufi->entry->advice].nforw, 758 ((ufi->entry->end - ufi->orig_rvaddr) >> PAGE_SHIFT) - 1); 759 /* 760 * note: "-1" because we don't want to count the 761 * faulting page as forw 762 */ 763 flt->npages = nback + nforw + 1; 764 flt->centeridx = nback; 765 766 flt->narrow = TRUE; /* ensure only once per-fault */ 767 } else { 768 /* narrow fault! */ 769 nback = nforw = 0; 770 flt->startva = ufi->orig_rvaddr; 771 flt->npages = 1; 772 flt->centeridx = 0; 773 } 774 775 /* 776 * if we've got an amap then lock it and extract current anons. 777 */ 778 if (amap) { 779 amap_lock(amap); 780 amap_lookups(&ufi->entry->aref, 781 flt->startva - ufi->entry->start, *ranons, flt->npages); 782 } else { 783 *ranons = NULL; /* to be safe */ 784 } 785 786 /* 787 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages 788 * now and then forget about them (for the rest of the fault). 789 */ 790 if (ufi->entry->advice == MADV_SEQUENTIAL && nback != 0) { 791 /* flush back-page anons? */ 792 if (amap) 793 uvmfault_anonflush(*ranons, nback); 794 795 /* 796 * flush object? 797 */ 798 if (uobj) { 799 voff_t uoff; 800 801 uoff = (flt->startva - ufi->entry->start) + ufi->entry->offset; 802 rw_enter(uobj->vmobjlock, RW_WRITE); 803 (void) uobj->pgops->pgo_flush(uobj, uoff, uoff + 804 ((vsize_t)nback << PAGE_SHIFT), PGO_DEACTIVATE); 805 rw_exit(uobj->vmobjlock); 806 } 807 808 /* now forget about the backpages */ 809 if (amap) 810 *ranons += nback; 811 flt->startva += ((vsize_t)nback << PAGE_SHIFT); 812 flt->npages -= nback; 813 flt->centeridx = 0; 814 } 815 816 return 0; 817 } 818 819 /* 820 * uvm_fault_upper_lookup: look up existing h/w mapping and amap. 821 * 822 * iterate range of interest: 823 * 1. check if h/w mapping exists. if yes, we don't care 824 * 2. check if anon exists. if not, page is lower. 825 * 3. if anon exists, enter h/w mapping for neighbors. 826 * 827 * => called with amap locked (if exists). 828 */ 829 boolean_t 830 uvm_fault_upper_lookup(struct uvm_faultinfo *ufi, 831 const struct uvm_faultctx *flt, struct vm_anon **anons, 832 struct vm_page **pages) 833 { 834 struct vm_amap *amap = ufi->entry->aref.ar_amap; 835 struct vm_anon *anon; 836 boolean_t shadowed; 837 vaddr_t currva; 838 paddr_t pa; 839 int lcv; 840 841 /* locked: maps(read), amap(if there) */ 842 KASSERT(amap == NULL || 843 rw_write_held(amap->am_lock)); 844 845 /* 846 * map in the backpages and frontpages we found in the amap in hopes 847 * of preventing future faults. we also init the pages[] array as 848 * we go. 849 */ 850 currva = flt->startva; 851 shadowed = FALSE; 852 for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) { 853 /* 854 * dont play with VAs that are already mapped 855 * except for center) 856 */ 857 if (lcv != flt->centeridx && 858 pmap_extract(ufi->orig_map->pmap, currva, &pa)) { 859 pages[lcv] = PGO_DONTCARE; 860 continue; 861 } 862 863 /* 864 * unmapped or center page. check if any anon at this level. 865 */ 866 if (amap == NULL || anons[lcv] == NULL) { 867 pages[lcv] = NULL; 868 continue; 869 } 870 871 /* 872 * check for present page and map if possible. 873 */ 874 pages[lcv] = PGO_DONTCARE; 875 if (lcv == flt->centeridx) { /* save center for later! */ 876 shadowed = TRUE; 877 continue; 878 } 879 anon = anons[lcv]; 880 KASSERT(anon->an_lock == amap->am_lock); 881 if (anon->an_page && 882 (anon->an_page->pg_flags & (PG_RELEASED|PG_BUSY)) == 0) { 883 uvm_lock_pageq(); 884 uvm_pageactivate(anon->an_page); /* reactivate */ 885 uvm_unlock_pageq(); 886 counters_inc(uvmexp_counters, flt_namap); 887 888 /* 889 * Since this isn't the page that's actually faulting, 890 * ignore pmap_enter() failures; it's not critical 891 * that we enter these right now. 892 */ 893 (void) pmap_enter(ufi->orig_map->pmap, currva, 894 VM_PAGE_TO_PHYS(anon->an_page) | flt->pa_flags, 895 (anon->an_ref > 1) ? 896 (flt->enter_prot & ~PROT_WRITE) : flt->enter_prot, 897 PMAP_CANFAIL | 898 (VM_MAPENT_ISWIRED(ufi->entry) ? PMAP_WIRED : 0)); 899 } 900 } 901 if (flt->npages > 1) 902 pmap_update(ufi->orig_map->pmap); 903 904 return shadowed; 905 } 906 907 /* 908 * uvm_fault_upper: handle upper fault. 909 * 910 * 1. acquire anon lock. 911 * 2. get anon. let uvmfault_anonget do the dirty work. 912 * 3. if COW, promote data to new anon 913 * 4. enter h/w mapping 914 */ 915 int 916 uvm_fault_upper(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 917 struct vm_anon **anons, vm_fault_t fault_type) 918 { 919 struct vm_amap *amap = ufi->entry->aref.ar_amap; 920 struct vm_anon *oanon, *anon = anons[flt->centeridx]; 921 struct vm_page *pg = NULL; 922 int error, ret; 923 924 /* locked: maps(read), amap, anon */ 925 KASSERT(rw_write_held(amap->am_lock)); 926 KASSERT(anon->an_lock == amap->am_lock); 927 928 /* 929 * no matter if we have case 1A or case 1B we are going to need to 930 * have the anon's memory resident. ensure that now. 931 */ 932 /* 933 * let uvmfault_anonget do the dirty work. 934 * if it fails (!OK) it will unlock everything for us. 935 * if it succeeds, locks are still valid and locked. 936 * also, if it is OK, then the anon's page is on the queues. 937 * if the page is on loan from a uvm_object, then anonget will 938 * lock that object for us if it does not fail. 939 */ 940 error = uvmfault_anonget(ufi, amap, anon); 941 switch (error) { 942 case VM_PAGER_OK: 943 break; 944 945 case VM_PAGER_REFAULT: 946 return ERESTART; 947 948 case VM_PAGER_ERROR: 949 /* 950 * An error occurred while trying to bring in the 951 * page -- this is the only error we return right 952 * now. 953 */ 954 return EACCES; /* XXX */ 955 default: 956 #ifdef DIAGNOSTIC 957 panic("uvm_fault: uvmfault_anonget -> %d", error); 958 #else 959 return EACCES; 960 #endif 961 } 962 963 KASSERT(rw_write_held(amap->am_lock)); 964 KASSERT(anon->an_lock == amap->am_lock); 965 966 /* 967 * if we are case 1B then we will need to allocate a new blank 968 * anon to transfer the data into. note that we have a lock 969 * on anon, so no one can busy or release the page until we are done. 970 * also note that the ref count can't drop to zero here because 971 * it is > 1 and we are only dropping one ref. 972 * 973 * in the (hopefully very rare) case that we are out of RAM we 974 * will unlock, wait for more RAM, and refault. 975 * 976 * if we are out of anon VM we wait for RAM to become available. 977 */ 978 979 if ((flt->access_type & PROT_WRITE) != 0 && anon->an_ref > 1) { 980 counters_inc(uvmexp_counters, flt_acow); 981 oanon = anon; /* oanon = old */ 982 anon = uvm_analloc(); 983 if (anon) { 984 anon->an_lock = amap->am_lock; 985 pg = uvm_pagealloc(NULL, 0, anon, 0); 986 } 987 988 /* check for out of RAM */ 989 if (anon == NULL || pg == NULL) { 990 uvmfault_unlockall(ufi, amap, NULL); 991 if (anon == NULL) 992 counters_inc(uvmexp_counters, flt_noanon); 993 else { 994 anon->an_lock = NULL; 995 anon->an_ref--; 996 uvm_anfree(anon); 997 counters_inc(uvmexp_counters, flt_noram); 998 } 999 1000 if (uvm_swapisfull()) 1001 return ENOMEM; 1002 1003 /* out of RAM, wait for more */ 1004 if (anon == NULL) 1005 uvm_anwait(); 1006 else 1007 uvm_wait("flt_noram3"); 1008 return ERESTART; 1009 } 1010 1011 /* got all resources, replace anon with nanon */ 1012 uvm_pagecopy(oanon->an_page, pg); /* pg now !PG_CLEAN */ 1013 /* un-busy! new page */ 1014 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE); 1015 UVM_PAGE_OWN(pg, NULL); 1016 ret = amap_add(&ufi->entry->aref, 1017 ufi->orig_rvaddr - ufi->entry->start, anon, 1); 1018 KASSERT(ret == 0); 1019 1020 /* deref: can not drop to zero here by defn! */ 1021 oanon->an_ref--; 1022 1023 #if defined(MULTIPROCESSOR) && !defined(__HAVE_PMAP_MPSAFE_ENTER_COW) 1024 /* 1025 * If there are multiple threads, either uvm or the 1026 * pmap has to make sure no threads see the old RO 1027 * mapping once any have seen the new RW mapping. 1028 * uvm does it by inserting the new mapping RO and 1029 * letting it fault again. 1030 * This is only a problem on MP systems. 1031 */ 1032 if (P_HASSIBLING(curproc)) { 1033 flt->enter_prot &= ~PROT_WRITE; 1034 flt->access_type &= ~PROT_WRITE; 1035 } 1036 #endif 1037 1038 /* 1039 * note: anon is _not_ locked, but we have the sole references 1040 * to in from amap. 1041 * thus, no one can get at it until we are done with it. 1042 */ 1043 } else { 1044 counters_inc(uvmexp_counters, flt_anon); 1045 oanon = anon; 1046 pg = anon->an_page; 1047 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */ 1048 flt->enter_prot = flt->enter_prot & ~PROT_WRITE; 1049 } 1050 1051 /* 1052 * now map the page in . 1053 */ 1054 if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr, 1055 VM_PAGE_TO_PHYS(pg) | flt->pa_flags, flt->enter_prot, 1056 flt->access_type | PMAP_CANFAIL | (flt->wired ? PMAP_WIRED : 0)) != 0) { 1057 /* 1058 * No need to undo what we did; we can simply think of 1059 * this as the pmap throwing away the mapping information. 1060 * 1061 * We do, however, have to go through the ReFault path, 1062 * as the map may change while we're asleep. 1063 */ 1064 uvmfault_unlockall(ufi, amap, NULL); 1065 if (uvm_swapisfull()) { 1066 /* XXX instrumentation */ 1067 return ENOMEM; 1068 } 1069 /* XXX instrumentation */ 1070 uvm_wait("flt_pmfail1"); 1071 return ERESTART; 1072 } 1073 1074 /* 1075 * ... update the page queues. 1076 */ 1077 uvm_lock_pageq(); 1078 1079 if (fault_type == VM_FAULT_WIRE) { 1080 uvm_pagewire(pg); 1081 /* 1082 * since the now-wired page cannot be paged out, 1083 * release its swap resources for others to use. 1084 * since an anon with no swap cannot be PG_CLEAN, 1085 * clear its clean flag now. 1086 */ 1087 atomic_clearbits_int(&pg->pg_flags, PG_CLEAN); 1088 uvm_anon_dropswap(anon); 1089 } else { 1090 /* activate it */ 1091 uvm_pageactivate(pg); 1092 } 1093 1094 uvm_unlock_pageq(); 1095 1096 /* 1097 * done case 1! finish up by unlocking everything and returning success 1098 */ 1099 uvmfault_unlockall(ufi, amap, NULL); 1100 pmap_update(ufi->orig_map->pmap); 1101 return 0; 1102 } 1103 1104 /* 1105 * uvm_fault_lower_lookup: look up on-memory uobj pages. 1106 * 1107 * 1. get on-memory pages. 1108 * 2. if failed, give up (get only center page later). 1109 * 3. if succeeded, enter h/w mapping of neighbor pages. 1110 */ 1111 1112 struct vm_page * 1113 uvm_fault_lower_lookup( 1114 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 1115 struct vm_page **pages) 1116 { 1117 struct uvm_object *uobj = ufi->entry->object.uvm_obj; 1118 struct vm_page *uobjpage = NULL; 1119 int lcv, gotpages; 1120 vaddr_t currva; 1121 1122 rw_enter(uobj->vmobjlock, RW_WRITE); 1123 1124 counters_inc(uvmexp_counters, flt_lget); 1125 gotpages = flt->npages; 1126 (void) uobj->pgops->pgo_get(uobj, 1127 ufi->entry->offset + (flt->startva - ufi->entry->start), 1128 pages, &gotpages, flt->centeridx, 1129 flt->access_type & MASK(ufi->entry), ufi->entry->advice, 1130 PGO_LOCKED); 1131 1132 /* 1133 * check for pages to map, if we got any 1134 */ 1135 if (gotpages == 0) { 1136 return NULL; 1137 } 1138 1139 currva = flt->startva; 1140 for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) { 1141 if (pages[lcv] == NULL || 1142 pages[lcv] == PGO_DONTCARE) 1143 continue; 1144 1145 KASSERT((pages[lcv]->pg_flags & PG_RELEASED) == 0); 1146 1147 /* 1148 * if center page is resident and not 1149 * PG_BUSY, then pgo_get made it PG_BUSY 1150 * for us and gave us a handle to it. 1151 * remember this page as "uobjpage." 1152 * (for later use). 1153 */ 1154 if (lcv == flt->centeridx) { 1155 uobjpage = pages[lcv]; 1156 continue; 1157 } 1158 1159 /* 1160 * note: calling pgo_get with locked data 1161 * structures returns us pages which are 1162 * neither busy nor released, so we don't 1163 * need to check for this. we can just 1164 * directly enter the page (after moving it 1165 * to the head of the active queue [useful?]). 1166 */ 1167 1168 uvm_lock_pageq(); 1169 uvm_pageactivate(pages[lcv]); /* reactivate */ 1170 uvm_unlock_pageq(); 1171 counters_inc(uvmexp_counters, flt_nomap); 1172 1173 /* 1174 * Since this page isn't the page that's 1175 * actually faulting, ignore pmap_enter() 1176 * failures; it's not critical that we 1177 * enter these right now. 1178 */ 1179 (void) pmap_enter(ufi->orig_map->pmap, currva, 1180 VM_PAGE_TO_PHYS(pages[lcv]) | flt->pa_flags, 1181 flt->enter_prot & MASK(ufi->entry), 1182 PMAP_CANFAIL | 1183 (flt->wired ? PMAP_WIRED : 0)); 1184 1185 /* 1186 * NOTE: page can't be PG_WANTED because 1187 * we've held the lock the whole time 1188 * we've had the handle. 1189 */ 1190 atomic_clearbits_int(&pages[lcv]->pg_flags, PG_BUSY); 1191 UVM_PAGE_OWN(pages[lcv], NULL); 1192 } 1193 pmap_update(ufi->orig_map->pmap); 1194 1195 return uobjpage; 1196 } 1197 1198 /* 1199 * uvm_fault_lower: handle lower fault. 1200 * 1201 */ 1202 int 1203 uvm_fault_lower(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1204 struct vm_page **pages, vm_fault_t fault_type) 1205 { 1206 struct vm_amap *amap = ufi->entry->aref.ar_amap; 1207 struct uvm_object *uobj = ufi->entry->object.uvm_obj; 1208 boolean_t promote, locked; 1209 int result; 1210 struct vm_page *uobjpage, *pg = NULL; 1211 struct vm_anon *anon = NULL; 1212 voff_t uoff; 1213 1214 /* 1215 * now, if the desired page is not shadowed by the amap and we have 1216 * a backing object that does not have a special fault routine, then 1217 * we ask (with pgo_get) the object for resident pages that we care 1218 * about and attempt to map them in. we do not let pgo_get block 1219 * (PGO_LOCKED). 1220 */ 1221 if (uobj == NULL) { 1222 /* zero fill; don't care neighbor pages */ 1223 uobjpage = NULL; 1224 } else { 1225 uobjpage = uvm_fault_lower_lookup(ufi, flt, pages); 1226 } 1227 1228 /* 1229 * note that at this point we are done with any front or back pages. 1230 * we are now going to focus on the center page (i.e. the one we've 1231 * faulted on). if we have faulted on the bottom (uobj) 1232 * layer [i.e. case 2] and the page was both present and available, 1233 * then we've got a pointer to it as "uobjpage" and we've already 1234 * made it BUSY. 1235 */ 1236 1237 /* 1238 * locked: 1239 */ 1240 KASSERT(amap == NULL || 1241 rw_write_held(amap->am_lock)); 1242 KASSERT(uobj == NULL || 1243 rw_write_held(uobj->vmobjlock)); 1244 1245 /* 1246 * note that uobjpage can not be PGO_DONTCARE at this point. we now 1247 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we 1248 * have a backing object, check and see if we are going to promote 1249 * the data up to an anon during the fault. 1250 */ 1251 if (uobj == NULL) { 1252 uobjpage = PGO_DONTCARE; 1253 promote = TRUE; /* always need anon here */ 1254 } else { 1255 KASSERT(uobjpage != PGO_DONTCARE); 1256 promote = (flt->access_type & PROT_WRITE) && 1257 UVM_ET_ISCOPYONWRITE(ufi->entry); 1258 } 1259 1260 /* 1261 * if uobjpage is not null then we do not need to do I/O to get the 1262 * uobjpage. 1263 * 1264 * if uobjpage is null, then we need to ask the pager to 1265 * get the data for us. once we have the data, we need to reverify 1266 * the state the world. we are currently not holding any resources. 1267 */ 1268 if (uobjpage) { 1269 /* update rusage counters */ 1270 curproc->p_ru.ru_minflt++; 1271 } else { 1272 int gotpages; 1273 1274 /* update rusage counters */ 1275 curproc->p_ru.ru_majflt++; 1276 1277 uvmfault_unlockall(ufi, amap, NULL); 1278 1279 counters_inc(uvmexp_counters, flt_get); 1280 gotpages = 1; 1281 uoff = (ufi->orig_rvaddr - ufi->entry->start) + ufi->entry->offset; 1282 result = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages, 1283 0, flt->access_type & MASK(ufi->entry), ufi->entry->advice, 1284 PGO_SYNCIO); 1285 1286 /* 1287 * recover from I/O 1288 */ 1289 if (result != VM_PAGER_OK) { 1290 KASSERT(result != VM_PAGER_PEND); 1291 1292 if (result == VM_PAGER_AGAIN) { 1293 tsleep_nsec(&nowake, PVM, "fltagain2", 1294 MSEC_TO_NSEC(5)); 1295 return ERESTART; 1296 } 1297 1298 if (!UVM_ET_ISNOFAULT(ufi->entry)) 1299 return (EIO); 1300 1301 uobjpage = PGO_DONTCARE; 1302 uobj = NULL; 1303 promote = TRUE; 1304 } 1305 1306 /* re-verify the state of the world. */ 1307 locked = uvmfault_relock(ufi); 1308 if (locked && amap != NULL) 1309 amap_lock(amap); 1310 1311 /* might be changed */ 1312 if (uobjpage != PGO_DONTCARE) { 1313 uobj = uobjpage->uobject; 1314 rw_enter(uobj->vmobjlock, RW_WRITE); 1315 } 1316 1317 /* 1318 * Re-verify that amap slot is still free. if there is 1319 * a problem, we clean up. 1320 */ 1321 if (locked && amap && amap_lookup(&ufi->entry->aref, 1322 ufi->orig_rvaddr - ufi->entry->start)) { 1323 if (locked) 1324 uvmfault_unlockall(ufi, amap, NULL); 1325 locked = FALSE; 1326 } 1327 1328 /* didn't get the lock? release the page and retry. */ 1329 if (locked == FALSE && uobjpage != PGO_DONTCARE) { 1330 uvm_lock_pageq(); 1331 /* make sure it is in queues */ 1332 uvm_pageactivate(uobjpage); 1333 uvm_unlock_pageq(); 1334 1335 if (uobjpage->pg_flags & PG_WANTED) 1336 /* still holding object lock */ 1337 wakeup(uobjpage); 1338 atomic_clearbits_int(&uobjpage->pg_flags, 1339 PG_BUSY|PG_WANTED); 1340 UVM_PAGE_OWN(uobjpage, NULL); 1341 } 1342 1343 if (locked == FALSE) { 1344 if (uobjpage != PGO_DONTCARE) 1345 rw_exit(uobj->vmobjlock); 1346 return ERESTART; 1347 } 1348 1349 /* 1350 * we have the data in uobjpage which is PG_BUSY 1351 */ 1352 } 1353 1354 /* 1355 * notes: 1356 * - at this point uobjpage can not be NULL 1357 * - at this point uobjpage could be PG_WANTED (handle later) 1358 */ 1359 if (promote == FALSE) { 1360 /* 1361 * we are not promoting. if the mapping is COW ensure that we 1362 * don't give more access than we should (e.g. when doing a read 1363 * fault on a COPYONWRITE mapping we want to map the COW page in 1364 * R/O even though the entry protection could be R/W). 1365 * 1366 * set "pg" to the page we want to map in (uobjpage, usually) 1367 */ 1368 counters_inc(uvmexp_counters, flt_obj); 1369 if (UVM_ET_ISCOPYONWRITE(ufi->entry)) 1370 flt->enter_prot &= ~PROT_WRITE; 1371 pg = uobjpage; /* map in the actual object */ 1372 1373 /* assert(uobjpage != PGO_DONTCARE) */ 1374 1375 /* 1376 * we are faulting directly on the page. 1377 */ 1378 } else { 1379 /* 1380 * if we are going to promote the data to an anon we 1381 * allocate a blank anon here and plug it into our amap. 1382 */ 1383 #ifdef DIAGNOSTIC 1384 if (amap == NULL) 1385 panic("uvm_fault: want to promote data, but no anon"); 1386 #endif 1387 1388 anon = uvm_analloc(); 1389 if (anon) { 1390 /* 1391 * In `Fill in data...' below, if 1392 * uobjpage == PGO_DONTCARE, we want 1393 * a zero'd, dirty page, so have 1394 * uvm_pagealloc() do that for us. 1395 */ 1396 anon->an_lock = amap->am_lock; 1397 pg = uvm_pagealloc(NULL, 0, anon, 1398 (uobjpage == PGO_DONTCARE) ? UVM_PGA_ZERO : 0); 1399 } 1400 1401 /* 1402 * out of memory resources? 1403 */ 1404 if (anon == NULL || pg == NULL) { 1405 /* 1406 * arg! must unbusy our page and fail or sleep. 1407 */ 1408 if (uobjpage != PGO_DONTCARE) { 1409 uvm_lock_pageq(); 1410 uvm_pageactivate(uobjpage); 1411 uvm_unlock_pageq(); 1412 1413 if (uobjpage->pg_flags & PG_WANTED) 1414 wakeup(uobjpage); 1415 atomic_clearbits_int(&uobjpage->pg_flags, 1416 PG_BUSY|PG_WANTED); 1417 UVM_PAGE_OWN(uobjpage, NULL); 1418 } 1419 1420 /* unlock and fail ... */ 1421 uvmfault_unlockall(ufi, amap, uobj); 1422 if (anon == NULL) 1423 counters_inc(uvmexp_counters, flt_noanon); 1424 else { 1425 anon->an_lock = NULL; 1426 anon->an_ref--; 1427 uvm_anfree(anon); 1428 counters_inc(uvmexp_counters, flt_noram); 1429 } 1430 1431 if (uvm_swapisfull()) 1432 return (ENOMEM); 1433 1434 /* out of RAM, wait for more */ 1435 if (anon == NULL) 1436 uvm_anwait(); 1437 else 1438 uvm_wait("flt_noram5"); 1439 return ERESTART; 1440 } 1441 1442 /* 1443 * fill in the data 1444 */ 1445 if (uobjpage != PGO_DONTCARE) { 1446 counters_inc(uvmexp_counters, flt_prcopy); 1447 /* copy page [pg now dirty] */ 1448 uvm_pagecopy(uobjpage, pg); 1449 1450 /* 1451 * promote to shared amap? make sure all sharing 1452 * procs see it 1453 */ 1454 if ((amap_flags(amap) & AMAP_SHARED) != 0) { 1455 pmap_page_protect(uobjpage, PROT_NONE); 1456 } 1457 #if defined(MULTIPROCESSOR) && !defined(__HAVE_PMAP_MPSAFE_ENTER_COW) 1458 /* 1459 * Otherwise: 1460 * If there are multiple threads, either uvm or the 1461 * pmap has to make sure no threads see the old RO 1462 * mapping once any have seen the new RW mapping. 1463 * uvm does it here by forcing it to PROT_NONE before 1464 * inserting the new mapping. 1465 */ 1466 else if (P_HASSIBLING(curproc)) { 1467 pmap_page_protect(uobjpage, PROT_NONE); 1468 } 1469 #endif 1470 1471 /* dispose of uobjpage. drop handle to uobj as well. */ 1472 if (uobjpage->pg_flags & PG_WANTED) 1473 wakeup(uobjpage); 1474 atomic_clearbits_int(&uobjpage->pg_flags, 1475 PG_BUSY|PG_WANTED); 1476 UVM_PAGE_OWN(uobjpage, NULL); 1477 uvm_lock_pageq(); 1478 uvm_pageactivate(uobjpage); 1479 uvm_unlock_pageq(); 1480 rw_exit(uobj->vmobjlock); 1481 uobj = NULL; 1482 } else { 1483 counters_inc(uvmexp_counters, flt_przero); 1484 /* 1485 * Page is zero'd and marked dirty by uvm_pagealloc() 1486 * above. 1487 */ 1488 } 1489 1490 if (amap_add(&ufi->entry->aref, 1491 ufi->orig_rvaddr - ufi->entry->start, anon, 0)) { 1492 uvmfault_unlockall(ufi, amap, uobj); 1493 uvm_anfree(anon); 1494 counters_inc(uvmexp_counters, flt_noamap); 1495 1496 if (uvm_swapisfull()) 1497 return (ENOMEM); 1498 1499 amap_populate(&ufi->entry->aref, 1500 ufi->orig_rvaddr - ufi->entry->start); 1501 return ERESTART; 1502 } 1503 } 1504 1505 /* note: pg is either the uobjpage or the new page in the new anon */ 1506 /* 1507 * all resources are present. we can now map it in and free our 1508 * resources. 1509 */ 1510 if (amap == NULL) 1511 KASSERT(anon == NULL); 1512 else { 1513 KASSERT(rw_write_held(amap->am_lock)); 1514 KASSERT(anon == NULL || anon->an_lock == amap->am_lock); 1515 } 1516 if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr, 1517 VM_PAGE_TO_PHYS(pg) | flt->pa_flags, flt->enter_prot, 1518 flt->access_type | PMAP_CANFAIL | (flt->wired ? PMAP_WIRED : 0)) != 0) { 1519 /* 1520 * No need to undo what we did; we can simply think of 1521 * this as the pmap throwing away the mapping information. 1522 * 1523 * We do, however, have to go through the ReFault path, 1524 * as the map may change while we're asleep. 1525 */ 1526 if (pg->pg_flags & PG_WANTED) 1527 wakeup(pg); 1528 1529 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED); 1530 UVM_PAGE_OWN(pg, NULL); 1531 uvmfault_unlockall(ufi, amap, uobj); 1532 if (uvm_swapisfull()) { 1533 /* XXX instrumentation */ 1534 return (ENOMEM); 1535 } 1536 /* XXX instrumentation */ 1537 uvm_wait("flt_pmfail2"); 1538 return ERESTART; 1539 } 1540 1541 if (fault_type == VM_FAULT_WIRE) { 1542 uvm_lock_pageq(); 1543 uvm_pagewire(pg); 1544 uvm_unlock_pageq(); 1545 if (pg->pg_flags & PQ_AOBJ) { 1546 /* 1547 * since the now-wired page cannot be paged out, 1548 * release its swap resources for others to use. 1549 * since an aobj page with no swap cannot be clean, 1550 * mark it dirty now. 1551 * 1552 * use pg->uobject here. if the page is from a 1553 * tmpfs vnode, the pages are backed by its UAO and 1554 * not the vnode. 1555 */ 1556 KASSERT(uobj != NULL); 1557 KASSERT(uobj->vmobjlock == pg->uobject->vmobjlock); 1558 atomic_clearbits_int(&pg->pg_flags, PG_CLEAN); 1559 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT); 1560 } 1561 } else { 1562 /* activate it */ 1563 uvm_lock_pageq(); 1564 uvm_pageactivate(pg); 1565 uvm_unlock_pageq(); 1566 } 1567 1568 if (pg->pg_flags & PG_WANTED) 1569 wakeup(pg); 1570 1571 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED); 1572 UVM_PAGE_OWN(pg, NULL); 1573 uvmfault_unlockall(ufi, amap, uobj); 1574 pmap_update(ufi->orig_map->pmap); 1575 1576 return (0); 1577 } 1578 1579 1580 /* 1581 * uvm_fault_wire: wire down a range of virtual addresses in a map. 1582 * 1583 * => map may be read-locked by caller, but MUST NOT be write-locked. 1584 * => if map is read-locked, any operations which may cause map to 1585 * be write-locked in uvm_fault() must be taken care of by 1586 * the caller. See uvm_map_pageable(). 1587 */ 1588 int 1589 uvm_fault_wire(vm_map_t map, vaddr_t start, vaddr_t end, vm_prot_t access_type) 1590 { 1591 vaddr_t va; 1592 int rv; 1593 1594 /* 1595 * now fault it in a page at a time. if the fault fails then we have 1596 * to undo what we have done. note that in uvm_fault PROT_NONE 1597 * is replaced with the max protection if fault_type is VM_FAULT_WIRE. 1598 */ 1599 for (va = start ; va < end ; va += PAGE_SIZE) { 1600 rv = uvm_fault(map, va, VM_FAULT_WIRE, access_type); 1601 if (rv) { 1602 if (va != start) { 1603 uvm_fault_unwire(map, start, va); 1604 } 1605 return (rv); 1606 } 1607 } 1608 1609 return (0); 1610 } 1611 1612 /* 1613 * uvm_fault_unwire(): unwire range of virtual space. 1614 */ 1615 void 1616 uvm_fault_unwire(vm_map_t map, vaddr_t start, vaddr_t end) 1617 { 1618 1619 vm_map_lock_read(map); 1620 uvm_fault_unwire_locked(map, start, end); 1621 vm_map_unlock_read(map); 1622 } 1623 1624 /* 1625 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire(). 1626 * 1627 * => map must be at least read-locked. 1628 */ 1629 void 1630 uvm_fault_unwire_locked(vm_map_t map, vaddr_t start, vaddr_t end) 1631 { 1632 vm_map_entry_t entry, oentry = NULL, next; 1633 pmap_t pmap = vm_map_pmap(map); 1634 vaddr_t va; 1635 paddr_t pa; 1636 struct vm_page *pg; 1637 1638 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0); 1639 vm_map_assert_anylock(map); 1640 1641 /* 1642 * we assume that the area we are unwiring has actually been wired 1643 * in the first place. this means that we should be able to extract 1644 * the PAs from the pmap. 1645 */ 1646 1647 /* 1648 * find the beginning map entry for the region. 1649 */ 1650 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map)); 1651 if (uvm_map_lookup_entry(map, start, &entry) == FALSE) 1652 panic("uvm_fault_unwire_locked: address not in map"); 1653 1654 for (va = start; va < end ; va += PAGE_SIZE) { 1655 if (pmap_extract(pmap, va, &pa) == FALSE) 1656 continue; 1657 1658 /* 1659 * find the map entry for the current address. 1660 */ 1661 KASSERT(va >= entry->start); 1662 while (entry && va >= entry->end) { 1663 next = RBT_NEXT(uvm_map_addr, entry); 1664 entry = next; 1665 } 1666 1667 if (entry == NULL) 1668 return; 1669 if (va < entry->start) 1670 continue; 1671 1672 /* 1673 * lock it. 1674 */ 1675 if (entry != oentry) { 1676 if (oentry != NULL) { 1677 uvm_map_unlock_entry(oentry); 1678 } 1679 uvm_map_lock_entry(entry); 1680 oentry = entry; 1681 } 1682 1683 /* 1684 * if the entry is no longer wired, tell the pmap. 1685 */ 1686 if (VM_MAPENT_ISWIRED(entry) == 0) 1687 pmap_unwire(pmap, va); 1688 1689 pg = PHYS_TO_VM_PAGE(pa); 1690 if (pg) { 1691 uvm_lock_pageq(); 1692 uvm_pageunwire(pg); 1693 uvm_unlock_pageq(); 1694 } 1695 } 1696 1697 if (oentry != NULL) { 1698 uvm_map_unlock_entry(oentry); 1699 } 1700 } 1701 1702 /* 1703 * uvmfault_unlockmaps: unlock the maps 1704 */ 1705 void 1706 uvmfault_unlockmaps(struct uvm_faultinfo *ufi, boolean_t write_locked) 1707 { 1708 /* 1709 * ufi can be NULL when this isn't really a fault, 1710 * but merely paging in anon data. 1711 */ 1712 if (ufi == NULL) { 1713 return; 1714 } 1715 1716 uvmfault_update_stats(ufi); 1717 if (write_locked) { 1718 vm_map_unlock(ufi->map); 1719 } else { 1720 vm_map_unlock_read(ufi->map); 1721 } 1722 } 1723 1724 /* 1725 * uvmfault_unlockall: unlock everything passed in. 1726 * 1727 * => maps must be read-locked (not write-locked). 1728 */ 1729 void 1730 uvmfault_unlockall(struct uvm_faultinfo *ufi, struct vm_amap *amap, 1731 struct uvm_object *uobj) 1732 { 1733 if (uobj) 1734 rw_exit(uobj->vmobjlock); 1735 if (amap != NULL) 1736 amap_unlock(amap); 1737 uvmfault_unlockmaps(ufi, FALSE); 1738 } 1739 1740 /* 1741 * uvmfault_lookup: lookup a virtual address in a map 1742 * 1743 * => caller must provide a uvm_faultinfo structure with the IN 1744 * params properly filled in 1745 * => we will lookup the map entry (handling submaps) as we go 1746 * => if the lookup is a success we will return with the maps locked 1747 * => if "write_lock" is TRUE, we write_lock the map, otherwise we only 1748 * get a read lock. 1749 * => note that submaps can only appear in the kernel and they are 1750 * required to use the same virtual addresses as the map they 1751 * are referenced by (thus address translation between the main 1752 * map and the submap is unnecessary). 1753 */ 1754 1755 boolean_t 1756 uvmfault_lookup(struct uvm_faultinfo *ufi, boolean_t write_lock) 1757 { 1758 vm_map_t tmpmap; 1759 1760 /* 1761 * init ufi values for lookup. 1762 */ 1763 ufi->map = ufi->orig_map; 1764 ufi->size = ufi->orig_size; 1765 1766 /* 1767 * keep going down levels until we are done. note that there can 1768 * only be two levels so we won't loop very long. 1769 */ 1770 while (1) { 1771 if (ufi->orig_rvaddr < ufi->map->min_offset || 1772 ufi->orig_rvaddr >= ufi->map->max_offset) 1773 return FALSE; 1774 1775 /* lock map */ 1776 if (write_lock) { 1777 vm_map_lock(ufi->map); 1778 } else { 1779 vm_map_lock_read(ufi->map); 1780 } 1781 1782 /* lookup */ 1783 if (!uvm_map_lookup_entry(ufi->map, ufi->orig_rvaddr, 1784 &ufi->entry)) { 1785 uvmfault_unlockmaps(ufi, write_lock); 1786 return FALSE; 1787 } 1788 1789 /* reduce size if necessary */ 1790 if (ufi->entry->end - ufi->orig_rvaddr < ufi->size) 1791 ufi->size = ufi->entry->end - ufi->orig_rvaddr; 1792 1793 /* 1794 * submap? replace map with the submap and lookup again. 1795 * note: VAs in submaps must match VAs in main map. 1796 */ 1797 if (UVM_ET_ISSUBMAP(ufi->entry)) { 1798 tmpmap = ufi->entry->object.sub_map; 1799 uvmfault_unlockmaps(ufi, write_lock); 1800 ufi->map = tmpmap; 1801 continue; 1802 } 1803 1804 /* 1805 * got it! 1806 */ 1807 ufi->mapv = ufi->map->timestamp; 1808 return TRUE; 1809 1810 } /* while loop */ 1811 1812 /*NOTREACHED*/ 1813 } 1814 1815 /* 1816 * uvmfault_relock: attempt to relock the same version of the map 1817 * 1818 * => fault data structures should be unlocked before calling. 1819 * => if a success (TRUE) maps will be locked after call. 1820 */ 1821 boolean_t 1822 uvmfault_relock(struct uvm_faultinfo *ufi) 1823 { 1824 /* 1825 * ufi can be NULL when this isn't really a fault, 1826 * but merely paging in anon data. 1827 */ 1828 if (ufi == NULL) { 1829 return TRUE; 1830 } 1831 1832 counters_inc(uvmexp_counters, flt_relck); 1833 1834 /* 1835 * relock map. fail if version mismatch (in which case nothing 1836 * gets locked). 1837 */ 1838 vm_map_lock_read(ufi->map); 1839 if (ufi->mapv != ufi->map->timestamp) { 1840 vm_map_unlock_read(ufi->map); 1841 return FALSE; 1842 } 1843 1844 counters_inc(uvmexp_counters, flt_relckok); 1845 return TRUE; /* got it! */ 1846 } 1847