1 /* $OpenBSD: uvm_fault.c,v 1.115 2021/02/16 09:10:17 mpi Exp $ */ 2 /* $NetBSD: uvm_fault.c,v 1.51 2000/08/06 00:22:53 thorpej Exp $ */ 3 4 /* 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp 29 */ 30 31 /* 32 * uvm_fault.c: fault handler 33 */ 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/percpu.h> 39 #include <sys/proc.h> 40 #include <sys/malloc.h> 41 #include <sys/mman.h> 42 #include <sys/tracepoint.h> 43 44 #include <uvm/uvm.h> 45 46 /* 47 * 48 * a word on page faults: 49 * 50 * types of page faults we handle: 51 * 52 * CASE 1: upper layer faults CASE 2: lower layer faults 53 * 54 * CASE 1A CASE 1B CASE 2A CASE 2B 55 * read/write1 write>1 read/write +-cow_write/zero 56 * | | | | 57 * +--|--+ +--|--+ +-----+ + | + | +-----+ 58 * amap | V | | ---------> new | | | | ^ | 59 * +-----+ +-----+ +-----+ + | + | +--|--+ 60 * | | | 61 * +-----+ +-----+ +--|--+ | +--|--+ 62 * uobj | d/c | | d/c | | V | +----+ | 63 * +-----+ +-----+ +-----+ +-----+ 64 * 65 * d/c = don't care 66 * 67 * case [0]: layerless fault 68 * no amap or uobj is present. this is an error. 69 * 70 * case [1]: upper layer fault [anon active] 71 * 1A: [read] or [write with anon->an_ref == 1] 72 * I/O takes place in upper level anon and uobj is not touched. 73 * 1B: [write with anon->an_ref > 1] 74 * new anon is alloc'd and data is copied off ["COW"] 75 * 76 * case [2]: lower layer fault [uobj] 77 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area] 78 * I/O takes place directly in object. 79 * 2B: [write to copy_on_write] or [read on NULL uobj] 80 * data is "promoted" from uobj to a new anon. 81 * if uobj is null, then we zero fill. 82 * 83 * we follow the standard UVM locking protocol ordering: 84 * 85 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ) 86 * we hold a PG_BUSY page if we unlock for I/O 87 * 88 * 89 * the code is structured as follows: 90 * 91 * - init the "IN" params in the ufi structure 92 * ReFault: (ERESTART returned to the loop in uvm_fault) 93 * - do lookups [locks maps], check protection, handle needs_copy 94 * - check for case 0 fault (error) 95 * - establish "range" of fault 96 * - if we have an amap lock it and extract the anons 97 * - if sequential advice deactivate pages behind us 98 * - at the same time check pmap for unmapped areas and anon for pages 99 * that we could map in (and do map it if found) 100 * - check object for resident pages that we could map in 101 * - if (case 2) goto Case2 102 * - >>> handle case 1 103 * - ensure source anon is resident in RAM 104 * - if case 1B alloc new anon and copy from source 105 * - map the correct page in 106 * Case2: 107 * - >>> handle case 2 108 * - ensure source page is resident (if uobj) 109 * - if case 2B alloc new anon and copy from source (could be zero 110 * fill if uobj == NULL) 111 * - map the correct page in 112 * - done! 113 * 114 * note on paging: 115 * if we have to do I/O we place a PG_BUSY page in the correct object, 116 * unlock everything, and do the I/O. when I/O is done we must reverify 117 * the state of the world before assuming that our data structures are 118 * valid. [because mappings could change while the map is unlocked] 119 * 120 * alternative 1: unbusy the page in question and restart the page fault 121 * from the top (ReFault). this is easy but does not take advantage 122 * of the information that we already have from our previous lookup, 123 * although it is possible that the "hints" in the vm_map will help here. 124 * 125 * alternative 2: the system already keeps track of a "version" number of 126 * a map. [i.e. every time you write-lock a map (e.g. to change a 127 * mapping) you bump the version number up by one...] so, we can save 128 * the version number of the map before we release the lock and start I/O. 129 * then when I/O is done we can relock and check the version numbers 130 * to see if anything changed. this might save us some over 1 because 131 * we don't have to unbusy the page and may be less compares(?). 132 * 133 * alternative 3: put in backpointers or a way to "hold" part of a map 134 * in place while I/O is in progress. this could be complex to 135 * implement (especially with structures like amap that can be referenced 136 * by multiple map entries, and figuring out what should wait could be 137 * complex as well...). 138 * 139 * we use alternative 2. given that we are multi-threaded now we may want 140 * to reconsider the choice. 141 */ 142 143 /* 144 * local data structures 145 */ 146 struct uvm_advice { 147 int nback; 148 int nforw; 149 }; 150 151 /* 152 * page range array: set up in uvmfault_init(). 153 */ 154 static struct uvm_advice uvmadvice[MADV_MASK + 1]; 155 156 #define UVM_MAXRANGE 16 /* must be max() of nback+nforw+1 */ 157 158 /* 159 * private prototypes 160 */ 161 static void uvmfault_amapcopy(struct uvm_faultinfo *); 162 static inline void uvmfault_anonflush(struct vm_anon **, int); 163 void uvmfault_unlockmaps(struct uvm_faultinfo *, boolean_t); 164 void uvmfault_update_stats(struct uvm_faultinfo *); 165 166 /* 167 * inline functions 168 */ 169 /* 170 * uvmfault_anonflush: try and deactivate pages in specified anons 171 * 172 * => does not have to deactivate page if it is busy 173 */ 174 static inline void 175 uvmfault_anonflush(struct vm_anon **anons, int n) 176 { 177 int lcv; 178 struct vm_page *pg; 179 180 for (lcv = 0; lcv < n; lcv++) { 181 if (anons[lcv] == NULL) 182 continue; 183 KASSERT(rw_lock_held(anons[lcv]->an_lock)); 184 pg = anons[lcv]->an_page; 185 if (pg && (pg->pg_flags & PG_BUSY) == 0) { 186 uvm_lock_pageq(); 187 if (pg->wire_count == 0) { 188 pmap_page_protect(pg, PROT_NONE); 189 uvm_pagedeactivate(pg); 190 } 191 uvm_unlock_pageq(); 192 } 193 } 194 } 195 196 /* 197 * normal functions 198 */ 199 /* 200 * uvmfault_init: compute proper values for the uvmadvice[] array. 201 */ 202 void 203 uvmfault_init(void) 204 { 205 int npages; 206 207 npages = atop(16384); 208 if (npages > 0) { 209 KASSERT(npages <= UVM_MAXRANGE / 2); 210 uvmadvice[MADV_NORMAL].nforw = npages; 211 uvmadvice[MADV_NORMAL].nback = npages - 1; 212 } 213 214 npages = atop(32768); 215 if (npages > 0) { 216 KASSERT(npages <= UVM_MAXRANGE / 2); 217 uvmadvice[MADV_SEQUENTIAL].nforw = npages - 1; 218 uvmadvice[MADV_SEQUENTIAL].nback = npages; 219 } 220 } 221 222 /* 223 * uvmfault_amapcopy: clear "needs_copy" in a map. 224 * 225 * => called with VM data structures unlocked (usually, see below) 226 * => we get a write lock on the maps and clear needs_copy for a VA 227 * => if we are out of RAM we sleep (waiting for more) 228 */ 229 static void 230 uvmfault_amapcopy(struct uvm_faultinfo *ufi) 231 { 232 for (;;) { 233 /* no mapping? give up. */ 234 if (uvmfault_lookup(ufi, TRUE) == FALSE) 235 return; 236 237 /* copy if needed. */ 238 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) 239 amap_copy(ufi->map, ufi->entry, M_NOWAIT, 240 UVM_ET_ISSTACK(ufi->entry) ? FALSE : TRUE, 241 ufi->orig_rvaddr, ufi->orig_rvaddr + 1); 242 243 /* didn't work? must be out of RAM. sleep. */ 244 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 245 uvmfault_unlockmaps(ufi, TRUE); 246 uvm_wait("fltamapcopy"); 247 continue; 248 } 249 250 /* got it! */ 251 uvmfault_unlockmaps(ufi, TRUE); 252 return; 253 } 254 /*NOTREACHED*/ 255 } 256 257 /* 258 * uvmfault_anonget: get data in an anon into a non-busy, non-released 259 * page in that anon. 260 * 261 * => Map, amap and thus anon should be locked by caller. 262 * => If we fail, we unlock everything and error is returned. 263 * => If we are successful, return with everything still locked. 264 * => We do not move the page on the queues [gets moved later]. If we 265 * allocate a new page [we_own], it gets put on the queues. Either way, 266 * the result is that the page is on the queues at return time 267 */ 268 int 269 uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap, 270 struct vm_anon *anon) 271 { 272 struct vm_page *pg; 273 int error; 274 275 KASSERT(rw_lock_held(anon->an_lock)); 276 KASSERT(anon->an_lock == amap->am_lock); 277 278 /* Increment the counters.*/ 279 counters_inc(uvmexp_counters, flt_anget); 280 if (anon->an_page) { 281 curproc->p_ru.ru_minflt++; 282 } else { 283 curproc->p_ru.ru_majflt++; 284 } 285 error = 0; 286 287 /* 288 * Loop until we get the anon data, or fail. 289 */ 290 for (;;) { 291 boolean_t we_own, locked; 292 /* 293 * Note: 'we_own' will become true if we set PG_BUSY on a page. 294 */ 295 we_own = FALSE; 296 pg = anon->an_page; 297 298 /* 299 * Is page resident? Make sure it is not busy/released. 300 */ 301 if (pg) { 302 KASSERT(pg->pg_flags & PQ_ANON); 303 KASSERT(pg->uanon == anon); 304 305 /* 306 * if the page is busy, we drop all the locks and 307 * try again. 308 */ 309 if ((pg->pg_flags & (PG_BUSY|PG_RELEASED)) == 0) 310 return (VM_PAGER_OK); 311 atomic_setbits_int(&pg->pg_flags, PG_WANTED); 312 counters_inc(uvmexp_counters, flt_pgwait); 313 314 /* 315 * The last unlock must be an atomic unlock and wait 316 * on the owner of page. 317 */ 318 if (pg->uobject) { 319 /* Owner of page is UVM object. */ 320 uvmfault_unlockall(ufi, amap, NULL); 321 tsleep_nsec(pg, PVM, "anonget1", INFSLP); 322 } else { 323 /* Owner of page is anon. */ 324 uvmfault_unlockall(ufi, NULL, NULL); 325 rwsleep_nsec(pg, anon->an_lock, PVM | PNORELOCK, 326 "anonget2", INFSLP); 327 } 328 } else { 329 /* 330 * No page, therefore allocate one. 331 */ 332 pg = uvm_pagealloc(NULL, 0, anon, 0); 333 if (pg == NULL) { 334 /* Out of memory. Wait a little. */ 335 uvmfault_unlockall(ufi, amap, NULL); 336 counters_inc(uvmexp_counters, flt_noram); 337 uvm_wait("flt_noram1"); 338 } else { 339 /* PG_BUSY bit is set. */ 340 we_own = TRUE; 341 uvmfault_unlockall(ufi, amap, NULL); 342 343 /* 344 * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN 345 * page into the uvm_swap_get function with 346 * all data structures unlocked. note that 347 * it is ok to read an_swslot here because 348 * we hold PG_BUSY on the page. 349 */ 350 counters_inc(uvmexp_counters, pageins); 351 error = uvm_swap_get(pg, anon->an_swslot, 352 PGO_SYNCIO); 353 354 /* 355 * We clean up after the I/O below in the 356 * 'we_own' case. 357 */ 358 } 359 } 360 361 /* 362 * Re-lock the map and anon. 363 */ 364 locked = uvmfault_relock(ufi); 365 if (locked || we_own) { 366 rw_enter(anon->an_lock, RW_WRITE); 367 } 368 369 /* 370 * If we own the page (i.e. we set PG_BUSY), then we need 371 * to clean up after the I/O. There are three cases to 372 * consider: 373 * 374 * 1) Page was released during I/O: free anon and ReFault. 375 * 2) I/O not OK. Free the page and cause the fault to fail. 376 * 3) I/O OK! Activate the page and sync with the non-we_own 377 * case (i.e. drop anon lock if not locked). 378 */ 379 if (we_own) { 380 if (pg->pg_flags & PG_WANTED) { 381 wakeup(pg); 382 } 383 /* un-busy! */ 384 atomic_clearbits_int(&pg->pg_flags, 385 PG_WANTED|PG_BUSY|PG_FAKE); 386 UVM_PAGE_OWN(pg, NULL); 387 388 /* 389 * if we were RELEASED during I/O, then our anon is 390 * no longer part of an amap. we need to free the 391 * anon and try again. 392 */ 393 if (pg->pg_flags & PG_RELEASED) { 394 pmap_page_protect(pg, PROT_NONE); 395 KASSERT(anon->an_ref == 0); 396 if (locked) 397 uvmfault_unlockall(ufi, NULL, NULL); 398 uvm_anon_release(anon); /* frees page for us */ 399 counters_inc(uvmexp_counters, flt_pgrele); 400 return (VM_PAGER_REFAULT); /* refault! */ 401 } 402 403 if (error != VM_PAGER_OK) { 404 KASSERT(error != VM_PAGER_PEND); 405 406 /* remove page from anon */ 407 anon->an_page = NULL; 408 409 /* 410 * Remove the swap slot from the anon and 411 * mark the anon as having no real slot. 412 * Do not free the swap slot, thus preventing 413 * it from being used again. 414 */ 415 uvm_swap_markbad(anon->an_swslot, 1); 416 anon->an_swslot = SWSLOT_BAD; 417 418 /* 419 * Note: page was never !PG_BUSY, so it 420 * cannot be mapped and thus no need to 421 * pmap_page_protect it... 422 */ 423 uvm_lock_pageq(); 424 uvm_pagefree(pg); 425 uvm_unlock_pageq(); 426 427 if (locked) { 428 uvmfault_unlockall(ufi, NULL, NULL); 429 } 430 rw_exit(anon->an_lock); 431 return (VM_PAGER_ERROR); 432 } 433 434 /* 435 * must be OK, clear modify (already PG_CLEAN) 436 * and activate 437 */ 438 pmap_clear_modify(pg); 439 uvm_lock_pageq(); 440 uvm_pageactivate(pg); 441 uvm_unlock_pageq(); 442 } 443 444 /* 445 * We were not able to re-lock the map - restart the fault. 446 */ 447 if (!locked) { 448 if (we_own) { 449 rw_exit(anon->an_lock); 450 } 451 return (VM_PAGER_REFAULT); 452 } 453 454 /* 455 * Verify that no one has touched the amap and moved 456 * the anon on us. 457 */ 458 if (ufi != NULL && amap_lookup(&ufi->entry->aref, 459 ufi->orig_rvaddr - ufi->entry->start) != anon) { 460 461 uvmfault_unlockall(ufi, amap, NULL); 462 return (VM_PAGER_REFAULT); 463 } 464 465 /* 466 * Retry.. 467 */ 468 counters_inc(uvmexp_counters, flt_anretry); 469 continue; 470 471 } 472 /*NOTREACHED*/ 473 } 474 475 /* 476 * Update statistics after fault resolution. 477 * - maxrss 478 */ 479 void 480 uvmfault_update_stats(struct uvm_faultinfo *ufi) 481 { 482 struct vm_map *map; 483 struct proc *p; 484 vsize_t res; 485 486 map = ufi->orig_map; 487 488 /* 489 * If this is a nested pmap (eg, a virtual machine pmap managed 490 * by vmm(4) on amd64/i386), don't do any updating, just return. 491 * 492 * pmap_nested() on other archs is #defined to 0, so this is a 493 * no-op. 494 */ 495 if (pmap_nested(map->pmap)) 496 return; 497 498 /* Update the maxrss for the process. */ 499 if (map->flags & VM_MAP_ISVMSPACE) { 500 p = curproc; 501 KASSERT(p != NULL && &p->p_vmspace->vm_map == map); 502 503 res = pmap_resident_count(map->pmap); 504 /* Convert res from pages to kilobytes. */ 505 res <<= (PAGE_SHIFT - 10); 506 507 if (p->p_ru.ru_maxrss < res) 508 p->p_ru.ru_maxrss = res; 509 } 510 } 511 512 /* 513 * F A U L T - m a i n e n t r y p o i n t 514 */ 515 516 /* 517 * uvm_fault: page fault handler 518 * 519 * => called from MD code to resolve a page fault 520 * => VM data structures usually should be unlocked. however, it is 521 * possible to call here with the main map locked if the caller 522 * gets a write lock, sets it recursive, and then calls us (c.f. 523 * uvm_map_pageable). this should be avoided because it keeps 524 * the map locked off during I/O. 525 * => MUST NEVER BE CALLED IN INTERRUPT CONTEXT 526 */ 527 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 528 ~PROT_WRITE : PROT_MASK) 529 struct uvm_faultctx { 530 /* 531 * the following members are set up by uvm_fault_check() and 532 * read-only after that. 533 */ 534 vm_prot_t enter_prot; 535 vm_prot_t access_type; 536 vaddr_t startva; 537 int npages; 538 int centeridx; 539 boolean_t narrow; 540 boolean_t wired; 541 paddr_t pa_flags; 542 }; 543 544 int uvm_fault_check( 545 struct uvm_faultinfo *, struct uvm_faultctx *, 546 struct vm_anon ***); 547 548 int uvm_fault_upper( 549 struct uvm_faultinfo *, struct uvm_faultctx *, 550 struct vm_anon **, vm_fault_t); 551 boolean_t uvm_fault_upper_lookup( 552 struct uvm_faultinfo *, const struct uvm_faultctx *, 553 struct vm_anon **, struct vm_page **); 554 555 int uvm_fault_lower( 556 struct uvm_faultinfo *, struct uvm_faultctx *, 557 struct vm_page **, vm_fault_t); 558 559 int 560 uvm_fault(vm_map_t orig_map, vaddr_t vaddr, vm_fault_t fault_type, 561 vm_prot_t access_type) 562 { 563 struct uvm_faultinfo ufi; 564 struct uvm_faultctx flt; 565 boolean_t shadowed; 566 struct vm_anon *anons_store[UVM_MAXRANGE], **anons; 567 struct vm_page *pages[UVM_MAXRANGE]; 568 int error; 569 570 counters_inc(uvmexp_counters, faults); 571 TRACEPOINT(uvm, fault, vaddr, fault_type, access_type, NULL); 572 573 /* 574 * init the IN parameters in the ufi 575 */ 576 ufi.orig_map = orig_map; 577 ufi.orig_rvaddr = trunc_page(vaddr); 578 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ 579 if (fault_type == VM_FAULT_WIRE) 580 flt.narrow = TRUE; /* don't look for neighborhood 581 * pages on wire */ 582 else 583 flt.narrow = FALSE; /* normal fault */ 584 flt.access_type = access_type; 585 586 587 error = ERESTART; 588 while (error == ERESTART) { /* ReFault: */ 589 anons = anons_store; 590 591 error = uvm_fault_check(&ufi, &flt, &anons); 592 if (error != 0) 593 continue; 594 595 /* True if there is an anon at the faulting address */ 596 shadowed = uvm_fault_upper_lookup(&ufi, &flt, anons, pages); 597 if (shadowed == TRUE) { 598 /* case 1: fault on an anon in our amap */ 599 error = uvm_fault_upper(&ufi, &flt, anons, fault_type); 600 } else { 601 /* case 2: fault on backing object or zero fill */ 602 KERNEL_LOCK(); 603 error = uvm_fault_lower(&ufi, &flt, pages, fault_type); 604 KERNEL_UNLOCK(); 605 } 606 } 607 608 return error; 609 } 610 611 /* 612 * uvm_fault_check: check prot, handle needs-copy, etc. 613 * 614 * 1. lookup entry. 615 * 2. check protection. 616 * 3. adjust fault condition (mainly for simulated fault). 617 * 4. handle needs-copy (lazy amap copy). 618 * 5. establish range of interest for neighbor fault (aka pre-fault). 619 * 6. look up anons (if amap exists). 620 * 7. flush pages (if MADV_SEQUENTIAL) 621 * 622 * => called with nothing locked. 623 * => if we fail (result != 0) we unlock everything. 624 * => initialize/adjust many members of flt. 625 */ 626 int 627 uvm_fault_check(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 628 struct vm_anon ***ranons) 629 { 630 struct vm_amap *amap; 631 struct uvm_object *uobj; 632 int nback, nforw; 633 634 /* 635 * lookup and lock the maps 636 */ 637 if (uvmfault_lookup(ufi, FALSE) == FALSE) { 638 return EFAULT; 639 } 640 /* locked: maps(read) */ 641 642 #ifdef DIAGNOSTIC 643 if ((ufi->map->flags & VM_MAP_PAGEABLE) == 0) 644 panic("uvm_fault: fault on non-pageable map (%p, 0x%lx)", 645 ufi->map, ufi->orig_rvaddr); 646 #endif 647 648 /* 649 * check protection 650 */ 651 if ((ufi->entry->protection & flt->access_type) != flt->access_type) { 652 uvmfault_unlockmaps(ufi, FALSE); 653 return EACCES; 654 } 655 656 /* 657 * "enter_prot" is the protection we want to enter the page in at. 658 * for certain pages (e.g. copy-on-write pages) this protection can 659 * be more strict than ufi->entry->protection. "wired" means either 660 * the entry is wired or we are fault-wiring the pg. 661 */ 662 663 flt->enter_prot = ufi->entry->protection; 664 flt->pa_flags = UVM_ET_ISWC(ufi->entry) ? PMAP_WC : 0; 665 flt->wired = VM_MAPENT_ISWIRED(ufi->entry) || (flt->narrow == TRUE); 666 if (flt->wired) 667 flt->access_type = flt->enter_prot; /* full access for wired */ 668 669 /* handle "needs_copy" case. */ 670 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 671 if ((flt->access_type & PROT_WRITE) || 672 (ufi->entry->object.uvm_obj == NULL)) { 673 /* need to clear */ 674 uvmfault_unlockmaps(ufi, FALSE); 675 uvmfault_amapcopy(ufi); 676 counters_inc(uvmexp_counters, flt_amcopy); 677 return ERESTART; 678 } else { 679 /* 680 * ensure that we pmap_enter page R/O since 681 * needs_copy is still true 682 */ 683 flt->enter_prot &= ~PROT_WRITE; 684 } 685 } 686 687 /* 688 * identify the players 689 */ 690 amap = ufi->entry->aref.ar_amap; /* upper layer */ 691 uobj = ufi->entry->object.uvm_obj; /* lower layer */ 692 693 /* 694 * check for a case 0 fault. if nothing backing the entry then 695 * error now. 696 */ 697 if (amap == NULL && uobj == NULL) { 698 uvmfault_unlockmaps(ufi, FALSE); 699 return EFAULT; 700 } 701 702 /* 703 * establish range of interest based on advice from mapper 704 * and then clip to fit map entry. note that we only want 705 * to do this the first time through the fault. if we 706 * ReFault we will disable this by setting "narrow" to true. 707 */ 708 if (flt->narrow == FALSE) { 709 710 /* wide fault (!narrow) */ 711 nback = min(uvmadvice[ufi->entry->advice].nback, 712 (ufi->orig_rvaddr - ufi->entry->start) >> PAGE_SHIFT); 713 flt->startva = ufi->orig_rvaddr - ((vsize_t)nback << PAGE_SHIFT); 714 nforw = min(uvmadvice[ufi->entry->advice].nforw, 715 ((ufi->entry->end - ufi->orig_rvaddr) >> PAGE_SHIFT) - 1); 716 /* 717 * note: "-1" because we don't want to count the 718 * faulting page as forw 719 */ 720 flt->npages = nback + nforw + 1; 721 flt->centeridx = nback; 722 723 flt->narrow = TRUE; /* ensure only once per-fault */ 724 } else { 725 /* narrow fault! */ 726 nback = nforw = 0; 727 flt->startva = ufi->orig_rvaddr; 728 flt->npages = 1; 729 flt->centeridx = 0; 730 } 731 732 /* 733 * if we've got an amap then lock it and extract current anons. 734 */ 735 if (amap) { 736 amap_lock(amap); 737 amap_lookups(&ufi->entry->aref, 738 flt->startva - ufi->entry->start, *ranons, flt->npages); 739 } else { 740 *ranons = NULL; /* to be safe */ 741 } 742 743 /* 744 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages 745 * now and then forget about them (for the rest of the fault). 746 */ 747 if (ufi->entry->advice == MADV_SEQUENTIAL && nback != 0) { 748 /* flush back-page anons? */ 749 if (amap) 750 uvmfault_anonflush(*ranons, nback); 751 752 /* flush object? */ 753 if (uobj) { 754 voff_t uoff; 755 756 uoff = (flt->startva - ufi->entry->start) + ufi->entry->offset; 757 KERNEL_LOCK(); 758 (void) uobj->pgops->pgo_flush(uobj, uoff, uoff + 759 ((vsize_t)nback << PAGE_SHIFT), PGO_DEACTIVATE); 760 KERNEL_UNLOCK(); 761 } 762 763 /* now forget about the backpages */ 764 if (amap) 765 *ranons += nback; 766 flt->startva += ((vsize_t)nback << PAGE_SHIFT); 767 flt->npages -= nback; 768 flt->centeridx = 0; 769 } 770 771 return 0; 772 } 773 774 /* 775 * uvm_fault_upper_lookup: look up existing h/w mapping and amap. 776 * 777 * iterate range of interest: 778 * 1. check if h/w mapping exists. if yes, we don't care 779 * 2. check if anon exists. if not, page is lower. 780 * 3. if anon exists, enter h/w mapping for neighbors. 781 * 782 * => called with amap locked (if exists). 783 */ 784 boolean_t 785 uvm_fault_upper_lookup(struct uvm_faultinfo *ufi, 786 const struct uvm_faultctx *flt, struct vm_anon **anons, 787 struct vm_page **pages) 788 { 789 struct vm_amap *amap = ufi->entry->aref.ar_amap; 790 struct vm_anon *anon; 791 boolean_t shadowed; 792 vaddr_t currva; 793 paddr_t pa; 794 int lcv; 795 796 /* locked: maps(read), amap(if there) */ 797 KASSERT(amap == NULL || 798 rw_write_held(amap->am_lock)); 799 800 /* 801 * map in the backpages and frontpages we found in the amap in hopes 802 * of preventing future faults. we also init the pages[] array as 803 * we go. 804 */ 805 currva = flt->startva; 806 shadowed = FALSE; 807 for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) { 808 /* 809 * dont play with VAs that are already mapped 810 * except for center) 811 */ 812 if (lcv != flt->centeridx && 813 pmap_extract(ufi->orig_map->pmap, currva, &pa)) { 814 pages[lcv] = PGO_DONTCARE; 815 continue; 816 } 817 818 /* 819 * unmapped or center page. check if any anon at this level. 820 */ 821 if (amap == NULL || anons[lcv] == NULL) { 822 pages[lcv] = NULL; 823 continue; 824 } 825 826 /* 827 * check for present page and map if possible. 828 */ 829 pages[lcv] = PGO_DONTCARE; 830 if (lcv == flt->centeridx) { /* save center for later! */ 831 shadowed = TRUE; 832 continue; 833 } 834 anon = anons[lcv]; 835 KASSERT(anon->an_lock == amap->am_lock); 836 if (anon->an_page && 837 (anon->an_page->pg_flags & (PG_RELEASED|PG_BUSY)) == 0) { 838 uvm_lock_pageq(); 839 uvm_pageactivate(anon->an_page); /* reactivate */ 840 uvm_unlock_pageq(); 841 counters_inc(uvmexp_counters, flt_namap); 842 843 /* 844 * Since this isn't the page that's actually faulting, 845 * ignore pmap_enter() failures; it's not critical 846 * that we enter these right now. 847 */ 848 (void) pmap_enter(ufi->orig_map->pmap, currva, 849 VM_PAGE_TO_PHYS(anon->an_page) | flt->pa_flags, 850 (anon->an_ref > 1) ? 851 (flt->enter_prot & ~PROT_WRITE) : flt->enter_prot, 852 PMAP_CANFAIL | 853 (VM_MAPENT_ISWIRED(ufi->entry) ? PMAP_WIRED : 0)); 854 } 855 } 856 if (flt->npages > 1) 857 pmap_update(ufi->orig_map->pmap); 858 859 return shadowed; 860 } 861 862 /* 863 * uvm_fault_upper: handle upper fault. 864 * 865 * 1. acquire anon lock. 866 * 2. get anon. let uvmfault_anonget do the dirty work. 867 * 3. if COW, promote data to new anon 868 * 4. enter h/w mapping 869 */ 870 int 871 uvm_fault_upper(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 872 struct vm_anon **anons, vm_fault_t fault_type) 873 { 874 struct vm_amap *amap = ufi->entry->aref.ar_amap; 875 struct vm_anon *oanon, *anon = anons[flt->centeridx]; 876 struct vm_page *pg = NULL; 877 int error, ret; 878 879 KASSERT(rw_write_held(amap->am_lock)); 880 KASSERT(anon->an_lock == amap->am_lock); 881 882 /* 883 * no matter if we have case 1A or case 1B we are going to need to 884 * have the anon's memory resident. ensure that now. 885 */ 886 /* 887 * let uvmfault_anonget do the dirty work. 888 * if it fails (!OK) it will unlock everything for us. 889 * if it succeeds, locks are still valid and locked. 890 * also, if it is OK, then the anon's page is on the queues. 891 * if the page is on loan from a uvm_object, then anonget will 892 * lock that object for us if it does not fail. 893 */ 894 error = uvmfault_anonget(ufi, amap, anon); 895 switch (error) { 896 case VM_PAGER_OK: 897 break; 898 899 case VM_PAGER_REFAULT: 900 return ERESTART; 901 902 case VM_PAGER_ERROR: 903 /* 904 * An error occured while trying to bring in the 905 * page -- this is the only error we return right 906 * now. 907 */ 908 return EACCES; /* XXX */ 909 default: 910 #ifdef DIAGNOSTIC 911 panic("uvm_fault: uvmfault_anonget -> %d", error); 912 #else 913 return EACCES; 914 #endif 915 } 916 917 KASSERT(rw_write_held(amap->am_lock)); 918 KASSERT(anon->an_lock == amap->am_lock); 919 920 /* 921 * if we are case 1B then we will need to allocate a new blank 922 * anon to transfer the data into. note that we have a lock 923 * on anon, so no one can busy or release the page until we are done. 924 * also note that the ref count can't drop to zero here because 925 * it is > 1 and we are only dropping one ref. 926 * 927 * in the (hopefully very rare) case that we are out of RAM we 928 * will unlock, wait for more RAM, and refault. 929 * 930 * if we are out of anon VM we wait for RAM to become available. 931 */ 932 933 if ((flt->access_type & PROT_WRITE) != 0 && anon->an_ref > 1) { 934 counters_inc(uvmexp_counters, flt_acow); 935 oanon = anon; /* oanon = old */ 936 anon = uvm_analloc(); 937 if (anon) { 938 anon->an_lock = amap->am_lock; 939 pg = uvm_pagealloc(NULL, 0, anon, 0); 940 } 941 942 /* check for out of RAM */ 943 if (anon == NULL || pg == NULL) { 944 uvmfault_unlockall(ufi, amap, NULL); 945 if (anon == NULL) 946 counters_inc(uvmexp_counters, flt_noanon); 947 else { 948 anon->an_lock = NULL; 949 anon->an_ref--; 950 uvm_anfree(anon); 951 counters_inc(uvmexp_counters, flt_noram); 952 } 953 954 if (uvm_swapisfull()) 955 return ENOMEM; 956 957 /* out of RAM, wait for more */ 958 if (anon == NULL) 959 uvm_anwait(); 960 else 961 uvm_wait("flt_noram3"); 962 return ERESTART; 963 } 964 965 /* got all resources, replace anon with nanon */ 966 uvm_pagecopy(oanon->an_page, pg); /* pg now !PG_CLEAN */ 967 /* un-busy! new page */ 968 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE); 969 UVM_PAGE_OWN(pg, NULL); 970 ret = amap_add(&ufi->entry->aref, 971 ufi->orig_rvaddr - ufi->entry->start, anon, 1); 972 KASSERT(ret == 0); 973 974 /* deref: can not drop to zero here by defn! */ 975 oanon->an_ref--; 976 977 /* 978 * note: anon is _not_ locked, but we have the sole references 979 * to in from amap. 980 * thus, no one can get at it until we are done with it. 981 */ 982 } else { 983 counters_inc(uvmexp_counters, flt_anon); 984 oanon = anon; 985 pg = anon->an_page; 986 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */ 987 flt->enter_prot = flt->enter_prot & ~PROT_WRITE; 988 } 989 990 /* 991 * now map the page in . 992 */ 993 if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr, 994 VM_PAGE_TO_PHYS(pg) | flt->pa_flags, flt->enter_prot, 995 flt->access_type | PMAP_CANFAIL | (flt->wired ? PMAP_WIRED : 0)) != 0) { 996 /* 997 * No need to undo what we did; we can simply think of 998 * this as the pmap throwing away the mapping information. 999 * 1000 * We do, however, have to go through the ReFault path, 1001 * as the map may change while we're asleep. 1002 */ 1003 uvmfault_unlockall(ufi, amap, NULL); 1004 if (uvm_swapisfull()) { 1005 /* XXX instrumentation */ 1006 return ENOMEM; 1007 } 1008 /* XXX instrumentation */ 1009 uvm_wait("flt_pmfail1"); 1010 return ERESTART; 1011 } 1012 1013 /* ... update the page queues. */ 1014 uvm_lock_pageq(); 1015 1016 if (fault_type == VM_FAULT_WIRE) { 1017 uvm_pagewire(pg); 1018 /* 1019 * since the now-wired page cannot be paged out, 1020 * release its swap resources for others to use. 1021 * since an anon with no swap cannot be PG_CLEAN, 1022 * clear its clean flag now. 1023 */ 1024 atomic_clearbits_int(&pg->pg_flags, PG_CLEAN); 1025 uvm_anon_dropswap(anon); 1026 } else { 1027 /* activate it */ 1028 uvm_pageactivate(pg); 1029 } 1030 1031 uvm_unlock_pageq(); 1032 1033 /* 1034 * done case 1! finish up by unlocking everything and returning success 1035 */ 1036 uvmfault_unlockall(ufi, amap, NULL); 1037 pmap_update(ufi->orig_map->pmap); 1038 return 0; 1039 } 1040 1041 /* 1042 * uvm_fault_lower: handle lower fault. 1043 * 1044 */ 1045 int 1046 uvm_fault_lower(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1047 struct vm_page **pages, vm_fault_t fault_type) 1048 { 1049 struct vm_amap *amap = ufi->entry->aref.ar_amap; 1050 struct uvm_object *uobj = ufi->entry->object.uvm_obj; 1051 boolean_t promote, locked; 1052 int result, lcv, gotpages; 1053 struct vm_page *uobjpage, *pg = NULL; 1054 struct vm_anon *anon = NULL; 1055 vaddr_t currva; 1056 voff_t uoff; 1057 1058 /* 1059 * if the desired page is not shadowed by the amap and we have a 1060 * backing object, then we check to see if the backing object would 1061 * prefer to handle the fault itself (rather than letting us do it 1062 * with the usual pgo_get hook). the backing object signals this by 1063 * providing a pgo_fault routine. 1064 */ 1065 if (uobj != NULL && uobj->pgops->pgo_fault != NULL) { 1066 result = uobj->pgops->pgo_fault(ufi, flt->startva, pages, 1067 flt->npages, flt->centeridx, fault_type, flt->access_type, 1068 PGO_LOCKED); 1069 1070 if (result == VM_PAGER_OK) 1071 return (0); /* pgo_fault did pmap enter */ 1072 else if (result == VM_PAGER_REFAULT) 1073 return ERESTART; /* try again! */ 1074 else 1075 return (EACCES); 1076 } 1077 1078 /* 1079 * now, if the desired page is not shadowed by the amap and we have 1080 * a backing object that does not have a special fault routine, then 1081 * we ask (with pgo_get) the object for resident pages that we care 1082 * about and attempt to map them in. we do not let pgo_get block 1083 * (PGO_LOCKED). 1084 * 1085 * ("get" has the option of doing a pmap_enter for us) 1086 */ 1087 if (uobj != NULL) { 1088 counters_inc(uvmexp_counters, flt_lget); 1089 gotpages = flt->npages; 1090 (void) uobj->pgops->pgo_get(uobj, ufi->entry->offset + 1091 (flt->startva - ufi->entry->start), 1092 pages, &gotpages, flt->centeridx, 1093 flt->access_type & MASK(ufi->entry), 1094 ufi->entry->advice, PGO_LOCKED); 1095 1096 /* check for pages to map, if we got any */ 1097 uobjpage = NULL; 1098 if (gotpages) { 1099 currva = flt->startva; 1100 for (lcv = 0 ; lcv < flt->npages ; 1101 lcv++, currva += PAGE_SIZE) { 1102 if (pages[lcv] == NULL || 1103 pages[lcv] == PGO_DONTCARE) 1104 continue; 1105 1106 KASSERT((pages[lcv]->pg_flags & PG_RELEASED) == 0); 1107 1108 /* 1109 * if center page is resident and not 1110 * PG_BUSY, then pgo_get made it PG_BUSY 1111 * for us and gave us a handle to it. 1112 * remember this page as "uobjpage." 1113 * (for later use). 1114 */ 1115 if (lcv == flt->centeridx) { 1116 uobjpage = pages[lcv]; 1117 continue; 1118 } 1119 1120 /* 1121 * note: calling pgo_get with locked data 1122 * structures returns us pages which are 1123 * neither busy nor released, so we don't 1124 * need to check for this. we can just 1125 * directly enter the page (after moving it 1126 * to the head of the active queue [useful?]). 1127 */ 1128 1129 uvm_lock_pageq(); 1130 uvm_pageactivate(pages[lcv]); /* reactivate */ 1131 uvm_unlock_pageq(); 1132 counters_inc(uvmexp_counters, flt_nomap); 1133 1134 /* 1135 * Since this page isn't the page that's 1136 * actually faulting, ignore pmap_enter() 1137 * failures; it's not critical that we 1138 * enter these right now. 1139 */ 1140 (void) pmap_enter(ufi->orig_map->pmap, currva, 1141 VM_PAGE_TO_PHYS(pages[lcv]) | flt->pa_flags, 1142 flt->enter_prot & MASK(ufi->entry), 1143 PMAP_CANFAIL | 1144 (flt->wired ? PMAP_WIRED : 0)); 1145 1146 /* 1147 * NOTE: page can't be PG_WANTED because 1148 * we've held the lock the whole time 1149 * we've had the handle. 1150 */ 1151 atomic_clearbits_int(&pages[lcv]->pg_flags, 1152 PG_BUSY); 1153 UVM_PAGE_OWN(pages[lcv], NULL); 1154 } /* for "lcv" loop */ 1155 pmap_update(ufi->orig_map->pmap); 1156 } /* "gotpages" != 0 */ 1157 /* note: object still _locked_ */ 1158 } else { 1159 uobjpage = NULL; 1160 } 1161 1162 /* 1163 * note that at this point we are done with any front or back pages. 1164 * we are now going to focus on the center page (i.e. the one we've 1165 * faulted on). if we have faulted on the bottom (uobj) 1166 * layer [i.e. case 2] and the page was both present and available, 1167 * then we've got a pointer to it as "uobjpage" and we've already 1168 * made it BUSY. 1169 */ 1170 1171 /* 1172 * note that uobjpage can not be PGO_DONTCARE at this point. we now 1173 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we 1174 * have a backing object, check and see if we are going to promote 1175 * the data up to an anon during the fault. 1176 */ 1177 if (uobj == NULL) { 1178 uobjpage = PGO_DONTCARE; 1179 promote = TRUE; /* always need anon here */ 1180 } else { 1181 KASSERT(uobjpage != PGO_DONTCARE); 1182 promote = (flt->access_type & PROT_WRITE) && 1183 UVM_ET_ISCOPYONWRITE(ufi->entry); 1184 } 1185 1186 /* 1187 * if uobjpage is not null then we do not need to do I/O to get the 1188 * uobjpage. 1189 * 1190 * if uobjpage is null, then we need to ask the pager to 1191 * get the data for us. once we have the data, we need to reverify 1192 * the state the world. we are currently not holding any resources. 1193 */ 1194 if (uobjpage) { 1195 /* update rusage counters */ 1196 curproc->p_ru.ru_minflt++; 1197 } else { 1198 /* update rusage counters */ 1199 curproc->p_ru.ru_majflt++; 1200 1201 uvmfault_unlockall(ufi, amap, NULL); 1202 1203 counters_inc(uvmexp_counters, flt_get); 1204 gotpages = 1; 1205 uoff = (ufi->orig_rvaddr - ufi->entry->start) + ufi->entry->offset; 1206 result = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages, 1207 0, flt->access_type & MASK(ufi->entry), ufi->entry->advice, 1208 PGO_SYNCIO); 1209 1210 /* recover from I/O */ 1211 if (result != VM_PAGER_OK) { 1212 KASSERT(result != VM_PAGER_PEND); 1213 1214 if (result == VM_PAGER_AGAIN) { 1215 tsleep_nsec(&nowake, PVM, "fltagain2", 1216 SEC_TO_NSEC(1)); 1217 return ERESTART; 1218 } 1219 1220 if (!UVM_ET_ISNOFAULT(ufi->entry)) 1221 return (EIO); 1222 1223 uobjpage = PGO_DONTCARE; 1224 promote = TRUE; 1225 } 1226 1227 /* re-verify the state of the world. */ 1228 locked = uvmfault_relock(ufi); 1229 if (locked && amap != NULL) 1230 amap_lock(amap); 1231 1232 /* 1233 * Re-verify that amap slot is still free. if there is 1234 * a problem, we clean up. 1235 */ 1236 if (locked && amap && amap_lookup(&ufi->entry->aref, 1237 ufi->orig_rvaddr - ufi->entry->start)) { 1238 if (locked) 1239 uvmfault_unlockall(ufi, amap, NULL); 1240 locked = FALSE; 1241 } 1242 1243 /* didn't get the lock? release the page and retry. */ 1244 if (locked == FALSE && uobjpage != PGO_DONTCARE) { 1245 uvm_lock_pageq(); 1246 /* make sure it is in queues */ 1247 uvm_pageactivate(uobjpage); 1248 uvm_unlock_pageq(); 1249 1250 if (uobjpage->pg_flags & PG_WANTED) 1251 /* still holding object lock */ 1252 wakeup(uobjpage); 1253 atomic_clearbits_int(&uobjpage->pg_flags, 1254 PG_BUSY|PG_WANTED); 1255 UVM_PAGE_OWN(uobjpage, NULL); 1256 return ERESTART; 1257 } 1258 if (locked == FALSE) 1259 return ERESTART; 1260 1261 /* 1262 * we have the data in uobjpage which is PG_BUSY 1263 */ 1264 } 1265 1266 /* 1267 * notes: 1268 * - at this point uobjpage can not be NULL 1269 * - at this point uobjpage could be PG_WANTED (handle later) 1270 */ 1271 if (promote == FALSE) { 1272 /* 1273 * we are not promoting. if the mapping is COW ensure that we 1274 * don't give more access than we should (e.g. when doing a read 1275 * fault on a COPYONWRITE mapping we want to map the COW page in 1276 * R/O even though the entry protection could be R/W). 1277 * 1278 * set "pg" to the page we want to map in (uobjpage, usually) 1279 */ 1280 counters_inc(uvmexp_counters, flt_obj); 1281 if (UVM_ET_ISCOPYONWRITE(ufi->entry)) 1282 flt->enter_prot &= ~PROT_WRITE; 1283 pg = uobjpage; /* map in the actual object */ 1284 1285 /* assert(uobjpage != PGO_DONTCARE) */ 1286 1287 /* 1288 * we are faulting directly on the page. 1289 */ 1290 } else { 1291 /* 1292 * if we are going to promote the data to an anon we 1293 * allocate a blank anon here and plug it into our amap. 1294 */ 1295 #ifdef DIAGNOSTIC 1296 if (amap == NULL) 1297 panic("uvm_fault: want to promote data, but no anon"); 1298 #endif 1299 1300 anon = uvm_analloc(); 1301 if (anon) { 1302 /* 1303 * In `Fill in data...' below, if 1304 * uobjpage == PGO_DONTCARE, we want 1305 * a zero'd, dirty page, so have 1306 * uvm_pagealloc() do that for us. 1307 */ 1308 anon->an_lock = amap->am_lock; 1309 pg = uvm_pagealloc(NULL, 0, anon, 1310 (uobjpage == PGO_DONTCARE) ? UVM_PGA_ZERO : 0); 1311 } 1312 1313 /* 1314 * out of memory resources? 1315 */ 1316 if (anon == NULL || pg == NULL) { 1317 /* arg! must unbusy our page and fail or sleep. */ 1318 if (uobjpage != PGO_DONTCARE) { 1319 uvm_lock_pageq(); 1320 uvm_pageactivate(uobjpage); 1321 uvm_unlock_pageq(); 1322 1323 if (uobjpage->pg_flags & PG_WANTED) 1324 wakeup(uobjpage); 1325 atomic_clearbits_int(&uobjpage->pg_flags, 1326 PG_BUSY|PG_WANTED); 1327 UVM_PAGE_OWN(uobjpage, NULL); 1328 } 1329 1330 /* unlock and fail ... */ 1331 uvmfault_unlockall(ufi, amap, uobj); 1332 if (anon == NULL) 1333 counters_inc(uvmexp_counters, flt_noanon); 1334 else { 1335 anon->an_lock = NULL; 1336 anon->an_ref--; 1337 uvm_anfree(anon); 1338 counters_inc(uvmexp_counters, flt_noram); 1339 } 1340 1341 if (uvm_swapisfull()) 1342 return (ENOMEM); 1343 1344 /* out of RAM, wait for more */ 1345 if (anon == NULL) 1346 uvm_anwait(); 1347 else 1348 uvm_wait("flt_noram5"); 1349 return ERESTART; 1350 } 1351 1352 /* fill in the data */ 1353 if (uobjpage != PGO_DONTCARE) { 1354 counters_inc(uvmexp_counters, flt_prcopy); 1355 /* copy page [pg now dirty] */ 1356 uvm_pagecopy(uobjpage, pg); 1357 1358 /* 1359 * promote to shared amap? make sure all sharing 1360 * procs see it 1361 */ 1362 if ((amap_flags(amap) & AMAP_SHARED) != 0) { 1363 pmap_page_protect(uobjpage, PROT_NONE); 1364 } 1365 1366 /* dispose of uobjpage. drop handle to uobj as well. */ 1367 if (uobjpage->pg_flags & PG_WANTED) 1368 wakeup(uobjpage); 1369 atomic_clearbits_int(&uobjpage->pg_flags, 1370 PG_BUSY|PG_WANTED); 1371 UVM_PAGE_OWN(uobjpage, NULL); 1372 uvm_lock_pageq(); 1373 uvm_pageactivate(uobjpage); 1374 uvm_unlock_pageq(); 1375 uobj = NULL; 1376 } else { 1377 counters_inc(uvmexp_counters, flt_przero); 1378 /* 1379 * Page is zero'd and marked dirty by uvm_pagealloc() 1380 * above. 1381 */ 1382 } 1383 1384 if (amap_add(&ufi->entry->aref, 1385 ufi->orig_rvaddr - ufi->entry->start, anon, 0)) { 1386 uvmfault_unlockall(ufi, amap, NULL); 1387 uvm_anfree(anon); 1388 counters_inc(uvmexp_counters, flt_noamap); 1389 1390 if (uvm_swapisfull()) 1391 return (ENOMEM); 1392 1393 amap_populate(&ufi->entry->aref, 1394 ufi->orig_rvaddr - ufi->entry->start); 1395 return ERESTART; 1396 } 1397 } 1398 1399 /* note: pg is either the uobjpage or the new page in the new anon */ 1400 /* 1401 * all resources are present. we can now map it in and free our 1402 * resources. 1403 */ 1404 if (amap == NULL) 1405 KASSERT(anon == NULL); 1406 else { 1407 KASSERT(rw_write_held(amap->am_lock)); 1408 KASSERT(anon == NULL || anon->an_lock == amap->am_lock); 1409 } 1410 if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr, 1411 VM_PAGE_TO_PHYS(pg) | flt->pa_flags, flt->enter_prot, 1412 flt->access_type | PMAP_CANFAIL | (flt->wired ? PMAP_WIRED : 0)) != 0) { 1413 /* 1414 * No need to undo what we did; we can simply think of 1415 * this as the pmap throwing away the mapping information. 1416 * 1417 * We do, however, have to go through the ReFault path, 1418 * as the map may change while we're asleep. 1419 */ 1420 if (pg->pg_flags & PG_WANTED) 1421 wakeup(pg); 1422 1423 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED); 1424 UVM_PAGE_OWN(pg, NULL); 1425 uvmfault_unlockall(ufi, amap, uobj); 1426 if (uvm_swapisfull()) { 1427 /* XXX instrumentation */ 1428 return (ENOMEM); 1429 } 1430 /* XXX instrumentation */ 1431 uvm_wait("flt_pmfail2"); 1432 return ERESTART; 1433 } 1434 1435 uvm_lock_pageq(); 1436 1437 if (fault_type == VM_FAULT_WIRE) { 1438 uvm_pagewire(pg); 1439 if (pg->pg_flags & PQ_AOBJ) { 1440 /* 1441 * since the now-wired page cannot be paged out, 1442 * release its swap resources for others to use. 1443 * since an aobj page with no swap cannot be PG_CLEAN, 1444 * clear its clean flag now. 1445 */ 1446 atomic_clearbits_int(&pg->pg_flags, PG_CLEAN); 1447 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT); 1448 } 1449 } else { 1450 /* activate it */ 1451 uvm_pageactivate(pg); 1452 } 1453 uvm_unlock_pageq(); 1454 1455 if (pg->pg_flags & PG_WANTED) 1456 wakeup(pg); 1457 1458 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED); 1459 UVM_PAGE_OWN(pg, NULL); 1460 uvmfault_unlockall(ufi, amap, uobj); 1461 pmap_update(ufi->orig_map->pmap); 1462 1463 return (0); 1464 } 1465 1466 1467 /* 1468 * uvm_fault_wire: wire down a range of virtual addresses in a map. 1469 * 1470 * => map may be read-locked by caller, but MUST NOT be write-locked. 1471 * => if map is read-locked, any operations which may cause map to 1472 * be write-locked in uvm_fault() must be taken care of by 1473 * the caller. See uvm_map_pageable(). 1474 */ 1475 int 1476 uvm_fault_wire(vm_map_t map, vaddr_t start, vaddr_t end, vm_prot_t access_type) 1477 { 1478 vaddr_t va; 1479 int rv; 1480 1481 /* 1482 * now fault it in a page at a time. if the fault fails then we have 1483 * to undo what we have done. note that in uvm_fault PROT_NONE 1484 * is replaced with the max protection if fault_type is VM_FAULT_WIRE. 1485 */ 1486 for (va = start ; va < end ; va += PAGE_SIZE) { 1487 rv = uvm_fault(map, va, VM_FAULT_WIRE, access_type); 1488 if (rv) { 1489 if (va != start) { 1490 uvm_fault_unwire(map, start, va); 1491 } 1492 return (rv); 1493 } 1494 } 1495 1496 return (0); 1497 } 1498 1499 /* 1500 * uvm_fault_unwire(): unwire range of virtual space. 1501 */ 1502 void 1503 uvm_fault_unwire(vm_map_t map, vaddr_t start, vaddr_t end) 1504 { 1505 1506 vm_map_lock_read(map); 1507 uvm_fault_unwire_locked(map, start, end); 1508 vm_map_unlock_read(map); 1509 } 1510 1511 /* 1512 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire(). 1513 * 1514 * => map must be at least read-locked. 1515 */ 1516 void 1517 uvm_fault_unwire_locked(vm_map_t map, vaddr_t start, vaddr_t end) 1518 { 1519 vm_map_entry_t entry, next; 1520 pmap_t pmap = vm_map_pmap(map); 1521 vaddr_t va; 1522 paddr_t pa; 1523 struct vm_page *pg; 1524 1525 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0); 1526 1527 /* 1528 * we assume that the area we are unwiring has actually been wired 1529 * in the first place. this means that we should be able to extract 1530 * the PAs from the pmap. we also lock out the page daemon so that 1531 * we can call uvm_pageunwire. 1532 */ 1533 uvm_lock_pageq(); 1534 1535 /* find the beginning map entry for the region. */ 1536 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map)); 1537 if (uvm_map_lookup_entry(map, start, &entry) == FALSE) 1538 panic("uvm_fault_unwire_locked: address not in map"); 1539 1540 for (va = start; va < end ; va += PAGE_SIZE) { 1541 if (pmap_extract(pmap, va, &pa) == FALSE) 1542 continue; 1543 1544 /* find the map entry for the current address. */ 1545 KASSERT(va >= entry->start); 1546 while (va >= entry->end) { 1547 next = RBT_NEXT(uvm_map_addr, entry); 1548 KASSERT(next != NULL && next->start <= entry->end); 1549 entry = next; 1550 } 1551 1552 /* if the entry is no longer wired, tell the pmap. */ 1553 if (VM_MAPENT_ISWIRED(entry) == 0) 1554 pmap_unwire(pmap, va); 1555 1556 pg = PHYS_TO_VM_PAGE(pa); 1557 if (pg) 1558 uvm_pageunwire(pg); 1559 } 1560 1561 uvm_unlock_pageq(); 1562 } 1563 1564 /* 1565 * uvmfault_unlockmaps: unlock the maps 1566 */ 1567 void 1568 uvmfault_unlockmaps(struct uvm_faultinfo *ufi, boolean_t write_locked) 1569 { 1570 /* 1571 * ufi can be NULL when this isn't really a fault, 1572 * but merely paging in anon data. 1573 */ 1574 if (ufi == NULL) { 1575 return; 1576 } 1577 1578 uvmfault_update_stats(ufi); 1579 if (write_locked) { 1580 vm_map_unlock(ufi->map); 1581 } else { 1582 vm_map_unlock_read(ufi->map); 1583 } 1584 } 1585 1586 /* 1587 * uvmfault_unlockall: unlock everything passed in. 1588 * 1589 * => maps must be read-locked (not write-locked). 1590 */ 1591 void 1592 uvmfault_unlockall(struct uvm_faultinfo *ufi, struct vm_amap *amap, 1593 struct uvm_object *uobj) 1594 { 1595 if (amap != NULL) 1596 amap_unlock(amap); 1597 uvmfault_unlockmaps(ufi, FALSE); 1598 } 1599 1600 /* 1601 * uvmfault_lookup: lookup a virtual address in a map 1602 * 1603 * => caller must provide a uvm_faultinfo structure with the IN 1604 * params properly filled in 1605 * => we will lookup the map entry (handling submaps) as we go 1606 * => if the lookup is a success we will return with the maps locked 1607 * => if "write_lock" is TRUE, we write_lock the map, otherwise we only 1608 * get a read lock. 1609 * => note that submaps can only appear in the kernel and they are 1610 * required to use the same virtual addresses as the map they 1611 * are referenced by (thus address translation between the main 1612 * map and the submap is unnecessary). 1613 */ 1614 1615 boolean_t 1616 uvmfault_lookup(struct uvm_faultinfo *ufi, boolean_t write_lock) 1617 { 1618 vm_map_t tmpmap; 1619 1620 /* init ufi values for lookup. */ 1621 ufi->map = ufi->orig_map; 1622 ufi->size = ufi->orig_size; 1623 1624 /* 1625 * keep going down levels until we are done. note that there can 1626 * only be two levels so we won't loop very long. 1627 */ 1628 while (1) { 1629 if (ufi->orig_rvaddr < ufi->map->min_offset || 1630 ufi->orig_rvaddr >= ufi->map->max_offset) 1631 return(FALSE); 1632 1633 /* lock map */ 1634 if (write_lock) { 1635 vm_map_lock(ufi->map); 1636 } else { 1637 vm_map_lock_read(ufi->map); 1638 } 1639 1640 /* lookup */ 1641 if (!uvm_map_lookup_entry(ufi->map, ufi->orig_rvaddr, 1642 &ufi->entry)) { 1643 uvmfault_unlockmaps(ufi, write_lock); 1644 return(FALSE); 1645 } 1646 1647 /* reduce size if necessary */ 1648 if (ufi->entry->end - ufi->orig_rvaddr < ufi->size) 1649 ufi->size = ufi->entry->end - ufi->orig_rvaddr; 1650 1651 /* 1652 * submap? replace map with the submap and lookup again. 1653 * note: VAs in submaps must match VAs in main map. 1654 */ 1655 if (UVM_ET_ISSUBMAP(ufi->entry)) { 1656 tmpmap = ufi->entry->object.sub_map; 1657 uvmfault_unlockmaps(ufi, write_lock); 1658 ufi->map = tmpmap; 1659 continue; 1660 } 1661 1662 /* got it! */ 1663 ufi->mapv = ufi->map->timestamp; 1664 return(TRUE); 1665 1666 } 1667 /*NOTREACHED*/ 1668 } 1669 1670 /* 1671 * uvmfault_relock: attempt to relock the same version of the map 1672 * 1673 * => fault data structures should be unlocked before calling. 1674 * => if a success (TRUE) maps will be locked after call. 1675 */ 1676 boolean_t 1677 uvmfault_relock(struct uvm_faultinfo *ufi) 1678 { 1679 /* 1680 * ufi can be NULL when this isn't really a fault, 1681 * but merely paging in anon data. 1682 */ 1683 if (ufi == NULL) { 1684 return TRUE; 1685 } 1686 1687 counters_inc(uvmexp_counters, flt_relck); 1688 1689 /* 1690 * relock map. fail if version mismatch (in which case nothing 1691 * gets locked). 1692 */ 1693 vm_map_lock_read(ufi->map); 1694 if (ufi->mapv != ufi->map->timestamp) { 1695 vm_map_unlock_read(ufi->map); 1696 return(FALSE); 1697 } 1698 1699 counters_inc(uvmexp_counters, flt_relckok); 1700 return(TRUE); /* got it! */ 1701 } 1702