1 /* $OpenBSD: uvm_fault.c,v 1.120 2021/03/26 13:40:05 mpi Exp $ */ 2 /* $NetBSD: uvm_fault.c,v 1.51 2000/08/06 00:22:53 thorpej Exp $ */ 3 4 /* 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp 29 */ 30 31 /* 32 * uvm_fault.c: fault handler 33 */ 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/percpu.h> 39 #include <sys/proc.h> 40 #include <sys/malloc.h> 41 #include <sys/mman.h> 42 #include <sys/tracepoint.h> 43 44 #include <uvm/uvm.h> 45 46 /* 47 * 48 * a word on page faults: 49 * 50 * types of page faults we handle: 51 * 52 * CASE 1: upper layer faults CASE 2: lower layer faults 53 * 54 * CASE 1A CASE 1B CASE 2A CASE 2B 55 * read/write1 write>1 read/write +-cow_write/zero 56 * | | | | 57 * +--|--+ +--|--+ +-----+ + | + | +-----+ 58 * amap | V | | ---------> new | | | | ^ | 59 * +-----+ +-----+ +-----+ + | + | +--|--+ 60 * | | | 61 * +-----+ +-----+ +--|--+ | +--|--+ 62 * uobj | d/c | | d/c | | V | +----+ | 63 * +-----+ +-----+ +-----+ +-----+ 64 * 65 * d/c = don't care 66 * 67 * case [0]: layerless fault 68 * no amap or uobj is present. this is an error. 69 * 70 * case [1]: upper layer fault [anon active] 71 * 1A: [read] or [write with anon->an_ref == 1] 72 * I/O takes place in upper level anon and uobj is not touched. 73 * 1B: [write with anon->an_ref > 1] 74 * new anon is alloc'd and data is copied off ["COW"] 75 * 76 * case [2]: lower layer fault [uobj] 77 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area] 78 * I/O takes place directly in object. 79 * 2B: [write to copy_on_write] or [read on NULL uobj] 80 * data is "promoted" from uobj to a new anon. 81 * if uobj is null, then we zero fill. 82 * 83 * we follow the standard UVM locking protocol ordering: 84 * 85 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ) 86 * we hold a PG_BUSY page if we unlock for I/O 87 * 88 * 89 * the code is structured as follows: 90 * 91 * - init the "IN" params in the ufi structure 92 * ReFault: (ERESTART returned to the loop in uvm_fault) 93 * - do lookups [locks maps], check protection, handle needs_copy 94 * - check for case 0 fault (error) 95 * - establish "range" of fault 96 * - if we have an amap lock it and extract the anons 97 * - if sequential advice deactivate pages behind us 98 * - at the same time check pmap for unmapped areas and anon for pages 99 * that we could map in (and do map it if found) 100 * - check object for resident pages that we could map in 101 * - if (case 2) goto Case2 102 * - >>> handle case 1 103 * - ensure source anon is resident in RAM 104 * - if case 1B alloc new anon and copy from source 105 * - map the correct page in 106 * Case2: 107 * - >>> handle case 2 108 * - ensure source page is resident (if uobj) 109 * - if case 2B alloc new anon and copy from source (could be zero 110 * fill if uobj == NULL) 111 * - map the correct page in 112 * - done! 113 * 114 * note on paging: 115 * if we have to do I/O we place a PG_BUSY page in the correct object, 116 * unlock everything, and do the I/O. when I/O is done we must reverify 117 * the state of the world before assuming that our data structures are 118 * valid. [because mappings could change while the map is unlocked] 119 * 120 * alternative 1: unbusy the page in question and restart the page fault 121 * from the top (ReFault). this is easy but does not take advantage 122 * of the information that we already have from our previous lookup, 123 * although it is possible that the "hints" in the vm_map will help here. 124 * 125 * alternative 2: the system already keeps track of a "version" number of 126 * a map. [i.e. every time you write-lock a map (e.g. to change a 127 * mapping) you bump the version number up by one...] so, we can save 128 * the version number of the map before we release the lock and start I/O. 129 * then when I/O is done we can relock and check the version numbers 130 * to see if anything changed. this might save us some over 1 because 131 * we don't have to unbusy the page and may be less compares(?). 132 * 133 * alternative 3: put in backpointers or a way to "hold" part of a map 134 * in place while I/O is in progress. this could be complex to 135 * implement (especially with structures like amap that can be referenced 136 * by multiple map entries, and figuring out what should wait could be 137 * complex as well...). 138 * 139 * we use alternative 2. given that we are multi-threaded now we may want 140 * to reconsider the choice. 141 */ 142 143 /* 144 * local data structures 145 */ 146 struct uvm_advice { 147 int nback; 148 int nforw; 149 }; 150 151 /* 152 * page range array: set up in uvmfault_init(). 153 */ 154 static struct uvm_advice uvmadvice[MADV_MASK + 1]; 155 156 #define UVM_MAXRANGE 16 /* must be max() of nback+nforw+1 */ 157 158 /* 159 * private prototypes 160 */ 161 static void uvmfault_amapcopy(struct uvm_faultinfo *); 162 static inline void uvmfault_anonflush(struct vm_anon **, int); 163 void uvmfault_unlockmaps(struct uvm_faultinfo *, boolean_t); 164 void uvmfault_update_stats(struct uvm_faultinfo *); 165 166 /* 167 * inline functions 168 */ 169 /* 170 * uvmfault_anonflush: try and deactivate pages in specified anons 171 * 172 * => does not have to deactivate page if it is busy 173 */ 174 static inline void 175 uvmfault_anonflush(struct vm_anon **anons, int n) 176 { 177 int lcv; 178 struct vm_page *pg; 179 180 for (lcv = 0; lcv < n; lcv++) { 181 if (anons[lcv] == NULL) 182 continue; 183 KASSERT(rw_lock_held(anons[lcv]->an_lock)); 184 pg = anons[lcv]->an_page; 185 if (pg && (pg->pg_flags & PG_BUSY) == 0) { 186 uvm_lock_pageq(); 187 if (pg->wire_count == 0) { 188 pmap_page_protect(pg, PROT_NONE); 189 uvm_pagedeactivate(pg); 190 } 191 uvm_unlock_pageq(); 192 } 193 } 194 } 195 196 /* 197 * normal functions 198 */ 199 /* 200 * uvmfault_init: compute proper values for the uvmadvice[] array. 201 */ 202 void 203 uvmfault_init(void) 204 { 205 int npages; 206 207 npages = atop(16384); 208 if (npages > 0) { 209 KASSERT(npages <= UVM_MAXRANGE / 2); 210 uvmadvice[MADV_NORMAL].nforw = npages; 211 uvmadvice[MADV_NORMAL].nback = npages - 1; 212 } 213 214 npages = atop(32768); 215 if (npages > 0) { 216 KASSERT(npages <= UVM_MAXRANGE / 2); 217 uvmadvice[MADV_SEQUENTIAL].nforw = npages - 1; 218 uvmadvice[MADV_SEQUENTIAL].nback = npages; 219 } 220 } 221 222 /* 223 * uvmfault_amapcopy: clear "needs_copy" in a map. 224 * 225 * => called with VM data structures unlocked (usually, see below) 226 * => we get a write lock on the maps and clear needs_copy for a VA 227 * => if we are out of RAM we sleep (waiting for more) 228 */ 229 static void 230 uvmfault_amapcopy(struct uvm_faultinfo *ufi) 231 { 232 for (;;) { 233 /* 234 * no mapping? give up. 235 */ 236 if (uvmfault_lookup(ufi, TRUE) == FALSE) 237 return; 238 239 /* 240 * copy if needed. 241 */ 242 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) 243 amap_copy(ufi->map, ufi->entry, M_NOWAIT, 244 UVM_ET_ISSTACK(ufi->entry) ? FALSE : TRUE, 245 ufi->orig_rvaddr, ufi->orig_rvaddr + 1); 246 247 /* 248 * didn't work? must be out of RAM. unlock and sleep. 249 */ 250 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 251 uvmfault_unlockmaps(ufi, TRUE); 252 uvm_wait("fltamapcopy"); 253 continue; 254 } 255 256 /* 257 * got it! unlock and return. 258 */ 259 uvmfault_unlockmaps(ufi, TRUE); 260 return; 261 } 262 /*NOTREACHED*/ 263 } 264 265 /* 266 * uvmfault_anonget: get data in an anon into a non-busy, non-released 267 * page in that anon. 268 * 269 * => Map, amap and thus anon should be locked by caller. 270 * => If we fail, we unlock everything and error is returned. 271 * => If we are successful, return with everything still locked. 272 * => We do not move the page on the queues [gets moved later]. If we 273 * allocate a new page [we_own], it gets put on the queues. Either way, 274 * the result is that the page is on the queues at return time 275 */ 276 int 277 uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap, 278 struct vm_anon *anon) 279 { 280 struct vm_page *pg; 281 int error; 282 283 KASSERT(rw_lock_held(anon->an_lock)); 284 KASSERT(anon->an_lock == amap->am_lock); 285 286 /* Increment the counters.*/ 287 counters_inc(uvmexp_counters, flt_anget); 288 if (anon->an_page) { 289 curproc->p_ru.ru_minflt++; 290 } else { 291 curproc->p_ru.ru_majflt++; 292 } 293 error = 0; 294 295 /* 296 * Loop until we get the anon data, or fail. 297 */ 298 for (;;) { 299 boolean_t we_own, locked; 300 /* 301 * Note: 'we_own' will become true if we set PG_BUSY on a page. 302 */ 303 we_own = FALSE; 304 pg = anon->an_page; 305 306 /* 307 * Is page resident? Make sure it is not busy/released. 308 */ 309 if (pg) { 310 KASSERT(pg->pg_flags & PQ_ANON); 311 KASSERT(pg->uanon == anon); 312 313 /* 314 * if the page is busy, we drop all the locks and 315 * try again. 316 */ 317 if ((pg->pg_flags & (PG_BUSY|PG_RELEASED)) == 0) 318 return (VM_PAGER_OK); 319 atomic_setbits_int(&pg->pg_flags, PG_WANTED); 320 counters_inc(uvmexp_counters, flt_pgwait); 321 322 /* 323 * The last unlock must be an atomic unlock and wait 324 * on the owner of page. 325 */ 326 if (pg->uobject) { 327 /* Owner of page is UVM object. */ 328 uvmfault_unlockall(ufi, amap, NULL); 329 tsleep_nsec(pg, PVM, "anonget1", INFSLP); 330 } else { 331 /* Owner of page is anon. */ 332 uvmfault_unlockall(ufi, NULL, NULL); 333 rwsleep_nsec(pg, anon->an_lock, PVM | PNORELOCK, 334 "anonget2", INFSLP); 335 } 336 } else { 337 /* 338 * No page, therefore allocate one. 339 */ 340 pg = uvm_pagealloc(NULL, 0, anon, 0); 341 if (pg == NULL) { 342 /* Out of memory. Wait a little. */ 343 uvmfault_unlockall(ufi, amap, NULL); 344 counters_inc(uvmexp_counters, flt_noram); 345 uvm_wait("flt_noram1"); 346 } else { 347 /* PG_BUSY bit is set. */ 348 we_own = TRUE; 349 uvmfault_unlockall(ufi, amap, NULL); 350 351 /* 352 * Pass a PG_BUSY+PG_FAKE+PG_CLEAN page into 353 * the uvm_swap_get() function with all data 354 * structures unlocked. Note that it is OK 355 * to read an_swslot here, because we hold 356 * PG_BUSY on the page. 357 */ 358 counters_inc(uvmexp_counters, pageins); 359 error = uvm_swap_get(pg, anon->an_swslot, 360 PGO_SYNCIO); 361 362 /* 363 * We clean up after the I/O below in the 364 * 'we_own' case. 365 */ 366 } 367 } 368 369 /* 370 * Re-lock the map and anon. 371 */ 372 locked = uvmfault_relock(ufi); 373 if (locked || we_own) { 374 rw_enter(anon->an_lock, RW_WRITE); 375 } 376 377 /* 378 * If we own the page (i.e. we set PG_BUSY), then we need 379 * to clean up after the I/O. There are three cases to 380 * consider: 381 * 382 * 1) Page was released during I/O: free anon and ReFault. 383 * 2) I/O not OK. Free the page and cause the fault to fail. 384 * 3) I/O OK! Activate the page and sync with the non-we_own 385 * case (i.e. drop anon lock if not locked). 386 */ 387 if (we_own) { 388 if (pg->pg_flags & PG_WANTED) { 389 wakeup(pg); 390 } 391 /* un-busy! */ 392 atomic_clearbits_int(&pg->pg_flags, 393 PG_WANTED|PG_BUSY|PG_FAKE); 394 UVM_PAGE_OWN(pg, NULL); 395 396 /* 397 * if we were RELEASED during I/O, then our anon is 398 * no longer part of an amap. we need to free the 399 * anon and try again. 400 */ 401 if (pg->pg_flags & PG_RELEASED) { 402 pmap_page_protect(pg, PROT_NONE); 403 KASSERT(anon->an_ref == 0); 404 /* 405 * Released while we had unlocked amap. 406 */ 407 if (locked) 408 uvmfault_unlockall(ufi, NULL, NULL); 409 uvm_anon_release(anon); /* frees page for us */ 410 counters_inc(uvmexp_counters, flt_pgrele); 411 return (VM_PAGER_REFAULT); /* refault! */ 412 } 413 414 if (error != VM_PAGER_OK) { 415 KASSERT(error != VM_PAGER_PEND); 416 417 /* remove page from anon */ 418 anon->an_page = NULL; 419 420 /* 421 * Remove the swap slot from the anon and 422 * mark the anon as having no real slot. 423 * Do not free the swap slot, thus preventing 424 * it from being used again. 425 */ 426 uvm_swap_markbad(anon->an_swslot, 1); 427 anon->an_swslot = SWSLOT_BAD; 428 429 /* 430 * Note: page was never !PG_BUSY, so it 431 * cannot be mapped and thus no need to 432 * pmap_page_protect() it. 433 */ 434 uvm_lock_pageq(); 435 uvm_pagefree(pg); 436 uvm_unlock_pageq(); 437 438 if (locked) { 439 uvmfault_unlockall(ufi, NULL, NULL); 440 } 441 rw_exit(anon->an_lock); 442 return (VM_PAGER_ERROR); 443 } 444 445 /* 446 * We have successfully read the page, activate it. 447 */ 448 pmap_clear_modify(pg); 449 uvm_lock_pageq(); 450 uvm_pageactivate(pg); 451 uvm_unlock_pageq(); 452 } 453 454 /* 455 * We were not able to re-lock the map - restart the fault. 456 */ 457 if (!locked) { 458 if (we_own) { 459 rw_exit(anon->an_lock); 460 } 461 return (VM_PAGER_REFAULT); 462 } 463 464 /* 465 * Verify that no one has touched the amap and moved 466 * the anon on us. 467 */ 468 if (ufi != NULL && amap_lookup(&ufi->entry->aref, 469 ufi->orig_rvaddr - ufi->entry->start) != anon) { 470 471 uvmfault_unlockall(ufi, amap, NULL); 472 return (VM_PAGER_REFAULT); 473 } 474 475 /* 476 * Retry.. 477 */ 478 counters_inc(uvmexp_counters, flt_anretry); 479 continue; 480 481 } 482 /*NOTREACHED*/ 483 } 484 485 /* 486 * Update statistics after fault resolution. 487 * - maxrss 488 */ 489 void 490 uvmfault_update_stats(struct uvm_faultinfo *ufi) 491 { 492 struct vm_map *map; 493 struct proc *p; 494 vsize_t res; 495 496 map = ufi->orig_map; 497 498 /* 499 * If this is a nested pmap (eg, a virtual machine pmap managed 500 * by vmm(4) on amd64/i386), don't do any updating, just return. 501 * 502 * pmap_nested() on other archs is #defined to 0, so this is a 503 * no-op. 504 */ 505 if (pmap_nested(map->pmap)) 506 return; 507 508 /* Update the maxrss for the process. */ 509 if (map->flags & VM_MAP_ISVMSPACE) { 510 p = curproc; 511 KASSERT(p != NULL && &p->p_vmspace->vm_map == map); 512 513 res = pmap_resident_count(map->pmap); 514 /* Convert res from pages to kilobytes. */ 515 res <<= (PAGE_SHIFT - 10); 516 517 if (p->p_ru.ru_maxrss < res) 518 p->p_ru.ru_maxrss = res; 519 } 520 } 521 522 /* 523 * F A U L T - m a i n e n t r y p o i n t 524 */ 525 526 /* 527 * uvm_fault: page fault handler 528 * 529 * => called from MD code to resolve a page fault 530 * => VM data structures usually should be unlocked. however, it is 531 * possible to call here with the main map locked if the caller 532 * gets a write lock, sets it recursive, and then calls us (c.f. 533 * uvm_map_pageable). this should be avoided because it keeps 534 * the map locked off during I/O. 535 * => MUST NEVER BE CALLED IN INTERRUPT CONTEXT 536 */ 537 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 538 ~PROT_WRITE : PROT_MASK) 539 struct uvm_faultctx { 540 /* 541 * the following members are set up by uvm_fault_check() and 542 * read-only after that. 543 */ 544 vm_prot_t enter_prot; 545 vm_prot_t access_type; 546 vaddr_t startva; 547 int npages; 548 int centeridx; 549 boolean_t narrow; 550 boolean_t wired; 551 paddr_t pa_flags; 552 }; 553 554 int uvm_fault_check( 555 struct uvm_faultinfo *, struct uvm_faultctx *, 556 struct vm_anon ***); 557 558 int uvm_fault_upper( 559 struct uvm_faultinfo *, struct uvm_faultctx *, 560 struct vm_anon **, vm_fault_t); 561 boolean_t uvm_fault_upper_lookup( 562 struct uvm_faultinfo *, const struct uvm_faultctx *, 563 struct vm_anon **, struct vm_page **); 564 565 int uvm_fault_lower( 566 struct uvm_faultinfo *, struct uvm_faultctx *, 567 struct vm_page **, vm_fault_t); 568 569 int 570 uvm_fault(vm_map_t orig_map, vaddr_t vaddr, vm_fault_t fault_type, 571 vm_prot_t access_type) 572 { 573 struct uvm_faultinfo ufi; 574 struct uvm_faultctx flt; 575 boolean_t shadowed; 576 struct vm_anon *anons_store[UVM_MAXRANGE], **anons; 577 struct vm_page *pages[UVM_MAXRANGE]; 578 int error; 579 580 counters_inc(uvmexp_counters, faults); 581 TRACEPOINT(uvm, fault, vaddr, fault_type, access_type, NULL); 582 583 /* 584 * init the IN parameters in the ufi 585 */ 586 ufi.orig_map = orig_map; 587 ufi.orig_rvaddr = trunc_page(vaddr); 588 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ 589 if (fault_type == VM_FAULT_WIRE) 590 flt.narrow = TRUE; /* don't look for neighborhood 591 * pages on wire */ 592 else 593 flt.narrow = FALSE; /* normal fault */ 594 flt.access_type = access_type; 595 596 597 error = ERESTART; 598 while (error == ERESTART) { /* ReFault: */ 599 anons = anons_store; 600 601 error = uvm_fault_check(&ufi, &flt, &anons); 602 if (error != 0) 603 continue; 604 605 /* True if there is an anon at the faulting address */ 606 shadowed = uvm_fault_upper_lookup(&ufi, &flt, anons, pages); 607 if (shadowed == TRUE) { 608 /* case 1: fault on an anon in our amap */ 609 error = uvm_fault_upper(&ufi, &flt, anons, fault_type); 610 } else { 611 struct uvm_object *uobj = ufi.entry->object.uvm_obj; 612 613 /* 614 * if the desired page is not shadowed by the amap and 615 * we have a backing object, then we check to see if 616 * the backing object would prefer to handle the fault 617 * itself (rather than letting us do it with the usual 618 * pgo_get hook). the backing object signals this by 619 * providing a pgo_fault routine. 620 */ 621 if (uobj != NULL && uobj->pgops->pgo_fault != NULL) { 622 KERNEL_LOCK(); 623 error = uobj->pgops->pgo_fault(&ufi, 624 flt.startva, pages, flt.npages, 625 flt.centeridx, fault_type, flt.access_type, 626 PGO_LOCKED); 627 KERNEL_UNLOCK(); 628 629 if (error == VM_PAGER_OK) 630 error = 0; 631 else if (error == VM_PAGER_REFAULT) 632 error = ERESTART; 633 else 634 error = EACCES; 635 } else { 636 /* case 2: fault on backing obj or zero fill */ 637 KERNEL_LOCK(); 638 error = uvm_fault_lower(&ufi, &flt, pages, 639 fault_type); 640 KERNEL_UNLOCK(); 641 } 642 } 643 } 644 645 return error; 646 } 647 648 /* 649 * uvm_fault_check: check prot, handle needs-copy, etc. 650 * 651 * 1. lookup entry. 652 * 2. check protection. 653 * 3. adjust fault condition (mainly for simulated fault). 654 * 4. handle needs-copy (lazy amap copy). 655 * 5. establish range of interest for neighbor fault (aka pre-fault). 656 * 6. look up anons (if amap exists). 657 * 7. flush pages (if MADV_SEQUENTIAL) 658 * 659 * => called with nothing locked. 660 * => if we fail (result != 0) we unlock everything. 661 * => initialize/adjust many members of flt. 662 */ 663 int 664 uvm_fault_check(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 665 struct vm_anon ***ranons) 666 { 667 struct vm_amap *amap; 668 struct uvm_object *uobj; 669 int nback, nforw; 670 671 /* 672 * lookup and lock the maps 673 */ 674 if (uvmfault_lookup(ufi, FALSE) == FALSE) { 675 return EFAULT; 676 } 677 /* locked: maps(read) */ 678 679 #ifdef DIAGNOSTIC 680 if ((ufi->map->flags & VM_MAP_PAGEABLE) == 0) 681 panic("uvm_fault: fault on non-pageable map (%p, 0x%lx)", 682 ufi->map, ufi->orig_rvaddr); 683 #endif 684 685 /* 686 * check protection 687 */ 688 if ((ufi->entry->protection & flt->access_type) != flt->access_type) { 689 uvmfault_unlockmaps(ufi, FALSE); 690 return EACCES; 691 } 692 693 /* 694 * "enter_prot" is the protection we want to enter the page in at. 695 * for certain pages (e.g. copy-on-write pages) this protection can 696 * be more strict than ufi->entry->protection. "wired" means either 697 * the entry is wired or we are fault-wiring the pg. 698 */ 699 700 flt->enter_prot = ufi->entry->protection; 701 flt->pa_flags = UVM_ET_ISWC(ufi->entry) ? PMAP_WC : 0; 702 flt->wired = VM_MAPENT_ISWIRED(ufi->entry) || (flt->narrow == TRUE); 703 if (flt->wired) 704 flt->access_type = flt->enter_prot; /* full access for wired */ 705 706 /* handle "needs_copy" case. */ 707 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 708 if ((flt->access_type & PROT_WRITE) || 709 (ufi->entry->object.uvm_obj == NULL)) { 710 /* need to clear */ 711 uvmfault_unlockmaps(ufi, FALSE); 712 uvmfault_amapcopy(ufi); 713 counters_inc(uvmexp_counters, flt_amcopy); 714 return ERESTART; 715 } else { 716 /* 717 * ensure that we pmap_enter page R/O since 718 * needs_copy is still true 719 */ 720 flt->enter_prot &= ~PROT_WRITE; 721 } 722 } 723 724 /* 725 * identify the players 726 */ 727 amap = ufi->entry->aref.ar_amap; /* upper layer */ 728 uobj = ufi->entry->object.uvm_obj; /* lower layer */ 729 730 /* 731 * check for a case 0 fault. if nothing backing the entry then 732 * error now. 733 */ 734 if (amap == NULL && uobj == NULL) { 735 uvmfault_unlockmaps(ufi, FALSE); 736 return EFAULT; 737 } 738 739 /* 740 * establish range of interest based on advice from mapper 741 * and then clip to fit map entry. note that we only want 742 * to do this the first time through the fault. if we 743 * ReFault we will disable this by setting "narrow" to true. 744 */ 745 if (flt->narrow == FALSE) { 746 747 /* wide fault (!narrow) */ 748 nback = min(uvmadvice[ufi->entry->advice].nback, 749 (ufi->orig_rvaddr - ufi->entry->start) >> PAGE_SHIFT); 750 flt->startva = ufi->orig_rvaddr - ((vsize_t)nback << PAGE_SHIFT); 751 nforw = min(uvmadvice[ufi->entry->advice].nforw, 752 ((ufi->entry->end - ufi->orig_rvaddr) >> PAGE_SHIFT) - 1); 753 /* 754 * note: "-1" because we don't want to count the 755 * faulting page as forw 756 */ 757 flt->npages = nback + nforw + 1; 758 flt->centeridx = nback; 759 760 flt->narrow = TRUE; /* ensure only once per-fault */ 761 } else { 762 /* narrow fault! */ 763 nback = nforw = 0; 764 flt->startva = ufi->orig_rvaddr; 765 flt->npages = 1; 766 flt->centeridx = 0; 767 } 768 769 /* 770 * if we've got an amap then lock it and extract current anons. 771 */ 772 if (amap) { 773 amap_lock(amap); 774 amap_lookups(&ufi->entry->aref, 775 flt->startva - ufi->entry->start, *ranons, flt->npages); 776 } else { 777 *ranons = NULL; /* to be safe */ 778 } 779 780 /* 781 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages 782 * now and then forget about them (for the rest of the fault). 783 */ 784 if (ufi->entry->advice == MADV_SEQUENTIAL && nback != 0) { 785 /* flush back-page anons? */ 786 if (amap) 787 uvmfault_anonflush(*ranons, nback); 788 789 /* 790 * flush object? 791 */ 792 if (uobj) { 793 voff_t uoff; 794 795 uoff = (flt->startva - ufi->entry->start) + ufi->entry->offset; 796 KERNEL_LOCK(); 797 (void) uobj->pgops->pgo_flush(uobj, uoff, uoff + 798 ((vsize_t)nback << PAGE_SHIFT), PGO_DEACTIVATE); 799 KERNEL_UNLOCK(); 800 } 801 802 /* now forget about the backpages */ 803 if (amap) 804 *ranons += nback; 805 flt->startva += ((vsize_t)nback << PAGE_SHIFT); 806 flt->npages -= nback; 807 flt->centeridx = 0; 808 } 809 810 return 0; 811 } 812 813 /* 814 * uvm_fault_upper_lookup: look up existing h/w mapping and amap. 815 * 816 * iterate range of interest: 817 * 1. check if h/w mapping exists. if yes, we don't care 818 * 2. check if anon exists. if not, page is lower. 819 * 3. if anon exists, enter h/w mapping for neighbors. 820 * 821 * => called with amap locked (if exists). 822 */ 823 boolean_t 824 uvm_fault_upper_lookup(struct uvm_faultinfo *ufi, 825 const struct uvm_faultctx *flt, struct vm_anon **anons, 826 struct vm_page **pages) 827 { 828 struct vm_amap *amap = ufi->entry->aref.ar_amap; 829 struct vm_anon *anon; 830 boolean_t shadowed; 831 vaddr_t currva; 832 paddr_t pa; 833 int lcv; 834 835 /* locked: maps(read), amap(if there) */ 836 KASSERT(amap == NULL || 837 rw_write_held(amap->am_lock)); 838 839 /* 840 * map in the backpages and frontpages we found in the amap in hopes 841 * of preventing future faults. we also init the pages[] array as 842 * we go. 843 */ 844 currva = flt->startva; 845 shadowed = FALSE; 846 for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) { 847 /* 848 * dont play with VAs that are already mapped 849 * except for center) 850 */ 851 if (lcv != flt->centeridx && 852 pmap_extract(ufi->orig_map->pmap, currva, &pa)) { 853 pages[lcv] = PGO_DONTCARE; 854 continue; 855 } 856 857 /* 858 * unmapped or center page. check if any anon at this level. 859 */ 860 if (amap == NULL || anons[lcv] == NULL) { 861 pages[lcv] = NULL; 862 continue; 863 } 864 865 /* 866 * check for present page and map if possible. 867 */ 868 pages[lcv] = PGO_DONTCARE; 869 if (lcv == flt->centeridx) { /* save center for later! */ 870 shadowed = TRUE; 871 continue; 872 } 873 anon = anons[lcv]; 874 KASSERT(anon->an_lock == amap->am_lock); 875 if (anon->an_page && 876 (anon->an_page->pg_flags & (PG_RELEASED|PG_BUSY)) == 0) { 877 uvm_lock_pageq(); 878 uvm_pageactivate(anon->an_page); /* reactivate */ 879 uvm_unlock_pageq(); 880 counters_inc(uvmexp_counters, flt_namap); 881 882 /* 883 * Since this isn't the page that's actually faulting, 884 * ignore pmap_enter() failures; it's not critical 885 * that we enter these right now. 886 */ 887 (void) pmap_enter(ufi->orig_map->pmap, currva, 888 VM_PAGE_TO_PHYS(anon->an_page) | flt->pa_flags, 889 (anon->an_ref > 1) ? 890 (flt->enter_prot & ~PROT_WRITE) : flt->enter_prot, 891 PMAP_CANFAIL | 892 (VM_MAPENT_ISWIRED(ufi->entry) ? PMAP_WIRED : 0)); 893 } 894 } 895 if (flt->npages > 1) 896 pmap_update(ufi->orig_map->pmap); 897 898 return shadowed; 899 } 900 901 /* 902 * uvm_fault_upper: handle upper fault. 903 * 904 * 1. acquire anon lock. 905 * 2. get anon. let uvmfault_anonget do the dirty work. 906 * 3. if COW, promote data to new anon 907 * 4. enter h/w mapping 908 */ 909 int 910 uvm_fault_upper(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 911 struct vm_anon **anons, vm_fault_t fault_type) 912 { 913 struct vm_amap *amap = ufi->entry->aref.ar_amap; 914 struct vm_anon *oanon, *anon = anons[flt->centeridx]; 915 struct vm_page *pg = NULL; 916 int error, ret; 917 918 /* locked: maps(read), amap, anon */ 919 KASSERT(rw_write_held(amap->am_lock)); 920 KASSERT(anon->an_lock == amap->am_lock); 921 922 /* 923 * no matter if we have case 1A or case 1B we are going to need to 924 * have the anon's memory resident. ensure that now. 925 */ 926 /* 927 * let uvmfault_anonget do the dirty work. 928 * if it fails (!OK) it will unlock everything for us. 929 * if it succeeds, locks are still valid and locked. 930 * also, if it is OK, then the anon's page is on the queues. 931 * if the page is on loan from a uvm_object, then anonget will 932 * lock that object for us if it does not fail. 933 */ 934 error = uvmfault_anonget(ufi, amap, anon); 935 switch (error) { 936 case VM_PAGER_OK: 937 break; 938 939 case VM_PAGER_REFAULT: 940 return ERESTART; 941 942 case VM_PAGER_ERROR: 943 /* 944 * An error occurred while trying to bring in the 945 * page -- this is the only error we return right 946 * now. 947 */ 948 return EACCES; /* XXX */ 949 default: 950 #ifdef DIAGNOSTIC 951 panic("uvm_fault: uvmfault_anonget -> %d", error); 952 #else 953 return EACCES; 954 #endif 955 } 956 957 KASSERT(rw_write_held(amap->am_lock)); 958 KASSERT(anon->an_lock == amap->am_lock); 959 960 /* 961 * if we are case 1B then we will need to allocate a new blank 962 * anon to transfer the data into. note that we have a lock 963 * on anon, so no one can busy or release the page until we are done. 964 * also note that the ref count can't drop to zero here because 965 * it is > 1 and we are only dropping one ref. 966 * 967 * in the (hopefully very rare) case that we are out of RAM we 968 * will unlock, wait for more RAM, and refault. 969 * 970 * if we are out of anon VM we wait for RAM to become available. 971 */ 972 973 if ((flt->access_type & PROT_WRITE) != 0 && anon->an_ref > 1) { 974 counters_inc(uvmexp_counters, flt_acow); 975 oanon = anon; /* oanon = old */ 976 anon = uvm_analloc(); 977 if (anon) { 978 anon->an_lock = amap->am_lock; 979 pg = uvm_pagealloc(NULL, 0, anon, 0); 980 } 981 982 /* check for out of RAM */ 983 if (anon == NULL || pg == NULL) { 984 uvmfault_unlockall(ufi, amap, NULL); 985 if (anon == NULL) 986 counters_inc(uvmexp_counters, flt_noanon); 987 else { 988 anon->an_lock = NULL; 989 anon->an_ref--; 990 uvm_anfree(anon); 991 counters_inc(uvmexp_counters, flt_noram); 992 } 993 994 if (uvm_swapisfull()) 995 return ENOMEM; 996 997 /* out of RAM, wait for more */ 998 if (anon == NULL) 999 uvm_anwait(); 1000 else 1001 uvm_wait("flt_noram3"); 1002 return ERESTART; 1003 } 1004 1005 /* got all resources, replace anon with nanon */ 1006 uvm_pagecopy(oanon->an_page, pg); /* pg now !PG_CLEAN */ 1007 /* un-busy! new page */ 1008 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE); 1009 UVM_PAGE_OWN(pg, NULL); 1010 ret = amap_add(&ufi->entry->aref, 1011 ufi->orig_rvaddr - ufi->entry->start, anon, 1); 1012 KASSERT(ret == 0); 1013 1014 /* deref: can not drop to zero here by defn! */ 1015 oanon->an_ref--; 1016 1017 /* 1018 * note: anon is _not_ locked, but we have the sole references 1019 * to in from amap. 1020 * thus, no one can get at it until we are done with it. 1021 */ 1022 } else { 1023 counters_inc(uvmexp_counters, flt_anon); 1024 oanon = anon; 1025 pg = anon->an_page; 1026 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */ 1027 flt->enter_prot = flt->enter_prot & ~PROT_WRITE; 1028 } 1029 1030 /* 1031 * now map the page in . 1032 */ 1033 if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr, 1034 VM_PAGE_TO_PHYS(pg) | flt->pa_flags, flt->enter_prot, 1035 flt->access_type | PMAP_CANFAIL | (flt->wired ? PMAP_WIRED : 0)) != 0) { 1036 /* 1037 * No need to undo what we did; we can simply think of 1038 * this as the pmap throwing away the mapping information. 1039 * 1040 * We do, however, have to go through the ReFault path, 1041 * as the map may change while we're asleep. 1042 */ 1043 uvmfault_unlockall(ufi, amap, NULL); 1044 if (uvm_swapisfull()) { 1045 /* XXX instrumentation */ 1046 return ENOMEM; 1047 } 1048 /* XXX instrumentation */ 1049 uvm_wait("flt_pmfail1"); 1050 return ERESTART; 1051 } 1052 1053 /* 1054 * ... update the page queues. 1055 */ 1056 uvm_lock_pageq(); 1057 1058 if (fault_type == VM_FAULT_WIRE) { 1059 uvm_pagewire(pg); 1060 /* 1061 * since the now-wired page cannot be paged out, 1062 * release its swap resources for others to use. 1063 * since an anon with no swap cannot be PG_CLEAN, 1064 * clear its clean flag now. 1065 */ 1066 atomic_clearbits_int(&pg->pg_flags, PG_CLEAN); 1067 uvm_anon_dropswap(anon); 1068 } else { 1069 /* activate it */ 1070 uvm_pageactivate(pg); 1071 } 1072 1073 uvm_unlock_pageq(); 1074 1075 /* 1076 * done case 1! finish up by unlocking everything and returning success 1077 */ 1078 uvmfault_unlockall(ufi, amap, NULL); 1079 pmap_update(ufi->orig_map->pmap); 1080 return 0; 1081 } 1082 1083 /* 1084 * uvm_fault_lower_lookup: look up on-memory uobj pages. 1085 * 1086 * 1. get on-memory pages. 1087 * 2. if failed, give up (get only center page later). 1088 * 3. if succeeded, enter h/w mapping of neighbor pages. 1089 */ 1090 1091 struct vm_page * 1092 uvm_fault_lower_lookup( 1093 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 1094 struct vm_page **pages) 1095 { 1096 struct uvm_object *uobj = ufi->entry->object.uvm_obj; 1097 struct vm_page *uobjpage = NULL; 1098 int lcv, gotpages; 1099 vaddr_t currva; 1100 1101 counters_inc(uvmexp_counters, flt_lget); 1102 gotpages = flt->npages; 1103 (void) uobj->pgops->pgo_get(uobj, 1104 ufi->entry->offset + (flt->startva - ufi->entry->start), 1105 pages, &gotpages, flt->centeridx, 1106 flt->access_type & MASK(ufi->entry), ufi->entry->advice, 1107 PGO_LOCKED); 1108 1109 /* 1110 * check for pages to map, if we got any 1111 */ 1112 if (gotpages == 0) { 1113 return NULL; 1114 } 1115 1116 currva = flt->startva; 1117 for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) { 1118 if (pages[lcv] == NULL || 1119 pages[lcv] == PGO_DONTCARE) 1120 continue; 1121 1122 KASSERT((pages[lcv]->pg_flags & PG_RELEASED) == 0); 1123 1124 /* 1125 * if center page is resident and not 1126 * PG_BUSY, then pgo_get made it PG_BUSY 1127 * for us and gave us a handle to it. 1128 * remember this page as "uobjpage." 1129 * (for later use). 1130 */ 1131 if (lcv == flt->centeridx) { 1132 uobjpage = pages[lcv]; 1133 continue; 1134 } 1135 1136 /* 1137 * note: calling pgo_get with locked data 1138 * structures returns us pages which are 1139 * neither busy nor released, so we don't 1140 * need to check for this. we can just 1141 * directly enter the page (after moving it 1142 * to the head of the active queue [useful?]). 1143 */ 1144 1145 uvm_lock_pageq(); 1146 uvm_pageactivate(pages[lcv]); /* reactivate */ 1147 uvm_unlock_pageq(); 1148 counters_inc(uvmexp_counters, flt_nomap); 1149 1150 /* 1151 * Since this page isn't the page that's 1152 * actually faulting, ignore pmap_enter() 1153 * failures; it's not critical that we 1154 * enter these right now. 1155 */ 1156 (void) pmap_enter(ufi->orig_map->pmap, currva, 1157 VM_PAGE_TO_PHYS(pages[lcv]) | flt->pa_flags, 1158 flt->enter_prot & MASK(ufi->entry), 1159 PMAP_CANFAIL | 1160 (flt->wired ? PMAP_WIRED : 0)); 1161 1162 /* 1163 * NOTE: page can't be PG_WANTED because 1164 * we've held the lock the whole time 1165 * we've had the handle. 1166 */ 1167 atomic_clearbits_int(&pages[lcv]->pg_flags, PG_BUSY); 1168 UVM_PAGE_OWN(pages[lcv], NULL); 1169 } 1170 pmap_update(ufi->orig_map->pmap); 1171 1172 return uobjpage; 1173 } 1174 1175 /* 1176 * uvm_fault_lower: handle lower fault. 1177 * 1178 */ 1179 int 1180 uvm_fault_lower(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1181 struct vm_page **pages, vm_fault_t fault_type) 1182 { 1183 struct vm_amap *amap = ufi->entry->aref.ar_amap; 1184 struct uvm_object *uobj = ufi->entry->object.uvm_obj; 1185 boolean_t promote, locked; 1186 int result; 1187 struct vm_page *uobjpage, *pg = NULL; 1188 struct vm_anon *anon = NULL; 1189 voff_t uoff; 1190 1191 /* 1192 * now, if the desired page is not shadowed by the amap and we have 1193 * a backing object that does not have a special fault routine, then 1194 * we ask (with pgo_get) the object for resident pages that we care 1195 * about and attempt to map them in. we do not let pgo_get block 1196 * (PGO_LOCKED). 1197 */ 1198 if (uobj == NULL) { 1199 /* zero fill; don't care neighbor pages */ 1200 uobjpage = NULL; 1201 } else { 1202 uobjpage = uvm_fault_lower_lookup(ufi, flt, pages); 1203 } 1204 1205 /* 1206 * note that at this point we are done with any front or back pages. 1207 * we are now going to focus on the center page (i.e. the one we've 1208 * faulted on). if we have faulted on the bottom (uobj) 1209 * layer [i.e. case 2] and the page was both present and available, 1210 * then we've got a pointer to it as "uobjpage" and we've already 1211 * made it BUSY. 1212 */ 1213 1214 /* 1215 * note that uobjpage can not be PGO_DONTCARE at this point. we now 1216 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we 1217 * have a backing object, check and see if we are going to promote 1218 * the data up to an anon during the fault. 1219 */ 1220 if (uobj == NULL) { 1221 uobjpage = PGO_DONTCARE; 1222 promote = TRUE; /* always need anon here */ 1223 } else { 1224 KASSERT(uobjpage != PGO_DONTCARE); 1225 promote = (flt->access_type & PROT_WRITE) && 1226 UVM_ET_ISCOPYONWRITE(ufi->entry); 1227 } 1228 1229 /* 1230 * if uobjpage is not null then we do not need to do I/O to get the 1231 * uobjpage. 1232 * 1233 * if uobjpage is null, then we need to ask the pager to 1234 * get the data for us. once we have the data, we need to reverify 1235 * the state the world. we are currently not holding any resources. 1236 */ 1237 if (uobjpage) { 1238 /* update rusage counters */ 1239 curproc->p_ru.ru_minflt++; 1240 } else { 1241 int gotpages; 1242 1243 /* update rusage counters */ 1244 curproc->p_ru.ru_majflt++; 1245 1246 uvmfault_unlockall(ufi, amap, NULL); 1247 1248 counters_inc(uvmexp_counters, flt_get); 1249 gotpages = 1; 1250 uoff = (ufi->orig_rvaddr - ufi->entry->start) + ufi->entry->offset; 1251 result = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages, 1252 0, flt->access_type & MASK(ufi->entry), ufi->entry->advice, 1253 PGO_SYNCIO); 1254 1255 /* 1256 * recover from I/O 1257 */ 1258 if (result != VM_PAGER_OK) { 1259 KASSERT(result != VM_PAGER_PEND); 1260 1261 if (result == VM_PAGER_AGAIN) { 1262 tsleep_nsec(&nowake, PVM, "fltagain2", 1263 SEC_TO_NSEC(1)); 1264 return ERESTART; 1265 } 1266 1267 if (!UVM_ET_ISNOFAULT(ufi->entry)) 1268 return (EIO); 1269 1270 uobjpage = PGO_DONTCARE; 1271 promote = TRUE; 1272 } 1273 1274 /* re-verify the state of the world. */ 1275 locked = uvmfault_relock(ufi); 1276 if (locked && amap != NULL) 1277 amap_lock(amap); 1278 1279 /* 1280 * Re-verify that amap slot is still free. if there is 1281 * a problem, we clean up. 1282 */ 1283 if (locked && amap && amap_lookup(&ufi->entry->aref, 1284 ufi->orig_rvaddr - ufi->entry->start)) { 1285 if (locked) 1286 uvmfault_unlockall(ufi, amap, NULL); 1287 locked = FALSE; 1288 } 1289 1290 /* didn't get the lock? release the page and retry. */ 1291 if (locked == FALSE && uobjpage != PGO_DONTCARE) { 1292 uvm_lock_pageq(); 1293 /* make sure it is in queues */ 1294 uvm_pageactivate(uobjpage); 1295 uvm_unlock_pageq(); 1296 1297 if (uobjpage->pg_flags & PG_WANTED) 1298 /* still holding object lock */ 1299 wakeup(uobjpage); 1300 atomic_clearbits_int(&uobjpage->pg_flags, 1301 PG_BUSY|PG_WANTED); 1302 UVM_PAGE_OWN(uobjpage, NULL); 1303 return ERESTART; 1304 } 1305 if (locked == FALSE) 1306 return ERESTART; 1307 1308 /* 1309 * we have the data in uobjpage which is PG_BUSY 1310 */ 1311 } 1312 1313 /* 1314 * notes: 1315 * - at this point uobjpage can not be NULL 1316 * - at this point uobjpage could be PG_WANTED (handle later) 1317 */ 1318 if (promote == FALSE) { 1319 /* 1320 * we are not promoting. if the mapping is COW ensure that we 1321 * don't give more access than we should (e.g. when doing a read 1322 * fault on a COPYONWRITE mapping we want to map the COW page in 1323 * R/O even though the entry protection could be R/W). 1324 * 1325 * set "pg" to the page we want to map in (uobjpage, usually) 1326 */ 1327 counters_inc(uvmexp_counters, flt_obj); 1328 if (UVM_ET_ISCOPYONWRITE(ufi->entry)) 1329 flt->enter_prot &= ~PROT_WRITE; 1330 pg = uobjpage; /* map in the actual object */ 1331 1332 /* assert(uobjpage != PGO_DONTCARE) */ 1333 1334 /* 1335 * we are faulting directly on the page. 1336 */ 1337 } else { 1338 /* 1339 * if we are going to promote the data to an anon we 1340 * allocate a blank anon here and plug it into our amap. 1341 */ 1342 #ifdef DIAGNOSTIC 1343 if (amap == NULL) 1344 panic("uvm_fault: want to promote data, but no anon"); 1345 #endif 1346 1347 anon = uvm_analloc(); 1348 if (anon) { 1349 /* 1350 * In `Fill in data...' below, if 1351 * uobjpage == PGO_DONTCARE, we want 1352 * a zero'd, dirty page, so have 1353 * uvm_pagealloc() do that for us. 1354 */ 1355 anon->an_lock = amap->am_lock; 1356 pg = uvm_pagealloc(NULL, 0, anon, 1357 (uobjpage == PGO_DONTCARE) ? UVM_PGA_ZERO : 0); 1358 } 1359 1360 /* 1361 * out of memory resources? 1362 */ 1363 if (anon == NULL || pg == NULL) { 1364 /* 1365 * arg! must unbusy our page and fail or sleep. 1366 */ 1367 if (uobjpage != PGO_DONTCARE) { 1368 uvm_lock_pageq(); 1369 uvm_pageactivate(uobjpage); 1370 uvm_unlock_pageq(); 1371 1372 if (uobjpage->pg_flags & PG_WANTED) 1373 wakeup(uobjpage); 1374 atomic_clearbits_int(&uobjpage->pg_flags, 1375 PG_BUSY|PG_WANTED); 1376 UVM_PAGE_OWN(uobjpage, NULL); 1377 } 1378 1379 /* unlock and fail ... */ 1380 uvmfault_unlockall(ufi, amap, uobj); 1381 if (anon == NULL) 1382 counters_inc(uvmexp_counters, flt_noanon); 1383 else { 1384 anon->an_lock = NULL; 1385 anon->an_ref--; 1386 uvm_anfree(anon); 1387 counters_inc(uvmexp_counters, flt_noram); 1388 } 1389 1390 if (uvm_swapisfull()) 1391 return (ENOMEM); 1392 1393 /* out of RAM, wait for more */ 1394 if (anon == NULL) 1395 uvm_anwait(); 1396 else 1397 uvm_wait("flt_noram5"); 1398 return ERESTART; 1399 } 1400 1401 /* 1402 * fill in the data 1403 */ 1404 if (uobjpage != PGO_DONTCARE) { 1405 counters_inc(uvmexp_counters, flt_prcopy); 1406 /* copy page [pg now dirty] */ 1407 uvm_pagecopy(uobjpage, pg); 1408 1409 /* 1410 * promote to shared amap? make sure all sharing 1411 * procs see it 1412 */ 1413 if ((amap_flags(amap) & AMAP_SHARED) != 0) { 1414 pmap_page_protect(uobjpage, PROT_NONE); 1415 } 1416 1417 /* dispose of uobjpage. drop handle to uobj as well. */ 1418 if (uobjpage->pg_flags & PG_WANTED) 1419 wakeup(uobjpage); 1420 atomic_clearbits_int(&uobjpage->pg_flags, 1421 PG_BUSY|PG_WANTED); 1422 UVM_PAGE_OWN(uobjpage, NULL); 1423 uvm_lock_pageq(); 1424 uvm_pageactivate(uobjpage); 1425 uvm_unlock_pageq(); 1426 uobj = NULL; 1427 } else { 1428 counters_inc(uvmexp_counters, flt_przero); 1429 /* 1430 * Page is zero'd and marked dirty by uvm_pagealloc() 1431 * above. 1432 */ 1433 } 1434 1435 if (amap_add(&ufi->entry->aref, 1436 ufi->orig_rvaddr - ufi->entry->start, anon, 0)) { 1437 uvmfault_unlockall(ufi, amap, NULL); 1438 uvm_anfree(anon); 1439 counters_inc(uvmexp_counters, flt_noamap); 1440 1441 if (uvm_swapisfull()) 1442 return (ENOMEM); 1443 1444 amap_populate(&ufi->entry->aref, 1445 ufi->orig_rvaddr - ufi->entry->start); 1446 return ERESTART; 1447 } 1448 } 1449 1450 /* note: pg is either the uobjpage or the new page in the new anon */ 1451 /* 1452 * all resources are present. we can now map it in and free our 1453 * resources. 1454 */ 1455 if (amap == NULL) 1456 KASSERT(anon == NULL); 1457 else { 1458 KASSERT(rw_write_held(amap->am_lock)); 1459 KASSERT(anon == NULL || anon->an_lock == amap->am_lock); 1460 } 1461 if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr, 1462 VM_PAGE_TO_PHYS(pg) | flt->pa_flags, flt->enter_prot, 1463 flt->access_type | PMAP_CANFAIL | (flt->wired ? PMAP_WIRED : 0)) != 0) { 1464 /* 1465 * No need to undo what we did; we can simply think of 1466 * this as the pmap throwing away the mapping information. 1467 * 1468 * We do, however, have to go through the ReFault path, 1469 * as the map may change while we're asleep. 1470 */ 1471 if (pg->pg_flags & PG_WANTED) 1472 wakeup(pg); 1473 1474 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED); 1475 UVM_PAGE_OWN(pg, NULL); 1476 uvmfault_unlockall(ufi, amap, uobj); 1477 if (uvm_swapisfull()) { 1478 /* XXX instrumentation */ 1479 return (ENOMEM); 1480 } 1481 /* XXX instrumentation */ 1482 uvm_wait("flt_pmfail2"); 1483 return ERESTART; 1484 } 1485 1486 uvm_lock_pageq(); 1487 1488 if (fault_type == VM_FAULT_WIRE) { 1489 uvm_pagewire(pg); 1490 if (pg->pg_flags & PQ_AOBJ) { 1491 /* 1492 * since the now-wired page cannot be paged out, 1493 * release its swap resources for others to use. 1494 * since an aobj page with no swap cannot be PG_CLEAN, 1495 * clear its clean flag now. 1496 */ 1497 atomic_clearbits_int(&pg->pg_flags, PG_CLEAN); 1498 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT); 1499 } 1500 } else { 1501 /* activate it */ 1502 uvm_pageactivate(pg); 1503 } 1504 uvm_unlock_pageq(); 1505 1506 if (pg->pg_flags & PG_WANTED) 1507 wakeup(pg); 1508 1509 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED); 1510 UVM_PAGE_OWN(pg, NULL); 1511 uvmfault_unlockall(ufi, amap, uobj); 1512 pmap_update(ufi->orig_map->pmap); 1513 1514 return (0); 1515 } 1516 1517 1518 /* 1519 * uvm_fault_wire: wire down a range of virtual addresses in a map. 1520 * 1521 * => map may be read-locked by caller, but MUST NOT be write-locked. 1522 * => if map is read-locked, any operations which may cause map to 1523 * be write-locked in uvm_fault() must be taken care of by 1524 * the caller. See uvm_map_pageable(). 1525 */ 1526 int 1527 uvm_fault_wire(vm_map_t map, vaddr_t start, vaddr_t end, vm_prot_t access_type) 1528 { 1529 vaddr_t va; 1530 int rv; 1531 1532 /* 1533 * now fault it in a page at a time. if the fault fails then we have 1534 * to undo what we have done. note that in uvm_fault PROT_NONE 1535 * is replaced with the max protection if fault_type is VM_FAULT_WIRE. 1536 */ 1537 for (va = start ; va < end ; va += PAGE_SIZE) { 1538 rv = uvm_fault(map, va, VM_FAULT_WIRE, access_type); 1539 if (rv) { 1540 if (va != start) { 1541 uvm_fault_unwire(map, start, va); 1542 } 1543 return (rv); 1544 } 1545 } 1546 1547 return (0); 1548 } 1549 1550 /* 1551 * uvm_fault_unwire(): unwire range of virtual space. 1552 */ 1553 void 1554 uvm_fault_unwire(vm_map_t map, vaddr_t start, vaddr_t end) 1555 { 1556 1557 vm_map_lock_read(map); 1558 uvm_fault_unwire_locked(map, start, end); 1559 vm_map_unlock_read(map); 1560 } 1561 1562 /* 1563 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire(). 1564 * 1565 * => map must be at least read-locked. 1566 */ 1567 void 1568 uvm_fault_unwire_locked(vm_map_t map, vaddr_t start, vaddr_t end) 1569 { 1570 vm_map_entry_t entry, next; 1571 pmap_t pmap = vm_map_pmap(map); 1572 vaddr_t va; 1573 paddr_t pa; 1574 struct vm_page *pg; 1575 1576 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0); 1577 1578 /* 1579 * we assume that the area we are unwiring has actually been wired 1580 * in the first place. this means that we should be able to extract 1581 * the PAs from the pmap. we also lock out the page daemon so that 1582 * we can call uvm_pageunwire. 1583 */ 1584 1585 uvm_lock_pageq(); 1586 1587 /* 1588 * find the beginning map entry for the region. 1589 */ 1590 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map)); 1591 if (uvm_map_lookup_entry(map, start, &entry) == FALSE) 1592 panic("uvm_fault_unwire_locked: address not in map"); 1593 1594 for (va = start; va < end ; va += PAGE_SIZE) { 1595 if (pmap_extract(pmap, va, &pa) == FALSE) 1596 continue; 1597 1598 /* 1599 * find the map entry for the current address. 1600 */ 1601 KASSERT(va >= entry->start); 1602 while (va >= entry->end) { 1603 next = RBT_NEXT(uvm_map_addr, entry); 1604 KASSERT(next != NULL && next->start <= entry->end); 1605 entry = next; 1606 } 1607 1608 /* 1609 * if the entry is no longer wired, tell the pmap. 1610 */ 1611 if (VM_MAPENT_ISWIRED(entry) == 0) 1612 pmap_unwire(pmap, va); 1613 1614 pg = PHYS_TO_VM_PAGE(pa); 1615 if (pg) 1616 uvm_pageunwire(pg); 1617 } 1618 1619 uvm_unlock_pageq(); 1620 } 1621 1622 /* 1623 * uvmfault_unlockmaps: unlock the maps 1624 */ 1625 void 1626 uvmfault_unlockmaps(struct uvm_faultinfo *ufi, boolean_t write_locked) 1627 { 1628 /* 1629 * ufi can be NULL when this isn't really a fault, 1630 * but merely paging in anon data. 1631 */ 1632 if (ufi == NULL) { 1633 return; 1634 } 1635 1636 uvmfault_update_stats(ufi); 1637 if (write_locked) { 1638 vm_map_unlock(ufi->map); 1639 } else { 1640 vm_map_unlock_read(ufi->map); 1641 } 1642 } 1643 1644 /* 1645 * uvmfault_unlockall: unlock everything passed in. 1646 * 1647 * => maps must be read-locked (not write-locked). 1648 */ 1649 void 1650 uvmfault_unlockall(struct uvm_faultinfo *ufi, struct vm_amap *amap, 1651 struct uvm_object *uobj) 1652 { 1653 if (amap != NULL) 1654 amap_unlock(amap); 1655 uvmfault_unlockmaps(ufi, FALSE); 1656 } 1657 1658 /* 1659 * uvmfault_lookup: lookup a virtual address in a map 1660 * 1661 * => caller must provide a uvm_faultinfo structure with the IN 1662 * params properly filled in 1663 * => we will lookup the map entry (handling submaps) as we go 1664 * => if the lookup is a success we will return with the maps locked 1665 * => if "write_lock" is TRUE, we write_lock the map, otherwise we only 1666 * get a read lock. 1667 * => note that submaps can only appear in the kernel and they are 1668 * required to use the same virtual addresses as the map they 1669 * are referenced by (thus address translation between the main 1670 * map and the submap is unnecessary). 1671 */ 1672 1673 boolean_t 1674 uvmfault_lookup(struct uvm_faultinfo *ufi, boolean_t write_lock) 1675 { 1676 vm_map_t tmpmap; 1677 1678 /* 1679 * init ufi values for lookup. 1680 */ 1681 ufi->map = ufi->orig_map; 1682 ufi->size = ufi->orig_size; 1683 1684 /* 1685 * keep going down levels until we are done. note that there can 1686 * only be two levels so we won't loop very long. 1687 */ 1688 while (1) { 1689 if (ufi->orig_rvaddr < ufi->map->min_offset || 1690 ufi->orig_rvaddr >= ufi->map->max_offset) 1691 return FALSE; 1692 1693 /* lock map */ 1694 if (write_lock) { 1695 vm_map_lock(ufi->map); 1696 } else { 1697 vm_map_lock_read(ufi->map); 1698 } 1699 1700 /* lookup */ 1701 if (!uvm_map_lookup_entry(ufi->map, ufi->orig_rvaddr, 1702 &ufi->entry)) { 1703 uvmfault_unlockmaps(ufi, write_lock); 1704 return FALSE; 1705 } 1706 1707 /* reduce size if necessary */ 1708 if (ufi->entry->end - ufi->orig_rvaddr < ufi->size) 1709 ufi->size = ufi->entry->end - ufi->orig_rvaddr; 1710 1711 /* 1712 * submap? replace map with the submap and lookup again. 1713 * note: VAs in submaps must match VAs in main map. 1714 */ 1715 if (UVM_ET_ISSUBMAP(ufi->entry)) { 1716 tmpmap = ufi->entry->object.sub_map; 1717 uvmfault_unlockmaps(ufi, write_lock); 1718 ufi->map = tmpmap; 1719 continue; 1720 } 1721 1722 /* 1723 * got it! 1724 */ 1725 ufi->mapv = ufi->map->timestamp; 1726 return TRUE; 1727 1728 } /* while loop */ 1729 1730 /*NOTREACHED*/ 1731 } 1732 1733 /* 1734 * uvmfault_relock: attempt to relock the same version of the map 1735 * 1736 * => fault data structures should be unlocked before calling. 1737 * => if a success (TRUE) maps will be locked after call. 1738 */ 1739 boolean_t 1740 uvmfault_relock(struct uvm_faultinfo *ufi) 1741 { 1742 /* 1743 * ufi can be NULL when this isn't really a fault, 1744 * but merely paging in anon data. 1745 */ 1746 if (ufi == NULL) { 1747 return TRUE; 1748 } 1749 1750 counters_inc(uvmexp_counters, flt_relck); 1751 1752 /* 1753 * relock map. fail if version mismatch (in which case nothing 1754 * gets locked). 1755 */ 1756 vm_map_lock_read(ufi->map); 1757 if (ufi->mapv != ufi->map->timestamp) { 1758 vm_map_unlock_read(ufi->map); 1759 return FALSE; 1760 } 1761 1762 counters_inc(uvmexp_counters, flt_relckok); 1763 return TRUE; /* got it! */ 1764 } 1765