1 /* $OpenBSD: uvm_fault.c,v 1.159 2025/01/03 15:31:48 mpi Exp $ */ 2 /* $NetBSD: uvm_fault.c,v 1.51 2000/08/06 00:22:53 thorpej Exp $ */ 3 4 /* 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp 29 */ 30 31 /* 32 * uvm_fault.c: fault handler 33 */ 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/percpu.h> 39 #include <sys/proc.h> 40 #include <sys/malloc.h> 41 #include <sys/mman.h> 42 #include <sys/tracepoint.h> 43 44 #include <uvm/uvm.h> 45 46 /* 47 * 48 * a word on page faults: 49 * 50 * types of page faults we handle: 51 * 52 * CASE 1: upper layer faults CASE 2: lower layer faults 53 * 54 * CASE 1A CASE 1B CASE 2A CASE 2B 55 * read/write1 write>1 read/write +-cow_write/zero 56 * | | | | 57 * +--|--+ +--|--+ +-----+ + | + | +-----+ 58 * amap | V | | ---------> new | | | | ^ | 59 * +-----+ +-----+ +-----+ + | + | +--|--+ 60 * | | | 61 * +-----+ +-----+ +--|--+ | +--|--+ 62 * uobj | d/c | | d/c | | V | +----+ | 63 * +-----+ +-----+ +-----+ +-----+ 64 * 65 * d/c = don't care 66 * 67 * case [0]: layerless fault 68 * no amap or uobj is present. this is an error. 69 * 70 * case [1]: upper layer fault [anon active] 71 * 1A: [read] or [write with anon->an_ref == 1] 72 * I/O takes place in upper level anon and uobj is not touched. 73 * 1B: [write with anon->an_ref > 1] 74 * new anon is alloc'd and data is copied off ["COW"] 75 * 76 * case [2]: lower layer fault [uobj] 77 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area] 78 * I/O takes place directly in object. 79 * 2B: [write to copy_on_write] or [read on NULL uobj] 80 * data is "promoted" from uobj to a new anon. 81 * if uobj is null, then we zero fill. 82 * 83 * we follow the standard UVM locking protocol ordering: 84 * 85 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ) 86 * we hold a PG_BUSY page if we unlock for I/O 87 * 88 * 89 * the code is structured as follows: 90 * 91 * - init the "IN" params in the ufi structure 92 * ReFault: (ERESTART returned to the loop in uvm_fault) 93 * - do lookups [locks maps], check protection, handle needs_copy 94 * - check for case 0 fault (error) 95 * - establish "range" of fault 96 * - if we have an amap lock it and extract the anons 97 * - if sequential advice deactivate pages behind us 98 * - at the same time check pmap for unmapped areas and anon for pages 99 * that we could map in (and do map it if found) 100 * - check object for resident pages that we could map in 101 * - if (case 2) goto Case2 102 * - >>> handle case 1 103 * - ensure source anon is resident in RAM 104 * - if case 1B alloc new anon and copy from source 105 * - map the correct page in 106 * Case2: 107 * - >>> handle case 2 108 * - ensure source page is resident (if uobj) 109 * - if case 2B alloc new anon and copy from source (could be zero 110 * fill if uobj == NULL) 111 * - map the correct page in 112 * - done! 113 * 114 * note on paging: 115 * if we have to do I/O we place a PG_BUSY page in the correct object, 116 * unlock everything, and do the I/O. when I/O is done we must reverify 117 * the state of the world before assuming that our data structures are 118 * valid. [because mappings could change while the map is unlocked] 119 * 120 * alternative 1: unbusy the page in question and restart the page fault 121 * from the top (ReFault). this is easy but does not take advantage 122 * of the information that we already have from our previous lookup, 123 * although it is possible that the "hints" in the vm_map will help here. 124 * 125 * alternative 2: the system already keeps track of a "version" number of 126 * a map. [i.e. every time you write-lock a map (e.g. to change a 127 * mapping) you bump the version number up by one...] so, we can save 128 * the version number of the map before we release the lock and start I/O. 129 * then when I/O is done we can relock and check the version numbers 130 * to see if anything changed. this might save us some over 1 because 131 * we don't have to unbusy the page and may be less compares(?). 132 * 133 * alternative 3: put in backpointers or a way to "hold" part of a map 134 * in place while I/O is in progress. this could be complex to 135 * implement (especially with structures like amap that can be referenced 136 * by multiple map entries, and figuring out what should wait could be 137 * complex as well...). 138 * 139 * we use alternative 2. given that we are multi-threaded now we may want 140 * to reconsider the choice. 141 */ 142 143 /* 144 * local data structures 145 */ 146 struct uvm_advice { 147 int nback; 148 int nforw; 149 }; 150 151 /* 152 * page range array: set up in uvmfault_init(). 153 */ 154 static struct uvm_advice uvmadvice[MADV_MASK + 1]; 155 156 #define UVM_MAXRANGE 16 /* must be max() of nback+nforw+1 */ 157 158 /* 159 * private prototypes 160 */ 161 static void uvmfault_amapcopy(struct uvm_faultinfo *); 162 static inline void uvmfault_anonflush(struct vm_anon **, int); 163 void uvmfault_unlockmaps(struct uvm_faultinfo *, boolean_t); 164 void uvmfault_update_stats(struct uvm_faultinfo *); 165 166 /* 167 * inline functions 168 */ 169 /* 170 * uvmfault_anonflush: try and deactivate pages in specified anons 171 * 172 * => does not have to deactivate page if it is busy 173 */ 174 static inline void 175 uvmfault_anonflush(struct vm_anon **anons, int n) 176 { 177 int lcv; 178 struct vm_page *pg; 179 180 for (lcv = 0; lcv < n; lcv++) { 181 if (anons[lcv] == NULL) 182 continue; 183 KASSERT(rw_lock_held(anons[lcv]->an_lock)); 184 pg = anons[lcv]->an_page; 185 if (pg && (pg->pg_flags & PG_BUSY) == 0) { 186 uvm_lock_pageq(); 187 if (pg->wire_count == 0) { 188 uvm_pagedeactivate(pg); 189 } 190 uvm_unlock_pageq(); 191 } 192 } 193 } 194 195 /* 196 * normal functions 197 */ 198 /* 199 * uvmfault_init: compute proper values for the uvmadvice[] array. 200 */ 201 void 202 uvmfault_init(void) 203 { 204 int npages; 205 206 npages = atop(16384); 207 if (npages > 0) { 208 KASSERT(npages <= UVM_MAXRANGE / 2); 209 uvmadvice[MADV_NORMAL].nforw = npages; 210 uvmadvice[MADV_NORMAL].nback = npages - 1; 211 } 212 213 npages = atop(32768); 214 if (npages > 0) { 215 KASSERT(npages <= UVM_MAXRANGE / 2); 216 uvmadvice[MADV_SEQUENTIAL].nforw = npages - 1; 217 uvmadvice[MADV_SEQUENTIAL].nback = npages; 218 } 219 } 220 221 /* 222 * uvmfault_amapcopy: clear "needs_copy" in a map. 223 * 224 * => called with VM data structures unlocked (usually, see below) 225 * => we get a write lock on the maps and clear needs_copy for a VA 226 * => if we are out of RAM we sleep (waiting for more) 227 */ 228 static void 229 uvmfault_amapcopy(struct uvm_faultinfo *ufi) 230 { 231 for (;;) { 232 /* 233 * no mapping? give up. 234 */ 235 if (uvmfault_lookup(ufi, TRUE) == FALSE) 236 return; 237 238 /* 239 * copy if needed. 240 */ 241 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) 242 amap_copy(ufi->map, ufi->entry, M_NOWAIT, 243 UVM_ET_ISSTACK(ufi->entry) ? FALSE : TRUE, 244 ufi->orig_rvaddr, ufi->orig_rvaddr + 1); 245 246 /* 247 * didn't work? must be out of RAM. unlock and sleep. 248 */ 249 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 250 uvmfault_unlockmaps(ufi, TRUE); 251 uvm_wait("fltamapcopy"); 252 continue; 253 } 254 255 /* 256 * got it! unlock and return. 257 */ 258 uvmfault_unlockmaps(ufi, TRUE); 259 return; 260 } 261 /*NOTREACHED*/ 262 } 263 264 /* 265 * uvmfault_anonget: get data in an anon into a non-busy, non-released 266 * page in that anon. 267 * 268 * => Map, amap and thus anon should be locked by caller. 269 * => If we fail, we unlock everything and error is returned. 270 * => If we are successful, return with everything still locked. 271 * => We do not move the page on the queues [gets moved later]. If we 272 * allocate a new page [we_own], it gets put on the queues. Either way, 273 * the result is that the page is on the queues at return time 274 */ 275 int 276 uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap, 277 struct vm_anon *anon) 278 { 279 struct vm_page *pg; 280 int error; 281 282 KASSERT(rw_lock_held(anon->an_lock)); 283 KASSERT(anon->an_lock == amap->am_lock); 284 285 /* Increment the counters.*/ 286 counters_inc(uvmexp_counters, flt_anget); 287 if (anon->an_page) { 288 curproc->p_ru.ru_minflt++; 289 } else { 290 curproc->p_ru.ru_majflt++; 291 } 292 error = 0; 293 294 /* 295 * Loop until we get the anon data, or fail. 296 */ 297 for (;;) { 298 boolean_t we_own, locked; 299 /* 300 * Note: 'we_own' will become true if we set PG_BUSY on a page. 301 */ 302 we_own = FALSE; 303 pg = anon->an_page; 304 305 /* 306 * Is page resident? Make sure it is not busy/released. 307 */ 308 if (pg) { 309 KASSERT(pg->pg_flags & PQ_ANON); 310 KASSERT(pg->uanon == anon); 311 312 /* 313 * if the page is busy, we drop all the locks and 314 * try again. 315 */ 316 if ((pg->pg_flags & (PG_BUSY|PG_RELEASED)) == 0) 317 return 0; 318 counters_inc(uvmexp_counters, flt_pgwait); 319 320 /* 321 * The last unlock must be an atomic unlock and wait 322 * on the owner of page. 323 */ 324 if (pg->uobject) { 325 /* Owner of page is UVM object. */ 326 uvmfault_unlockall(ufi, amap, NULL); 327 uvm_pagewait(pg, pg->uobject->vmobjlock, 328 "anonget1"); 329 } else { 330 /* Owner of page is anon. */ 331 uvmfault_unlockall(ufi, NULL, NULL); 332 uvm_pagewait(pg, anon->an_lock, "anonget2"); 333 } 334 } else { 335 /* 336 * No page, therefore allocate one. 337 */ 338 pg = uvm_pagealloc(NULL, 0, anon, 0); 339 if (pg == NULL) { 340 /* Out of memory. Wait a little. */ 341 uvmfault_unlockall(ufi, amap, NULL); 342 counters_inc(uvmexp_counters, flt_noram); 343 uvm_wait("flt_noram1"); 344 } else { 345 /* PG_BUSY bit is set. */ 346 we_own = TRUE; 347 uvmfault_unlockall(ufi, amap, NULL); 348 349 /* 350 * Pass a PG_BUSY+PG_FAKE+PG_CLEAN page into 351 * the uvm_swap_get() function with all data 352 * structures unlocked. Note that it is OK 353 * to read an_swslot here, because we hold 354 * PG_BUSY on the page. 355 */ 356 counters_inc(uvmexp_counters, pageins); 357 error = uvm_swap_get(pg, anon->an_swslot, 358 PGO_SYNCIO); 359 360 /* 361 * We clean up after the I/O below in the 362 * 'we_own' case. 363 */ 364 } 365 } 366 367 /* 368 * Re-lock the map and anon. 369 */ 370 locked = uvmfault_relock(ufi); 371 if (locked || we_own) { 372 rw_enter(anon->an_lock, RW_WRITE); 373 } 374 375 /* 376 * If we own the page (i.e. we set PG_BUSY), then we need 377 * to clean up after the I/O. There are three cases to 378 * consider: 379 * 380 * 1) Page was released during I/O: free anon and ReFault. 381 * 2) I/O not OK. Free the page and cause the fault to fail. 382 * 3) I/O OK! Activate the page and sync with the non-we_own 383 * case (i.e. drop anon lock if not locked). 384 */ 385 if (we_own) { 386 if (pg->pg_flags & PG_WANTED) { 387 wakeup(pg); 388 } 389 390 /* 391 * if we were RELEASED during I/O, then our anon is 392 * no longer part of an amap. we need to free the 393 * anon and try again. 394 */ 395 if (pg->pg_flags & PG_RELEASED) { 396 KASSERT(anon->an_ref == 0); 397 /* 398 * Released while we had unlocked amap. 399 */ 400 if (locked) 401 uvmfault_unlockall(ufi, NULL, NULL); 402 uvm_anon_release(anon); /* frees page for us */ 403 counters_inc(uvmexp_counters, flt_pgrele); 404 return ERESTART; /* refault! */ 405 } 406 407 if (error != VM_PAGER_OK) { 408 KASSERT(error != VM_PAGER_PEND); 409 410 /* remove page from anon */ 411 anon->an_page = NULL; 412 413 /* 414 * Remove the swap slot from the anon and 415 * mark the anon as having no real slot. 416 * Do not free the swap slot, thus preventing 417 * it from being used again. 418 */ 419 uvm_swap_markbad(anon->an_swslot, 1); 420 anon->an_swslot = SWSLOT_BAD; 421 422 /* 423 * Note: page was never !PG_BUSY, so it 424 * cannot be mapped and thus no need to 425 * pmap_page_protect() it. 426 */ 427 uvm_lock_pageq(); 428 uvm_pagefree(pg); 429 uvm_unlock_pageq(); 430 431 if (locked) { 432 uvmfault_unlockall(ufi, NULL, NULL); 433 } 434 rw_exit(anon->an_lock); 435 /* 436 * An error occurred while trying to bring 437 * in the page -- this is the only error we 438 * return right now. 439 */ 440 return EACCES; /* XXX */ 441 } 442 443 /* 444 * We have successfully read the page, activate it. 445 */ 446 pmap_clear_modify(pg); 447 uvm_lock_pageq(); 448 uvm_pageactivate(pg); 449 uvm_unlock_pageq(); 450 atomic_clearbits_int(&pg->pg_flags, 451 PG_WANTED|PG_BUSY|PG_FAKE); 452 UVM_PAGE_OWN(pg, NULL); 453 } 454 455 /* 456 * We were not able to re-lock the map - restart the fault. 457 */ 458 if (!locked) { 459 if (we_own) { 460 rw_exit(anon->an_lock); 461 } 462 return ERESTART; 463 } 464 465 /* 466 * Verify that no one has touched the amap and moved 467 * the anon on us. 468 */ 469 if (ufi != NULL && amap_lookup(&ufi->entry->aref, 470 ufi->orig_rvaddr - ufi->entry->start) != anon) { 471 472 uvmfault_unlockall(ufi, amap, NULL); 473 return ERESTART; 474 } 475 476 /* 477 * Retry.. 478 */ 479 counters_inc(uvmexp_counters, flt_anretry); 480 continue; 481 482 } 483 /*NOTREACHED*/ 484 } 485 486 /* 487 * uvmfault_promote: promote data to a new anon. used for 1B and 2B. 488 * 489 * 1. allocate an anon and a page. 490 * 2. fill its contents. 491 * 492 * => if we fail (result != 0) we unlock everything. 493 * => on success, return a new locked anon via 'nanon'. 494 * => it's caller's responsibility to put the promoted nanon->an_page to the 495 * page queue. 496 */ 497 int 498 uvmfault_promote(struct uvm_faultinfo *ufi, 499 struct vm_page *uobjpage, 500 struct vm_anon **nanon, /* OUT: allocated anon */ 501 struct vm_page **npg) 502 { 503 struct vm_amap *amap = ufi->entry->aref.ar_amap; 504 struct uvm_object *uobj = NULL; 505 struct vm_anon *anon; 506 struct vm_page *pg = NULL; 507 508 if (uobjpage != PGO_DONTCARE) 509 uobj = uobjpage->uobject; 510 511 KASSERT(uobj == NULL || rw_lock_held(uobj->vmobjlock)); 512 513 anon = uvm_analloc(); 514 if (anon) { 515 anon->an_lock = amap->am_lock; 516 pg = uvm_pagealloc(NULL, 0, anon, 517 (uobjpage == PGO_DONTCARE) ? UVM_PGA_ZERO : 0); 518 } 519 520 /* check for out of RAM */ 521 if (anon == NULL || pg == NULL) { 522 uvmfault_unlockall(ufi, amap, uobj); 523 if (anon == NULL) 524 counters_inc(uvmexp_counters, flt_noanon); 525 else { 526 anon->an_lock = NULL; 527 anon->an_ref--; 528 uvm_anfree(anon); 529 counters_inc(uvmexp_counters, flt_noram); 530 } 531 532 if (uvm_swapisfull()) 533 return ENOMEM; 534 535 /* out of RAM, wait for more */ 536 if (anon == NULL) 537 uvm_anwait(); 538 else 539 uvm_wait("flt_noram3"); 540 return ERESTART; 541 } 542 543 /* 544 * copy the page [pg now dirty] 545 */ 546 if (uobjpage != PGO_DONTCARE) 547 uvm_pagecopy(uobjpage, pg); 548 549 *nanon = anon; 550 *npg = pg; 551 return 0; 552 } 553 554 /* 555 * Update statistics after fault resolution. 556 * - maxrss 557 */ 558 void 559 uvmfault_update_stats(struct uvm_faultinfo *ufi) 560 { 561 struct vm_map *map; 562 struct proc *p; 563 vsize_t res; 564 565 map = ufi->orig_map; 566 567 /* 568 * If this is a nested pmap (eg, a virtual machine pmap managed 569 * by vmm(4) on amd64/i386), don't do any updating, just return. 570 * 571 * pmap_nested() on other archs is #defined to 0, so this is a 572 * no-op. 573 */ 574 if (pmap_nested(map->pmap)) 575 return; 576 577 /* Update the maxrss for the process. */ 578 if (map->flags & VM_MAP_ISVMSPACE) { 579 p = curproc; 580 KASSERT(p != NULL && &p->p_vmspace->vm_map == map); 581 582 res = pmap_resident_count(map->pmap); 583 /* Convert res from pages to kilobytes. */ 584 res <<= (PAGE_SHIFT - 10); 585 586 if (p->p_ru.ru_maxrss < res) 587 p->p_ru.ru_maxrss = res; 588 } 589 } 590 591 /* 592 * F A U L T - m a i n e n t r y p o i n t 593 */ 594 595 /* 596 * uvm_fault: page fault handler 597 * 598 * => called from MD code to resolve a page fault 599 * => VM data structures usually should be unlocked. however, it is 600 * possible to call here with the main map locked if the caller 601 * gets a write lock, sets it recursive, and then calls us (c.f. 602 * uvm_map_pageable). this should be avoided because it keeps 603 * the map locked off during I/O. 604 * => MUST NEVER BE CALLED IN INTERRUPT CONTEXT 605 */ 606 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 607 ~PROT_WRITE : PROT_MASK) 608 struct uvm_faultctx { 609 /* 610 * the following members are set up by uvm_fault_check() and 611 * read-only after that. 612 */ 613 vm_prot_t enter_prot; 614 vm_prot_t access_type; 615 vaddr_t startva; 616 int npages; 617 int centeridx; 618 boolean_t narrow; 619 boolean_t wired; 620 paddr_t pa_flags; 621 boolean_t promote; 622 int lower_lock_type; 623 }; 624 625 int uvm_fault_check( 626 struct uvm_faultinfo *, struct uvm_faultctx *, 627 struct vm_anon ***, vm_fault_t); 628 629 int uvm_fault_upper( 630 struct uvm_faultinfo *, struct uvm_faultctx *, 631 struct vm_anon **); 632 boolean_t uvm_fault_upper_lookup( 633 struct uvm_faultinfo *, const struct uvm_faultctx *, 634 struct vm_anon **, struct vm_page **); 635 636 int uvm_fault_lower( 637 struct uvm_faultinfo *, struct uvm_faultctx *, 638 struct vm_page **); 639 int uvm_fault_lower_io( 640 struct uvm_faultinfo *, struct uvm_faultctx *, 641 struct uvm_object **, struct vm_page **); 642 643 int 644 uvm_fault(vm_map_t orig_map, vaddr_t vaddr, vm_fault_t fault_type, 645 vm_prot_t access_type) 646 { 647 struct uvm_faultinfo ufi; 648 struct uvm_faultctx flt; 649 boolean_t shadowed; 650 struct vm_anon *anons_store[UVM_MAXRANGE], **anons; 651 struct vm_page *pages[UVM_MAXRANGE]; 652 int error; 653 654 counters_inc(uvmexp_counters, faults); 655 TRACEPOINT(uvm, fault, vaddr, fault_type, access_type, NULL); 656 657 /* 658 * init the IN parameters in the ufi 659 */ 660 ufi.orig_map = orig_map; 661 ufi.orig_rvaddr = trunc_page(vaddr); 662 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ 663 flt.access_type = access_type; 664 flt.narrow = FALSE; /* assume normal fault for now */ 665 flt.wired = FALSE; /* assume non-wired fault for now */ 666 flt.lower_lock_type = RW_WRITE; /* exclusive lock for now */ 667 668 error = ERESTART; 669 while (error == ERESTART) { /* ReFault: */ 670 anons = anons_store; 671 672 error = uvm_fault_check(&ufi, &flt, &anons, fault_type); 673 if (error != 0) 674 continue; 675 676 /* True if there is an anon at the faulting address */ 677 shadowed = uvm_fault_upper_lookup(&ufi, &flt, anons, pages); 678 if (shadowed == TRUE) { 679 /* case 1: fault on an anon in our amap */ 680 error = uvm_fault_upper(&ufi, &flt, anons); 681 } else { 682 struct uvm_object *uobj = ufi.entry->object.uvm_obj; 683 684 /* 685 * if the desired page is not shadowed by the amap and 686 * we have a backing object, then we check to see if 687 * the backing object would prefer to handle the fault 688 * itself (rather than letting us do it with the usual 689 * pgo_get hook). the backing object signals this by 690 * providing a pgo_fault routine. 691 */ 692 if (uobj != NULL && uobj->pgops->pgo_fault != NULL) { 693 rw_enter(uobj->vmobjlock, RW_WRITE); 694 KERNEL_LOCK(); 695 error = uobj->pgops->pgo_fault(&ufi, 696 flt.startva, pages, flt.npages, 697 flt.centeridx, fault_type, flt.access_type, 698 PGO_LOCKED); 699 KERNEL_UNLOCK(); 700 } else { 701 /* case 2: fault on backing obj or zero fill */ 702 error = uvm_fault_lower(&ufi, &flt, pages); 703 } 704 } 705 } 706 707 return error; 708 } 709 710 /* 711 * uvm_fault_check: check prot, handle needs-copy, etc. 712 * 713 * 1. lookup entry. 714 * 2. check protection. 715 * 3. adjust fault condition (mainly for simulated fault). 716 * 4. handle needs-copy (lazy amap copy). 717 * 5. establish range of interest for neighbor fault (aka pre-fault). 718 * 6. look up anons (if amap exists). 719 * 7. flush pages (if MADV_SEQUENTIAL) 720 * 721 * => called with nothing locked. 722 * => if we fail (result != 0) we unlock everything. 723 * => initialize/adjust many members of flt. 724 */ 725 int 726 uvm_fault_check(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 727 struct vm_anon ***ranons, vm_fault_t fault_type) 728 { 729 struct vm_amap *amap; 730 struct uvm_object *uobj; 731 int nback, nforw; 732 733 /* 734 * lookup and lock the maps 735 */ 736 if (uvmfault_lookup(ufi, FALSE) == FALSE) { 737 return EFAULT; 738 } 739 /* locked: maps(read) */ 740 741 #ifdef DIAGNOSTIC 742 if ((ufi->map->flags & VM_MAP_PAGEABLE) == 0) 743 panic("uvm_fault: fault on non-pageable map (%p, 0x%lx)", 744 ufi->map, ufi->orig_rvaddr); 745 #endif 746 747 /* 748 * check protection 749 */ 750 if ((ufi->entry->protection & flt->access_type) != flt->access_type) { 751 uvmfault_unlockmaps(ufi, FALSE); 752 return EACCES; 753 } 754 755 /* 756 * "enter_prot" is the protection we want to enter the page in at. 757 * for certain pages (e.g. copy-on-write pages) this protection can 758 * be more strict than ufi->entry->protection. "wired" means either 759 * the entry is wired or we are fault-wiring the pg. 760 */ 761 flt->enter_prot = ufi->entry->protection; 762 flt->pa_flags = UVM_ET_ISWC(ufi->entry) ? PMAP_WC : 0; 763 if (VM_MAPENT_ISWIRED(ufi->entry) || (fault_type == VM_FAULT_WIRE)) { 764 flt->wired = TRUE; 765 flt->access_type = flt->enter_prot; /* full access for wired */ 766 /* don't look for neighborhood * pages on "wire" fault */ 767 flt->narrow = TRUE; 768 } 769 770 /* handle "needs_copy" case. */ 771 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 772 if ((flt->access_type & PROT_WRITE) || 773 (ufi->entry->object.uvm_obj == NULL)) { 774 /* need to clear */ 775 uvmfault_unlockmaps(ufi, FALSE); 776 uvmfault_amapcopy(ufi); 777 counters_inc(uvmexp_counters, flt_amcopy); 778 return ERESTART; 779 } else { 780 /* 781 * ensure that we pmap_enter page R/O since 782 * needs_copy is still true 783 */ 784 flt->enter_prot &= ~PROT_WRITE; 785 } 786 } 787 788 /* 789 * identify the players 790 */ 791 amap = ufi->entry->aref.ar_amap; /* upper layer */ 792 uobj = ufi->entry->object.uvm_obj; /* lower layer */ 793 794 /* 795 * check for a case 0 fault. if nothing backing the entry then 796 * error now. 797 */ 798 if (amap == NULL && uobj == NULL) { 799 uvmfault_unlockmaps(ufi, FALSE); 800 return EFAULT; 801 } 802 803 /* 804 * for a case 2B fault waste no time on adjacent pages because 805 * they are likely already entered. 806 */ 807 if (uobj != NULL && amap != NULL && 808 (flt->access_type & PROT_WRITE) != 0) { 809 /* wide fault (!narrow) */ 810 flt->narrow = TRUE; 811 } 812 813 /* 814 * establish range of interest based on advice from mapper 815 * and then clip to fit map entry. note that we only want 816 * to do this the first time through the fault. if we 817 * ReFault we will disable this by setting "narrow" to true. 818 */ 819 if (flt->narrow == FALSE) { 820 821 /* wide fault (!narrow) */ 822 nback = min(uvmadvice[ufi->entry->advice].nback, 823 (ufi->orig_rvaddr - ufi->entry->start) >> PAGE_SHIFT); 824 flt->startva = ufi->orig_rvaddr - ((vsize_t)nback << PAGE_SHIFT); 825 nforw = min(uvmadvice[ufi->entry->advice].nforw, 826 ((ufi->entry->end - ufi->orig_rvaddr) >> PAGE_SHIFT) - 1); 827 /* 828 * note: "-1" because we don't want to count the 829 * faulting page as forw 830 */ 831 flt->npages = nback + nforw + 1; 832 flt->centeridx = nback; 833 834 flt->narrow = TRUE; /* ensure only once per-fault */ 835 } else { 836 /* narrow fault! */ 837 nback = nforw = 0; 838 flt->startva = ufi->orig_rvaddr; 839 flt->npages = 1; 840 flt->centeridx = 0; 841 } 842 843 /* 844 * if we've got an amap then lock it and extract current anons. 845 */ 846 if (amap) { 847 amap_lock(amap, RW_WRITE); 848 amap_lookups(&ufi->entry->aref, 849 flt->startva - ufi->entry->start, *ranons, flt->npages); 850 } else { 851 *ranons = NULL; /* to be safe */ 852 } 853 854 /* 855 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages 856 * now and then forget about them (for the rest of the fault). 857 */ 858 if (ufi->entry->advice == MADV_SEQUENTIAL && nback != 0) { 859 /* flush back-page anons? */ 860 if (amap) 861 uvmfault_anonflush(*ranons, nback); 862 863 /* 864 * flush object? 865 */ 866 if (uobj) { 867 voff_t uoff; 868 869 uoff = (flt->startva - ufi->entry->start) + ufi->entry->offset; 870 rw_enter(uobj->vmobjlock, RW_WRITE); 871 (void) uobj->pgops->pgo_flush(uobj, uoff, uoff + 872 ((vsize_t)nback << PAGE_SHIFT), PGO_DEACTIVATE); 873 rw_exit(uobj->vmobjlock); 874 } 875 876 /* now forget about the backpages */ 877 if (amap) 878 *ranons += nback; 879 flt->startva += ((vsize_t)nback << PAGE_SHIFT); 880 flt->npages -= nback; 881 flt->centeridx = 0; 882 } 883 884 return 0; 885 } 886 887 /* 888 * uvm_fault_upper_lookup: look up existing h/w mapping and amap. 889 * 890 * iterate range of interest: 891 * 1. check if h/w mapping exists. if yes, we don't care 892 * 2. check if anon exists. if not, page is lower. 893 * 3. if anon exists, enter h/w mapping for neighbors. 894 * 895 * => called with amap locked (if exists). 896 */ 897 boolean_t 898 uvm_fault_upper_lookup(struct uvm_faultinfo *ufi, 899 const struct uvm_faultctx *flt, struct vm_anon **anons, 900 struct vm_page **pages) 901 { 902 struct vm_amap *amap = ufi->entry->aref.ar_amap; 903 struct vm_anon *anon; 904 struct vm_page *pg; 905 boolean_t shadowed; 906 vaddr_t currva; 907 paddr_t pa; 908 int lcv, entered = 0; 909 910 /* locked: maps(read), amap(if there) */ 911 KASSERT(amap == NULL || 912 rw_write_held(amap->am_lock)); 913 914 /* 915 * map in the backpages and frontpages we found in the amap in hopes 916 * of preventing future faults. we also init the pages[] array as 917 * we go. 918 */ 919 currva = flt->startva; 920 shadowed = FALSE; 921 for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) { 922 /* 923 * unmapped or center page. check if any anon at this level. 924 */ 925 if (amap == NULL || anons[lcv] == NULL) { 926 pages[lcv] = NULL; 927 continue; 928 } 929 930 /* 931 * check for present page and map if possible. 932 */ 933 pages[lcv] = PGO_DONTCARE; 934 if (lcv == flt->centeridx) { /* save center for later! */ 935 shadowed = TRUE; 936 continue; 937 } 938 939 anon = anons[lcv]; 940 pg = anon->an_page; 941 942 KASSERT(anon->an_lock == amap->am_lock); 943 944 /* 945 * ignore busy pages. 946 * don't play with VAs that are already mapped. 947 */ 948 if (pg && (pg->pg_flags & (PG_RELEASED|PG_BUSY)) == 0 && 949 !pmap_extract(ufi->orig_map->pmap, currva, &pa)) { 950 uvm_lock_pageq(); 951 uvm_pageactivate(pg); /* reactivate */ 952 uvm_unlock_pageq(); 953 counters_inc(uvmexp_counters, flt_namap); 954 955 /* No fault-ahead when wired. */ 956 KASSERT(flt->wired == FALSE); 957 958 /* 959 * Since this isn't the page that's actually faulting, 960 * ignore pmap_enter() failures; it's not critical 961 * that we enter these right now. 962 */ 963 (void) pmap_enter(ufi->orig_map->pmap, currva, 964 VM_PAGE_TO_PHYS(pg) | flt->pa_flags, 965 (anon->an_ref > 1) ? 966 (flt->enter_prot & ~PROT_WRITE) : flt->enter_prot, 967 PMAP_CANFAIL); 968 entered++; 969 } 970 } 971 if (entered > 0) 972 pmap_update(ufi->orig_map->pmap); 973 974 return shadowed; 975 } 976 977 /* 978 * uvm_fault_upper: handle upper fault. 979 * 980 * 1. acquire anon lock. 981 * 2. get anon. let uvmfault_anonget do the dirty work. 982 * 3. if COW, promote data to new anon 983 * 4. enter h/w mapping 984 */ 985 int 986 uvm_fault_upper(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 987 struct vm_anon **anons) 988 { 989 struct vm_amap *amap = ufi->entry->aref.ar_amap; 990 struct vm_anon *oanon, *anon = anons[flt->centeridx]; 991 struct vm_page *pg = NULL; 992 int error, ret; 993 994 /* locked: maps(read), amap, anon */ 995 KASSERT(rw_write_held(amap->am_lock)); 996 KASSERT(anon->an_lock == amap->am_lock); 997 998 /* 999 * no matter if we have case 1A or case 1B we are going to need to 1000 * have the anon's memory resident. ensure that now. 1001 */ 1002 /* 1003 * let uvmfault_anonget do the dirty work. 1004 * if it fails (!OK) it will unlock everything for us. 1005 * if it succeeds, locks are still valid and locked. 1006 * also, if it is OK, then the anon's page is on the queues. 1007 * if the page is on loan from a uvm_object, then anonget will 1008 * lock that object for us if it does not fail. 1009 */ 1010 error = uvmfault_anonget(ufi, amap, anon); 1011 switch (error) { 1012 case 0: 1013 break; 1014 1015 case ERESTART: 1016 return ERESTART; 1017 1018 default: 1019 return error; 1020 } 1021 1022 KASSERT(rw_write_held(amap->am_lock)); 1023 KASSERT(anon->an_lock == amap->am_lock); 1024 1025 /* 1026 * if we are case 1B then we will need to allocate a new blank 1027 * anon to transfer the data into. note that we have a lock 1028 * on anon, so no one can busy or release the page until we are done. 1029 * also note that the ref count can't drop to zero here because 1030 * it is > 1 and we are only dropping one ref. 1031 * 1032 * in the (hopefully very rare) case that we are out of RAM we 1033 * will unlock, wait for more RAM, and refault. 1034 * 1035 * if we are out of anon VM we wait for RAM to become available. 1036 */ 1037 1038 if ((flt->access_type & PROT_WRITE) != 0 && anon->an_ref > 1) { 1039 /* promoting requires a write lock. */ 1040 KASSERT(rw_write_held(amap->am_lock)); 1041 1042 counters_inc(uvmexp_counters, flt_acow); 1043 oanon = anon; /* oanon = old */ 1044 1045 error = uvmfault_promote(ufi, oanon->an_page, &anon, &pg); 1046 if (error) 1047 return error; 1048 1049 /* un-busy! new page */ 1050 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE); 1051 UVM_PAGE_OWN(pg, NULL); 1052 ret = amap_add(&ufi->entry->aref, 1053 ufi->orig_rvaddr - ufi->entry->start, anon, 1); 1054 KASSERT(ret == 0); 1055 1056 KASSERT(anon->an_lock == oanon->an_lock); 1057 1058 /* deref: can not drop to zero here by defn! */ 1059 KASSERT(oanon->an_ref > 1); 1060 oanon->an_ref--; 1061 1062 #if defined(MULTIPROCESSOR) && !defined(__HAVE_PMAP_MPSAFE_ENTER_COW) 1063 /* 1064 * If there are multiple threads, either uvm or the 1065 * pmap has to make sure no threads see the old RO 1066 * mapping once any have seen the new RW mapping. 1067 * uvm does it by inserting the new mapping RO and 1068 * letting it fault again. 1069 * This is only a problem on MP systems. 1070 */ 1071 if (P_HASSIBLING(curproc)) { 1072 flt->enter_prot &= ~PROT_WRITE; 1073 flt->access_type &= ~PROT_WRITE; 1074 } 1075 #endif 1076 1077 /* 1078 * note: anon is _not_ locked, but we have the sole references 1079 * to in from amap. 1080 * thus, no one can get at it until we are done with it. 1081 */ 1082 } else { 1083 counters_inc(uvmexp_counters, flt_anon); 1084 oanon = anon; 1085 pg = anon->an_page; 1086 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */ 1087 flt->enter_prot = flt->enter_prot & ~PROT_WRITE; 1088 } 1089 1090 /* 1091 * now map the page in . 1092 */ 1093 if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr, 1094 VM_PAGE_TO_PHYS(pg) | flt->pa_flags, flt->enter_prot, 1095 flt->access_type | PMAP_CANFAIL | (flt->wired ? PMAP_WIRED : 0)) != 0) { 1096 /* 1097 * No need to undo what we did; we can simply think of 1098 * this as the pmap throwing away the mapping information. 1099 * 1100 * We do, however, have to go through the ReFault path, 1101 * as the map may change while we're asleep. 1102 */ 1103 uvmfault_unlockall(ufi, amap, NULL); 1104 if (uvm_swapisfull()) { 1105 /* XXX instrumentation */ 1106 return ENOMEM; 1107 } 1108 /* XXX instrumentation */ 1109 uvm_wait("flt_pmfail1"); 1110 return ERESTART; 1111 } 1112 1113 /* 1114 * ... update the page queues. 1115 */ 1116 uvm_lock_pageq(); 1117 if (flt->wired) { 1118 uvm_pagewire(pg); 1119 } else { 1120 uvm_pageactivate(pg); 1121 } 1122 uvm_unlock_pageq(); 1123 1124 if (flt->wired) { 1125 /* 1126 * since the now-wired page cannot be paged out, 1127 * release its swap resources for others to use. 1128 * since an anon with no swap cannot be PG_CLEAN, 1129 * clear its clean flag now. 1130 */ 1131 atomic_clearbits_int(&pg->pg_flags, PG_CLEAN); 1132 uvm_anon_dropswap(anon); 1133 } 1134 1135 /* 1136 * done case 1! finish up by unlocking everything and returning success 1137 */ 1138 uvmfault_unlockall(ufi, amap, NULL); 1139 pmap_update(ufi->orig_map->pmap); 1140 return 0; 1141 } 1142 1143 /* 1144 * uvm_fault_lower_lookup: look up on-memory uobj pages. 1145 * 1146 * 1. get on-memory pages. 1147 * 2. if failed, give up (get only center page later). 1148 * 3. if succeeded, enter h/w mapping of neighbor pages. 1149 */ 1150 1151 struct vm_page * 1152 uvm_fault_lower_lookup( 1153 struct uvm_faultinfo *ufi, const struct uvm_faultctx *flt, 1154 struct vm_page **pages) 1155 { 1156 struct uvm_object *uobj = ufi->entry->object.uvm_obj; 1157 struct vm_page *uobjpage = NULL; 1158 int lcv, gotpages, entered; 1159 vaddr_t currva; 1160 paddr_t pa; 1161 1162 rw_enter(uobj->vmobjlock, flt->lower_lock_type); 1163 1164 counters_inc(uvmexp_counters, flt_lget); 1165 gotpages = flt->npages; 1166 (void) uobj->pgops->pgo_get(uobj, 1167 ufi->entry->offset + (flt->startva - ufi->entry->start), 1168 pages, &gotpages, flt->centeridx, 1169 flt->access_type & MASK(ufi->entry), ufi->entry->advice, 1170 PGO_LOCKED); 1171 1172 /* 1173 * check for pages to map, if we got any 1174 */ 1175 if (gotpages == 0) { 1176 return NULL; 1177 } 1178 1179 entered = 0; 1180 currva = flt->startva; 1181 for (lcv = 0; lcv < flt->npages; lcv++, currva += PAGE_SIZE) { 1182 if (pages[lcv] == NULL || 1183 pages[lcv] == PGO_DONTCARE) 1184 continue; 1185 1186 KASSERT((pages[lcv]->pg_flags & PG_BUSY) == 0); 1187 KASSERT((pages[lcv]->pg_flags & PG_RELEASED) == 0); 1188 1189 /* 1190 * if center page is resident and not PG_BUSY, then pgo_get 1191 * gave us a handle to it. 1192 * remember this page as "uobjpage." (for later use). 1193 */ 1194 if (lcv == flt->centeridx) { 1195 uobjpage = pages[lcv]; 1196 continue; 1197 } 1198 1199 if (pmap_extract(ufi->orig_map->pmap, currva, &pa)) 1200 continue; 1201 1202 /* 1203 * calling pgo_get with PGO_LOCKED returns us pages which 1204 * are neither busy nor released, so we don't need to check 1205 * for this. we can just directly enter the pages. 1206 */ 1207 if (pages[lcv]->wire_count == 0) { 1208 uvm_lock_pageq(); 1209 uvm_pageactivate(pages[lcv]); 1210 uvm_unlock_pageq(); 1211 } 1212 counters_inc(uvmexp_counters, flt_nomap); 1213 1214 /* No fault-ahead when wired. */ 1215 KASSERT(flt->wired == FALSE); 1216 1217 /* 1218 * Since this page isn't the page that's actually faulting, 1219 * ignore pmap_enter() failures; it's not critical that we 1220 * enter these right now. 1221 * NOTE: page can't be PG_WANTED or PG_RELEASED because we've 1222 * held the lock the whole time we've had the handle. 1223 */ 1224 (void) pmap_enter(ufi->orig_map->pmap, currva, 1225 VM_PAGE_TO_PHYS(pages[lcv]) | flt->pa_flags, 1226 flt->enter_prot & MASK(ufi->entry), PMAP_CANFAIL); 1227 entered++; 1228 1229 } 1230 if (entered > 0) 1231 pmap_update(ufi->orig_map->pmap); 1232 1233 return uobjpage; 1234 } 1235 1236 /* 1237 * uvm_fault_lower: handle lower fault. 1238 * 1239 * 1. check uobj 1240 * 1.1. if null, ZFOD. 1241 * 1.2. if not null, look up unnmapped neighbor pages. 1242 * 2. for center page, check if promote. 1243 * 2.1. ZFOD always needs promotion. 1244 * 2.2. other uobjs, when entry is marked COW (usually MAP_PRIVATE vnode). 1245 * 3. if uobj is not ZFOD and page is not found, do i/o. 1246 * 4. dispatch either direct / promote fault. 1247 */ 1248 int 1249 uvm_fault_lower(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1250 struct vm_page **pages) 1251 { 1252 struct vm_amap *amap = ufi->entry->aref.ar_amap; 1253 struct uvm_object *uobj = ufi->entry->object.uvm_obj; 1254 int dropswap = 0; 1255 struct vm_page *uobjpage, *pg = NULL; 1256 struct vm_anon *anon = NULL; 1257 int error; 1258 1259 /* 1260 * now, if the desired page is not shadowed by the amap and we have 1261 * a backing object that does not have a special fault routine, then 1262 * we ask (with pgo_get) the object for resident pages that we care 1263 * about and attempt to map them in. we do not let pgo_get block 1264 * (PGO_LOCKED). 1265 */ 1266 if (uobj == NULL) { 1267 /* zero fill; don't care neighbor pages */ 1268 uobjpage = NULL; 1269 } else { 1270 uobjpage = uvm_fault_lower_lookup(ufi, flt, pages); 1271 } 1272 1273 /* 1274 * note that at this point we are done with any front or back pages. 1275 * we are now going to focus on the center page (i.e. the one we've 1276 * faulted on). if we have faulted on the bottom (uobj) 1277 * layer [i.e. case 2] and the page was both present and available, 1278 * then we've got a pointer to it as "uobjpage" and we've already 1279 * made it BUSY. 1280 */ 1281 1282 /* 1283 * locked: 1284 */ 1285 KASSERT(amap == NULL || 1286 rw_write_held(amap->am_lock)); 1287 KASSERT(uobj == NULL || 1288 rw_status(uobj->vmobjlock) == flt->lower_lock_type); 1289 1290 /* 1291 * note that uobjpage can not be PGO_DONTCARE at this point. we now 1292 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we 1293 * have a backing object, check and see if we are going to promote 1294 * the data up to an anon during the fault. 1295 */ 1296 if (uobj == NULL) { 1297 uobjpage = PGO_DONTCARE; 1298 flt->promote = TRUE; /* always need anon here */ 1299 } else { 1300 KASSERT(uobjpage != PGO_DONTCARE); 1301 flt->promote = (flt->access_type & PROT_WRITE) && 1302 UVM_ET_ISCOPYONWRITE(ufi->entry); 1303 } 1304 1305 /* 1306 * if uobjpage is not null then we do not need to do I/O to get the 1307 * uobjpage. 1308 * 1309 * if uobjpage is null, then we need to ask the pager to 1310 * get the data for us. once we have the data, we need to reverify 1311 * the state the world. we are currently not holding any resources. 1312 */ 1313 if (uobjpage) { 1314 /* update rusage counters */ 1315 curproc->p_ru.ru_minflt++; 1316 if (uobjpage != PGO_DONTCARE) { 1317 uvm_lock_pageq(); 1318 uvm_pageactivate(uobjpage); 1319 uvm_unlock_pageq(); 1320 } 1321 } else { 1322 error = uvm_fault_lower_io(ufi, flt, &uobj, &uobjpage); 1323 if (error != 0) 1324 return error; 1325 } 1326 1327 /* 1328 * notes: 1329 * - at this point uobjpage can not be NULL 1330 * - at this point uobjpage could be PG_WANTED (handle later) 1331 */ 1332 if (flt->promote == FALSE) { 1333 /* 1334 * we are not promoting. if the mapping is COW ensure that we 1335 * don't give more access than we should (e.g. when doing a read 1336 * fault on a COPYONWRITE mapping we want to map the COW page in 1337 * R/O even though the entry protection could be R/W). 1338 * 1339 * set "pg" to the page we want to map in (uobjpage, usually) 1340 */ 1341 counters_inc(uvmexp_counters, flt_obj); 1342 if (UVM_ET_ISCOPYONWRITE(ufi->entry)) 1343 flt->enter_prot &= ~PROT_WRITE; 1344 pg = uobjpage; /* map in the actual object */ 1345 1346 /* assert(uobjpage != PGO_DONTCARE) */ 1347 1348 /* 1349 * we are faulting directly on the page. 1350 */ 1351 } else { 1352 KASSERT(amap != NULL); 1353 1354 /* promoting requires a write lock. */ 1355 KASSERT(rw_write_held(amap->am_lock)); 1356 KASSERT(uobj == NULL || 1357 rw_status(uobj->vmobjlock) == flt->lower_lock_type); 1358 1359 /* 1360 * if we are going to promote the data to an anon we 1361 * allocate a blank anon here and plug it into our amap. 1362 */ 1363 error = uvmfault_promote(ufi, uobjpage, &anon, &pg); 1364 if (error) 1365 return error; 1366 1367 /* 1368 * fill in the data 1369 */ 1370 if (uobjpage != PGO_DONTCARE) { 1371 counters_inc(uvmexp_counters, flt_prcopy); 1372 1373 /* 1374 * promote to shared amap? make sure all sharing 1375 * procs see it 1376 */ 1377 if ((amap_flags(amap) & AMAP_SHARED) != 0) { 1378 pmap_page_protect(uobjpage, PROT_NONE); 1379 } 1380 #if defined(MULTIPROCESSOR) && !defined(__HAVE_PMAP_MPSAFE_ENTER_COW) 1381 /* 1382 * Otherwise: 1383 * If there are multiple threads, either uvm or the 1384 * pmap has to make sure no threads see the old RO 1385 * mapping once any have seen the new RW mapping. 1386 * uvm does it here by forcing it to PROT_NONE before 1387 * inserting the new mapping. 1388 */ 1389 else if (P_HASSIBLING(curproc)) { 1390 pmap_page_protect(uobjpage, PROT_NONE); 1391 } 1392 #endif 1393 /* done with copied uobjpage. */ 1394 rw_exit(uobj->vmobjlock); 1395 uobj = NULL; 1396 } else { 1397 counters_inc(uvmexp_counters, flt_przero); 1398 /* 1399 * Page is zero'd and marked dirty by uvm_pagealloc(), 1400 * called in uvmfault_promote() above. 1401 */ 1402 } 1403 1404 if (amap_add(&ufi->entry->aref, 1405 ufi->orig_rvaddr - ufi->entry->start, anon, 0)) { 1406 if (pg->pg_flags & PG_WANTED) 1407 wakeup(pg); 1408 1409 atomic_clearbits_int(&pg->pg_flags, 1410 PG_BUSY|PG_FAKE|PG_WANTED); 1411 UVM_PAGE_OWN(pg, NULL); 1412 uvmfault_unlockall(ufi, amap, uobj); 1413 uvm_anfree(anon); 1414 counters_inc(uvmexp_counters, flt_noamap); 1415 1416 if (uvm_swapisfull()) 1417 return (ENOMEM); 1418 1419 amap_populate(&ufi->entry->aref, 1420 ufi->orig_rvaddr - ufi->entry->start); 1421 return ERESTART; 1422 } 1423 } 1424 1425 /* 1426 * anon must be write locked (promotion). uobj can be either. 1427 * 1428 * Note: pg is either the uobjpage or the new page in the new anon. 1429 */ 1430 KASSERT(amap == NULL || 1431 rw_write_held(amap->am_lock)); 1432 KASSERT(uobj == NULL || 1433 rw_status(uobj->vmobjlock) == flt->lower_lock_type); 1434 KASSERT(anon == NULL || anon->an_lock == amap->am_lock); 1435 1436 /* 1437 * all resources are present. we can now map it in and free our 1438 * resources. 1439 */ 1440 if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr, 1441 VM_PAGE_TO_PHYS(pg) | flt->pa_flags, flt->enter_prot, 1442 flt->access_type | PMAP_CANFAIL | (flt->wired ? PMAP_WIRED : 0)) != 0) { 1443 /* 1444 * No need to undo what we did; we can simply think of 1445 * this as the pmap throwing away the mapping information. 1446 * 1447 * We do, however, have to go through the ReFault path, 1448 * as the map may change while we're asleep. 1449 */ 1450 if (pg->pg_flags & PG_WANTED) 1451 wakeup(pg); 1452 1453 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED); 1454 UVM_PAGE_OWN(pg, NULL); 1455 uvmfault_unlockall(ufi, amap, uobj); 1456 if (uvm_swapisfull()) { 1457 /* XXX instrumentation */ 1458 return (ENOMEM); 1459 } 1460 /* XXX instrumentation */ 1461 uvm_wait("flt_pmfail2"); 1462 return ERESTART; 1463 } 1464 1465 uvm_lock_pageq(); 1466 if (flt->wired) { 1467 uvm_pagewire(pg); 1468 if (pg->pg_flags & PQ_AOBJ) { 1469 /* 1470 * since the now-wired page cannot be paged out, 1471 * release its swap resources for others to use. 1472 * since an aobj page with no swap cannot be clean, 1473 * mark it dirty now. 1474 * 1475 * use pg->uobject here. if the page is from a 1476 * tmpfs vnode, the pages are backed by its UAO and 1477 * not the vnode. 1478 */ 1479 KASSERT(uobj != NULL); 1480 KASSERT(uobj->vmobjlock == pg->uobject->vmobjlock); 1481 atomic_clearbits_int(&pg->pg_flags, PG_CLEAN); 1482 dropswap = 1; 1483 } 1484 } else { 1485 uvm_pageactivate(pg); 1486 } 1487 uvm_unlock_pageq(); 1488 1489 if (dropswap) 1490 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT); 1491 1492 if (pg->pg_flags & PG_WANTED) 1493 wakeup(pg); 1494 1495 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED); 1496 UVM_PAGE_OWN(pg, NULL); 1497 uvmfault_unlockall(ufi, amap, uobj); 1498 pmap_update(ufi->orig_map->pmap); 1499 1500 return (0); 1501 } 1502 1503 /* 1504 * uvm_fault_lower_io: get lower page from backing store. 1505 * 1506 * 1. unlock everything, because i/o will block. 1507 * 2. call pgo_get. 1508 * 3. if failed, recover. 1509 * 4. if succeeded, relock everything and verify things. 1510 */ 1511 int 1512 uvm_fault_lower_io( 1513 struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 1514 struct uvm_object **ruobj, struct vm_page **ruobjpage) 1515 { 1516 struct vm_amap * const amap = ufi->entry->aref.ar_amap; 1517 struct uvm_object *uobj = *ruobj; 1518 struct vm_page *pg; 1519 boolean_t locked; 1520 int gotpages, advice; 1521 int result; 1522 voff_t uoff; 1523 vm_prot_t access_type; 1524 1525 /* grab everything we need from the entry before we unlock */ 1526 uoff = (ufi->orig_rvaddr - ufi->entry->start) + ufi->entry->offset; 1527 access_type = flt->access_type & MASK(ufi->entry); 1528 advice = ufi->entry->advice; 1529 1530 uvmfault_unlockall(ufi, amap, NULL); 1531 1532 /* update rusage counters */ 1533 curproc->p_ru.ru_majflt++; 1534 1535 KASSERT(rw_write_held(uobj->vmobjlock)); 1536 1537 counters_inc(uvmexp_counters, flt_get); 1538 gotpages = 1; 1539 pg = NULL; 1540 result = uobj->pgops->pgo_get(uobj, uoff, &pg, &gotpages, 1541 0, access_type, advice, PGO_SYNCIO); 1542 1543 /* 1544 * recover from I/O 1545 */ 1546 if (result != VM_PAGER_OK) { 1547 KASSERT(result != VM_PAGER_PEND); 1548 1549 if (result == VM_PAGER_AGAIN) { 1550 tsleep_nsec(&nowake, PVM, "fltagain2", MSEC_TO_NSEC(5)); 1551 return ERESTART; 1552 } 1553 1554 if (!UVM_ET_ISNOFAULT(ufi->entry)) 1555 return (EIO); 1556 1557 pg = PGO_DONTCARE; 1558 uobj = NULL; 1559 flt->promote = TRUE; 1560 } 1561 1562 /* re-verify the state of the world. */ 1563 locked = uvmfault_relock(ufi); 1564 if (locked && amap != NULL) 1565 amap_lock(amap, RW_WRITE); 1566 1567 /* might be changed */ 1568 if (pg != PGO_DONTCARE) { 1569 uobj = pg->uobject; 1570 rw_enter(uobj->vmobjlock, flt->lower_lock_type); 1571 KASSERT((pg->pg_flags & PG_BUSY) != 0); 1572 KASSERT(flt->lower_lock_type == RW_WRITE); 1573 } 1574 1575 /* 1576 * Re-verify that amap slot is still free. if there is 1577 * a problem, we clean up. 1578 */ 1579 if (locked && amap && amap_lookup(&ufi->entry->aref, 1580 ufi->orig_rvaddr - ufi->entry->start)) { 1581 if (locked) 1582 uvmfault_unlockall(ufi, amap, NULL); 1583 locked = FALSE; 1584 } 1585 1586 /* release the page now, still holding object lock */ 1587 if (pg != PGO_DONTCARE) { 1588 uvm_lock_pageq(); 1589 uvm_pageactivate(pg); 1590 uvm_unlock_pageq(); 1591 1592 if (pg->pg_flags & PG_WANTED) 1593 wakeup(pg); 1594 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_WANTED); 1595 UVM_PAGE_OWN(pg, NULL); 1596 } 1597 1598 if (locked == FALSE) { 1599 if (pg != PGO_DONTCARE) 1600 rw_exit(uobj->vmobjlock); 1601 return ERESTART; 1602 } 1603 1604 /* 1605 * we have the data in pg. we are holding object lock (so the page 1606 * can't be released on us). 1607 */ 1608 *ruobj = uobj; 1609 *ruobjpage = pg; 1610 return 0; 1611 } 1612 1613 /* 1614 * uvm_fault_wire: wire down a range of virtual addresses in a map. 1615 * 1616 * => map may be read-locked by caller, but MUST NOT be write-locked. 1617 * => if map is read-locked, any operations which may cause map to 1618 * be write-locked in uvm_fault() must be taken care of by 1619 * the caller. See uvm_map_pageable(). 1620 */ 1621 int 1622 uvm_fault_wire(vm_map_t map, vaddr_t start, vaddr_t end, vm_prot_t access_type) 1623 { 1624 vaddr_t va; 1625 int rv; 1626 1627 /* 1628 * now fault it in a page at a time. if the fault fails then we have 1629 * to undo what we have done. note that in uvm_fault PROT_NONE 1630 * is replaced with the max protection if fault_type is VM_FAULT_WIRE. 1631 */ 1632 for (va = start ; va < end ; va += PAGE_SIZE) { 1633 rv = uvm_fault(map, va, VM_FAULT_WIRE, access_type); 1634 if (rv) { 1635 if (va != start) { 1636 uvm_fault_unwire(map, start, va); 1637 } 1638 return (rv); 1639 } 1640 } 1641 1642 return (0); 1643 } 1644 1645 /* 1646 * uvm_fault_unwire(): unwire range of virtual space. 1647 */ 1648 void 1649 uvm_fault_unwire(vm_map_t map, vaddr_t start, vaddr_t end) 1650 { 1651 1652 vm_map_lock_read(map); 1653 uvm_fault_unwire_locked(map, start, end); 1654 vm_map_unlock_read(map); 1655 } 1656 1657 /* 1658 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire(). 1659 * 1660 * => map must be at least read-locked. 1661 */ 1662 void 1663 uvm_fault_unwire_locked(vm_map_t map, vaddr_t start, vaddr_t end) 1664 { 1665 vm_map_entry_t entry, oentry = NULL, next; 1666 pmap_t pmap = vm_map_pmap(map); 1667 vaddr_t va; 1668 paddr_t pa; 1669 struct vm_page *pg; 1670 1671 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0); 1672 vm_map_assert_anylock(map); 1673 1674 /* 1675 * we assume that the area we are unwiring has actually been wired 1676 * in the first place. this means that we should be able to extract 1677 * the PAs from the pmap. 1678 */ 1679 1680 /* 1681 * find the beginning map entry for the region. 1682 */ 1683 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map)); 1684 if (uvm_map_lookup_entry(map, start, &entry) == FALSE) 1685 panic("uvm_fault_unwire_locked: address not in map"); 1686 1687 for (va = start; va < end ; va += PAGE_SIZE) { 1688 /* 1689 * find the map entry for the current address. 1690 */ 1691 KASSERT(va >= entry->start); 1692 while (entry && va >= entry->end) { 1693 next = RBT_NEXT(uvm_map_addr, entry); 1694 entry = next; 1695 } 1696 1697 if (entry == NULL) 1698 return; 1699 if (va < entry->start) 1700 continue; 1701 1702 /* 1703 * lock it. 1704 */ 1705 if (entry != oentry) { 1706 if (oentry != NULL) { 1707 uvm_map_unlock_entry(oentry); 1708 } 1709 uvm_map_lock_entry(entry); 1710 oentry = entry; 1711 } 1712 1713 if (!pmap_extract(pmap, va, &pa)) 1714 continue; 1715 1716 /* 1717 * if the entry is no longer wired, tell the pmap. 1718 */ 1719 if (VM_MAPENT_ISWIRED(entry) == 0) 1720 pmap_unwire(pmap, va); 1721 1722 pg = PHYS_TO_VM_PAGE(pa); 1723 if (pg) { 1724 uvm_lock_pageq(); 1725 uvm_pageunwire(pg); 1726 uvm_unlock_pageq(); 1727 } 1728 } 1729 1730 if (oentry != NULL) { 1731 uvm_map_unlock_entry(oentry); 1732 } 1733 } 1734 1735 /* 1736 * uvmfault_unlockmaps: unlock the maps 1737 */ 1738 void 1739 uvmfault_unlockmaps(struct uvm_faultinfo *ufi, boolean_t write_locked) 1740 { 1741 /* 1742 * ufi can be NULL when this isn't really a fault, 1743 * but merely paging in anon data. 1744 */ 1745 if (ufi == NULL) { 1746 return; 1747 } 1748 1749 uvmfault_update_stats(ufi); 1750 if (write_locked) { 1751 vm_map_unlock(ufi->map); 1752 } else { 1753 vm_map_unlock_read(ufi->map); 1754 } 1755 } 1756 1757 /* 1758 * uvmfault_unlockall: unlock everything passed in. 1759 * 1760 * => maps must be read-locked (not write-locked). 1761 */ 1762 void 1763 uvmfault_unlockall(struct uvm_faultinfo *ufi, struct vm_amap *amap, 1764 struct uvm_object *uobj) 1765 { 1766 if (uobj) 1767 rw_exit(uobj->vmobjlock); 1768 if (amap != NULL) 1769 amap_unlock(amap); 1770 uvmfault_unlockmaps(ufi, FALSE); 1771 } 1772 1773 /* 1774 * uvmfault_lookup: lookup a virtual address in a map 1775 * 1776 * => caller must provide a uvm_faultinfo structure with the IN 1777 * params properly filled in 1778 * => we will lookup the map entry (handling submaps) as we go 1779 * => if the lookup is a success we will return with the maps locked 1780 * => if "write_lock" is TRUE, we write_lock the map, otherwise we only 1781 * get a read lock. 1782 * => note that submaps can only appear in the kernel and they are 1783 * required to use the same virtual addresses as the map they 1784 * are referenced by (thus address translation between the main 1785 * map and the submap is unnecessary). 1786 */ 1787 1788 boolean_t 1789 uvmfault_lookup(struct uvm_faultinfo *ufi, boolean_t write_lock) 1790 { 1791 vm_map_t tmpmap; 1792 1793 /* 1794 * init ufi values for lookup. 1795 */ 1796 ufi->map = ufi->orig_map; 1797 ufi->size = ufi->orig_size; 1798 1799 /* 1800 * keep going down levels until we are done. note that there can 1801 * only be two levels so we won't loop very long. 1802 */ 1803 while (1) { 1804 if (ufi->orig_rvaddr < ufi->map->min_offset || 1805 ufi->orig_rvaddr >= ufi->map->max_offset) 1806 return FALSE; 1807 1808 /* lock map */ 1809 if (write_lock) { 1810 vm_map_lock(ufi->map); 1811 } else { 1812 vm_map_lock_read(ufi->map); 1813 } 1814 1815 /* lookup */ 1816 if (!uvm_map_lookup_entry(ufi->map, ufi->orig_rvaddr, 1817 &ufi->entry)) { 1818 uvmfault_unlockmaps(ufi, write_lock); 1819 return FALSE; 1820 } 1821 1822 /* reduce size if necessary */ 1823 if (ufi->entry->end - ufi->orig_rvaddr < ufi->size) 1824 ufi->size = ufi->entry->end - ufi->orig_rvaddr; 1825 1826 /* 1827 * submap? replace map with the submap and lookup again. 1828 * note: VAs in submaps must match VAs in main map. 1829 */ 1830 if (UVM_ET_ISSUBMAP(ufi->entry)) { 1831 tmpmap = ufi->entry->object.sub_map; 1832 uvmfault_unlockmaps(ufi, write_lock); 1833 ufi->map = tmpmap; 1834 continue; 1835 } 1836 1837 /* 1838 * got it! 1839 */ 1840 ufi->mapv = ufi->map->timestamp; 1841 return TRUE; 1842 1843 } /* while loop */ 1844 1845 /*NOTREACHED*/ 1846 } 1847 1848 /* 1849 * uvmfault_relock: attempt to relock the same version of the map 1850 * 1851 * => fault data structures should be unlocked before calling. 1852 * => if a success (TRUE) maps will be locked after call. 1853 */ 1854 boolean_t 1855 uvmfault_relock(struct uvm_faultinfo *ufi) 1856 { 1857 /* 1858 * ufi can be NULL when this isn't really a fault, 1859 * but merely paging in anon data. 1860 */ 1861 if (ufi == NULL) { 1862 return TRUE; 1863 } 1864 1865 counters_inc(uvmexp_counters, flt_relck); 1866 1867 /* 1868 * relock map. fail if version mismatch (in which case nothing 1869 * gets locked). 1870 */ 1871 vm_map_lock_read(ufi->map); 1872 if (ufi->mapv != ufi->map->timestamp) { 1873 vm_map_unlock_read(ufi->map); 1874 return FALSE; 1875 } 1876 1877 counters_inc(uvmexp_counters, flt_relckok); 1878 return TRUE; /* got it! */ 1879 } 1880