1 /* $OpenBSD: uvm_fault.c,v 1.113 2021/01/19 13:21:36 mpi Exp $ */ 2 /* $NetBSD: uvm_fault.c,v 1.51 2000/08/06 00:22:53 thorpej Exp $ */ 3 4 /* 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp 29 */ 30 31 /* 32 * uvm_fault.c: fault handler 33 */ 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/percpu.h> 39 #include <sys/proc.h> 40 #include <sys/malloc.h> 41 #include <sys/mman.h> 42 #include <sys/tracepoint.h> 43 44 #include <uvm/uvm.h> 45 46 /* 47 * 48 * a word on page faults: 49 * 50 * types of page faults we handle: 51 * 52 * CASE 1: upper layer faults CASE 2: lower layer faults 53 * 54 * CASE 1A CASE 1B CASE 2A CASE 2B 55 * read/write1 write>1 read/write +-cow_write/zero 56 * | | | | 57 * +--|--+ +--|--+ +-----+ + | + | +-----+ 58 * amap | V | | ----------->new| | | | ^ | 59 * +-----+ +-----+ +-----+ + | + | +--|--+ 60 * | | | 61 * +-----+ +-----+ +--|--+ | +--|--+ 62 * uobj | d/c | | d/c | | V | +----| | 63 * +-----+ +-----+ +-----+ +-----+ 64 * 65 * d/c = don't care 66 * 67 * case [0]: layerless fault 68 * no amap or uobj is present. this is an error. 69 * 70 * case [1]: upper layer fault [anon active] 71 * 1A: [read] or [write with anon->an_ref == 1] 72 * I/O takes place in top level anon and uobj is not touched. 73 * 1B: [write with anon->an_ref > 1] 74 * new anon is alloc'd and data is copied off ["COW"] 75 * 76 * case [2]: lower layer fault [uobj] 77 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area] 78 * I/O takes place directly in object. 79 * 2B: [write to copy_on_write] or [read on NULL uobj] 80 * data is "promoted" from uobj to a new anon. 81 * if uobj is null, then we zero fill. 82 * 83 * we follow the standard UVM locking protocol ordering: 84 * 85 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ) 86 * we hold a PG_BUSY page if we unlock for I/O 87 * 88 * 89 * the code is structured as follows: 90 * 91 * - init the "IN" params in the ufi structure 92 * ReFault: 93 * - do lookups [locks maps], check protection, handle needs_copy 94 * - check for case 0 fault (error) 95 * - establish "range" of fault 96 * - if we have an amap lock it and extract the anons 97 * - if sequential advice deactivate pages behind us 98 * - at the same time check pmap for unmapped areas and anon for pages 99 * that we could map in (and do map it if found) 100 * - check object for resident pages that we could map in 101 * - if (case 2) goto Case2 102 * - >>> handle case 1 103 * - ensure source anon is resident in RAM 104 * - if case 1B alloc new anon and copy from source 105 * - map the correct page in 106 * Case2: 107 * - >>> handle case 2 108 * - ensure source page is resident (if uobj) 109 * - if case 2B alloc new anon and copy from source (could be zero 110 * fill if uobj == NULL) 111 * - map the correct page in 112 * - done! 113 * 114 * note on paging: 115 * if we have to do I/O we place a PG_BUSY page in the correct object, 116 * unlock everything, and do the I/O. when I/O is done we must reverify 117 * the state of the world before assuming that our data structures are 118 * valid. [because mappings could change while the map is unlocked] 119 * 120 * alternative 1: unbusy the page in question and restart the page fault 121 * from the top (ReFault). this is easy but does not take advantage 122 * of the information that we already have from our previous lookup, 123 * although it is possible that the "hints" in the vm_map will help here. 124 * 125 * alternative 2: the system already keeps track of a "version" number of 126 * a map. [i.e. every time you write-lock a map (e.g. to change a 127 * mapping) you bump the version number up by one...] so, we can save 128 * the version number of the map before we release the lock and start I/O. 129 * then when I/O is done we can relock and check the version numbers 130 * to see if anything changed. this might save us some over 1 because 131 * we don't have to unbusy the page and may be less compares(?). 132 * 133 * alternative 3: put in backpointers or a way to "hold" part of a map 134 * in place while I/O is in progress. this could be complex to 135 * implement (especially with structures like amap that can be referenced 136 * by multiple map entries, and figuring out what should wait could be 137 * complex as well...). 138 * 139 * we use alternative 2 currently. maybe alternative 3 would be useful 140 * in the future. XXX keep in mind for future consideration//rechecking. 141 */ 142 143 /* 144 * local data structures 145 */ 146 struct uvm_advice { 147 int nback; 148 int nforw; 149 }; 150 151 /* 152 * page range array: set up in uvmfault_init(). 153 */ 154 static struct uvm_advice uvmadvice[MADV_MASK + 1]; 155 156 #define UVM_MAXRANGE 16 /* must be max() of nback+nforw+1 */ 157 158 /* 159 * private prototypes 160 */ 161 static void uvmfault_amapcopy(struct uvm_faultinfo *); 162 static inline void uvmfault_anonflush(struct vm_anon **, int); 163 void uvmfault_unlockmaps(struct uvm_faultinfo *, boolean_t); 164 void uvmfault_update_stats(struct uvm_faultinfo *); 165 166 /* 167 * inline functions 168 */ 169 /* 170 * uvmfault_anonflush: try and deactivate pages in specified anons 171 * 172 * => does not have to deactivate page if it is busy 173 */ 174 static inline void 175 uvmfault_anonflush(struct vm_anon **anons, int n) 176 { 177 int lcv; 178 struct vm_page *pg; 179 180 for (lcv = 0 ; lcv < n ; lcv++) { 181 if (anons[lcv] == NULL) 182 continue; 183 KASSERT(rw_lock_held(anons[lcv]->an_lock)); 184 pg = anons[lcv]->an_page; 185 if (pg && (pg->pg_flags & PG_BUSY) == 0) { 186 uvm_lock_pageq(); 187 if (pg->wire_count == 0) { 188 pmap_page_protect(pg, PROT_NONE); 189 uvm_pagedeactivate(pg); 190 } 191 uvm_unlock_pageq(); 192 } 193 } 194 } 195 196 /* 197 * normal functions 198 */ 199 /* 200 * uvmfault_init: compute proper values for the uvmadvice[] array. 201 */ 202 void 203 uvmfault_init(void) 204 { 205 int npages; 206 207 npages = atop(16384); 208 if (npages > 0) { 209 KASSERT(npages <= UVM_MAXRANGE / 2); 210 uvmadvice[MADV_NORMAL].nforw = npages; 211 uvmadvice[MADV_NORMAL].nback = npages - 1; 212 } 213 214 npages = atop(32768); 215 if (npages > 0) { 216 KASSERT(npages <= UVM_MAXRANGE / 2); 217 uvmadvice[MADV_SEQUENTIAL].nforw = npages - 1; 218 uvmadvice[MADV_SEQUENTIAL].nback = npages; 219 } 220 } 221 222 /* 223 * uvmfault_amapcopy: clear "needs_copy" in a map. 224 * 225 * => if we are out of RAM we sleep (waiting for more) 226 */ 227 static void 228 uvmfault_amapcopy(struct uvm_faultinfo *ufi) 229 { 230 231 /* while we haven't done the job */ 232 while (1) { 233 /* no mapping? give up. */ 234 if (uvmfault_lookup(ufi, TRUE) == FALSE) 235 return; 236 237 /* copy if needed. */ 238 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) 239 amap_copy(ufi->map, ufi->entry, M_NOWAIT, 240 UVM_ET_ISSTACK(ufi->entry) ? FALSE : TRUE, 241 ufi->orig_rvaddr, ufi->orig_rvaddr + 1); 242 243 /* didn't work? must be out of RAM. sleep. */ 244 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 245 uvmfault_unlockmaps(ufi, TRUE); 246 uvm_wait("fltamapcopy"); 247 continue; 248 } 249 250 /* got it! */ 251 uvmfault_unlockmaps(ufi, TRUE); 252 return; 253 } 254 /*NOTREACHED*/ 255 } 256 257 /* 258 * uvmfault_anonget: get data in an anon into a non-busy, non-released 259 * page in that anon. 260 * 261 * => we don't move the page on the queues [gets moved later] 262 * => if we allocate a new page [we_own], it gets put on the queues. 263 * either way, the result is that the page is on the queues at return time 264 */ 265 int 266 uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap, 267 struct vm_anon *anon) 268 { 269 boolean_t we_own; /* we own anon's page? */ 270 boolean_t locked; /* did we relock? */ 271 struct vm_page *pg; 272 int result; 273 274 KASSERT(rw_lock_held(anon->an_lock)); 275 KASSERT(anon->an_lock == amap->am_lock); 276 277 result = 0; /* XXX shut up gcc */ 278 counters_inc(uvmexp_counters, flt_anget); 279 /* bump rusage counters */ 280 if (anon->an_page) 281 curproc->p_ru.ru_minflt++; 282 else 283 curproc->p_ru.ru_majflt++; 284 285 /* loop until we get it, or fail. */ 286 while (1) { 287 we_own = FALSE; /* TRUE if we set PG_BUSY on a page */ 288 pg = anon->an_page; 289 290 /* page there? make sure it is not busy/released. */ 291 if (pg) { 292 KASSERT(pg->pg_flags & PQ_ANON); 293 KASSERT(pg->uanon == anon); 294 295 /* 296 * if the page is busy, we drop all the locks and 297 * try again. 298 */ 299 if ((pg->pg_flags & (PG_BUSY|PG_RELEASED)) == 0) 300 return (VM_PAGER_OK); 301 atomic_setbits_int(&pg->pg_flags, PG_WANTED); 302 counters_inc(uvmexp_counters, flt_pgwait); 303 304 /* 305 * the last unlock must be an atomic unlock+wait on 306 * the owner of page 307 */ 308 if (pg->uobject) { 309 uvmfault_unlockall(ufi, amap, NULL); 310 tsleep_nsec(pg, PVM, "anonget1", INFSLP); 311 } else { 312 uvmfault_unlockall(ufi, NULL, NULL); 313 rwsleep_nsec(pg, anon->an_lock, PVM | PNORELOCK, 314 "anonget2", INFSLP); 315 } 316 /* ready to relock and try again */ 317 } else { 318 /* no page, we must try and bring it in. */ 319 pg = uvm_pagealloc(NULL, 0, anon, 0); 320 321 if (pg == NULL) { /* out of RAM. */ 322 uvmfault_unlockall(ufi, amap, NULL); 323 counters_inc(uvmexp_counters, flt_noram); 324 uvm_wait("flt_noram1"); 325 /* ready to relock and try again */ 326 } else { 327 /* we set the PG_BUSY bit */ 328 we_own = TRUE; 329 uvmfault_unlockall(ufi, amap, NULL); 330 331 /* 332 * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN 333 * page into the uvm_swap_get function with 334 * all data structures unlocked. note that 335 * it is ok to read an_swslot here because 336 * we hold PG_BUSY on the page. 337 */ 338 counters_inc(uvmexp_counters, pageins); 339 result = uvm_swap_get(pg, anon->an_swslot, 340 PGO_SYNCIO); 341 342 /* 343 * we clean up after the i/o below in the 344 * "we_own" case 345 */ 346 /* ready to relock and try again */ 347 } 348 } 349 350 /* now relock and try again */ 351 locked = uvmfault_relock(ufi); 352 if (locked || we_own) { 353 rw_enter(anon->an_lock, RW_WRITE); 354 } 355 356 /* 357 * if we own the page (i.e. we set PG_BUSY), then we need 358 * to clean up after the I/O. there are three cases to 359 * consider: 360 * [1] page released during I/O: free anon and ReFault. 361 * [2] I/O not OK. free the page and cause the fault 362 * to fail. 363 * [3] I/O OK! activate the page and sync with the 364 * non-we_own case (i.e. drop anon lock if not locked). 365 */ 366 if (we_own) { 367 if (pg->pg_flags & PG_WANTED) { 368 wakeup(pg); 369 } 370 /* un-busy! */ 371 atomic_clearbits_int(&pg->pg_flags, 372 PG_WANTED|PG_BUSY|PG_FAKE); 373 UVM_PAGE_OWN(pg, NULL); 374 375 /* 376 * if we were RELEASED during I/O, then our anon is 377 * no longer part of an amap. we need to free the 378 * anon and try again. 379 */ 380 if (pg->pg_flags & PG_RELEASED) { 381 pmap_page_protect(pg, PROT_NONE); 382 KASSERT(anon->an_ref == 0); 383 if (locked) 384 uvmfault_unlockall(ufi, amap, NULL); 385 uvm_anon_release(anon); /* frees page for us */ 386 counters_inc(uvmexp_counters, flt_pgrele); 387 return (VM_PAGER_REFAULT); /* refault! */ 388 } 389 390 if (result != VM_PAGER_OK) { 391 KASSERT(result != VM_PAGER_PEND); 392 393 /* remove page from anon */ 394 anon->an_page = NULL; 395 396 /* 397 * remove the swap slot from the anon 398 * and mark the anon as having no real slot. 399 * don't free the swap slot, thus preventing 400 * it from being used again. 401 */ 402 uvm_swap_markbad(anon->an_swslot, 1); 403 anon->an_swslot = SWSLOT_BAD; 404 405 /* 406 * note: page was never !PG_BUSY, so it 407 * can't be mapped and thus no need to 408 * pmap_page_protect it... 409 */ 410 uvm_lock_pageq(); 411 uvm_pagefree(pg); 412 uvm_unlock_pageq(); 413 414 if (locked) 415 uvmfault_unlockall(ufi, amap, NULL); 416 rw_exit(anon->an_lock); 417 return (VM_PAGER_ERROR); 418 } 419 420 /* 421 * must be OK, clear modify (already PG_CLEAN) 422 * and activate 423 */ 424 pmap_clear_modify(pg); 425 uvm_lock_pageq(); 426 uvm_pageactivate(pg); 427 uvm_unlock_pageq(); 428 } 429 430 /* we were not able to relock. restart fault. */ 431 if (!locked) { 432 if (we_own) { 433 rw_exit(anon->an_lock); 434 } 435 return (VM_PAGER_REFAULT); 436 } 437 438 /* verify no one touched the amap and moved the anon on us. */ 439 if (ufi != NULL && 440 amap_lookup(&ufi->entry->aref, 441 ufi->orig_rvaddr - ufi->entry->start) != anon) { 442 443 uvmfault_unlockall(ufi, amap, NULL); 444 return (VM_PAGER_REFAULT); 445 } 446 447 /* try it again! */ 448 counters_inc(uvmexp_counters, flt_anretry); 449 continue; 450 451 } /* while (1) */ 452 /*NOTREACHED*/ 453 } 454 455 /* 456 * Update statistics after fault resolution. 457 * - maxrss 458 */ 459 void 460 uvmfault_update_stats(struct uvm_faultinfo *ufi) 461 { 462 struct vm_map *map; 463 struct proc *p; 464 vsize_t res; 465 466 map = ufi->orig_map; 467 468 /* 469 * If this is a nested pmap (eg, a virtual machine pmap managed 470 * by vmm(4) on amd64/i386), don't do any updating, just return. 471 * 472 * pmap_nested() on other archs is #defined to 0, so this is a 473 * no-op. 474 */ 475 if (pmap_nested(map->pmap)) 476 return; 477 478 /* Update the maxrss for the process. */ 479 if (map->flags & VM_MAP_ISVMSPACE) { 480 p = curproc; 481 KASSERT(p != NULL && &p->p_vmspace->vm_map == map); 482 483 res = pmap_resident_count(map->pmap); 484 /* Convert res from pages to kilobytes. */ 485 res <<= (PAGE_SHIFT - 10); 486 487 if (p->p_ru.ru_maxrss < res) 488 p->p_ru.ru_maxrss = res; 489 } 490 } 491 492 struct uvm_faultctx { 493 /* 494 * the following members are set up by uvm_fault_check() and 495 * read-only after that. 496 */ 497 vm_prot_t enter_prot; 498 vm_prot_t access_type; 499 vaddr_t startva; 500 int npages; 501 int centeridx; 502 boolean_t narrow; 503 boolean_t wired; 504 paddr_t pa_flags; 505 }; 506 507 int uvm_fault_lower(struct uvm_faultinfo *, struct uvm_faultctx *, 508 struct vm_page **, vm_fault_t); 509 510 /* 511 * uvm_fault_check: check prot, handle needs-copy, etc. 512 * 513 * 1. lookup entry. 514 * 2. check protection. 515 * 3. adjust fault condition (mainly for simulated fault). 516 * 4. handle needs-copy (lazy amap copy). 517 * 5. establish range of interest for neighbor fault (aka pre-fault). 518 * 6. look up anons (if amap exists). 519 * 7. flush pages (if MADV_SEQUENTIAL) 520 * 521 * => called with nothing locked. 522 * => if we fail (result != 0) we unlock everything. 523 * => initialize/adjust many members of flt. 524 */ 525 int 526 uvm_fault_check(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 527 struct vm_anon ***ranons) 528 { 529 struct vm_amap *amap; 530 struct uvm_object *uobj; 531 int nback, nforw; 532 533 /* lookup and lock the maps */ 534 if (uvmfault_lookup(ufi, FALSE) == FALSE) { 535 return (EFAULT); 536 } 537 538 #ifdef DIAGNOSTIC 539 if ((ufi->map->flags & VM_MAP_PAGEABLE) == 0) 540 panic("uvm_fault: fault on non-pageable map (%p, 0x%lx)", 541 ufi->map, ufi->orig_rvaddr); 542 #endif 543 544 /* check protection */ 545 if ((ufi->entry->protection & flt->access_type) != flt->access_type) { 546 uvmfault_unlockmaps(ufi, FALSE); 547 return (EACCES); 548 } 549 550 /* 551 * "enter_prot" is the protection we want to enter the page in at. 552 * for certain pages (e.g. copy-on-write pages) this protection can 553 * be more strict than ufi->entry->protection. "wired" means either 554 * the entry is wired or we are fault-wiring the pg. 555 */ 556 557 flt->enter_prot = ufi->entry->protection; 558 flt->pa_flags = UVM_ET_ISWC(ufi->entry) ? PMAP_WC : 0; 559 flt->wired = VM_MAPENT_ISWIRED(ufi->entry) || (flt->narrow == TRUE); 560 if (flt->wired) 561 flt->access_type = flt->enter_prot; /* full access for wired */ 562 563 /* handle "needs_copy" case. */ 564 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 565 if ((flt->access_type & PROT_WRITE) || 566 (ufi->entry->object.uvm_obj == NULL)) { 567 /* need to clear */ 568 uvmfault_unlockmaps(ufi, FALSE); 569 uvmfault_amapcopy(ufi); 570 counters_inc(uvmexp_counters, flt_amcopy); 571 return (ERESTART); 572 } else { 573 /* 574 * ensure that we pmap_enter page R/O since 575 * needs_copy is still true 576 */ 577 flt->enter_prot &= ~PROT_WRITE; 578 } 579 } 580 581 /* identify the players */ 582 amap = ufi->entry->aref.ar_amap; /* top layer */ 583 uobj = ufi->entry->object.uvm_obj; /* bottom layer */ 584 585 /* 586 * check for a case 0 fault. if nothing backing the entry then 587 * error now. 588 */ 589 if (amap == NULL && uobj == NULL) { 590 uvmfault_unlockmaps(ufi, FALSE); 591 return (EFAULT); 592 } 593 594 /* 595 * establish range of interest based on advice from mapper 596 * and then clip to fit map entry. note that we only want 597 * to do this the first time through the fault. if we 598 * ReFault we will disable this by setting "narrow" to true. 599 */ 600 if (flt->narrow == FALSE) { 601 602 /* wide fault (!narrow) */ 603 nback = min(uvmadvice[ufi->entry->advice].nback, 604 (ufi->orig_rvaddr - ufi->entry->start) >> PAGE_SHIFT); 605 flt->startva = ufi->orig_rvaddr - ((vsize_t)nback << PAGE_SHIFT); 606 nforw = min(uvmadvice[ufi->entry->advice].nforw, 607 ((ufi->entry->end - ufi->orig_rvaddr) >> PAGE_SHIFT) - 1); 608 /* 609 * note: "-1" because we don't want to count the 610 * faulting page as forw 611 */ 612 flt->npages = nback + nforw + 1; 613 flt->centeridx = nback; 614 615 flt->narrow = TRUE; /* ensure only once per-fault */ 616 } else { 617 /* narrow fault! */ 618 nback = nforw = 0; 619 flt->startva = ufi->orig_rvaddr; 620 flt->npages = 1; 621 flt->centeridx = 0; 622 } 623 624 /* if we've got an amap, extract current anons. */ 625 if (amap) { 626 amap_lock(amap); 627 amap_lookups(&ufi->entry->aref, 628 flt->startva - ufi->entry->start, *ranons, flt->npages); 629 } else { 630 *ranons = NULL; /* to be safe */ 631 } 632 633 /* 634 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages 635 * now and then forget about them (for the rest of the fault). 636 */ 637 if (ufi->entry->advice == MADV_SEQUENTIAL && nback != 0) { 638 /* flush back-page anons? */ 639 if (amap) 640 uvmfault_anonflush(*ranons, nback); 641 642 /* flush object? */ 643 if (uobj) { 644 voff_t uoff; 645 646 uoff = (flt->startva - ufi->entry->start) + ufi->entry->offset; 647 KERNEL_LOCK(); 648 (void) uobj->pgops->pgo_flush(uobj, uoff, uoff + 649 ((vsize_t)nback << PAGE_SHIFT), PGO_DEACTIVATE); 650 KERNEL_UNLOCK(); 651 } 652 653 /* now forget about the backpages */ 654 if (amap) 655 *ranons += nback; 656 flt->startva += ((vsize_t)nback << PAGE_SHIFT); 657 flt->npages -= nback; 658 flt->centeridx = 0; 659 } 660 661 return 0; 662 } 663 664 /* 665 * uvm_fault_upper: handle upper fault (case 1A & 1B) 666 * 667 * 1. get anon. let uvmfault_anonget do the dirty work. 668 * 2. if COW, promote data to new anon 669 * 3. enter h/w mapping 670 */ 671 int 672 uvm_fault_upper(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 673 struct vm_anon **anons, vm_fault_t fault_type) 674 { 675 struct vm_amap *amap = ufi->entry->aref.ar_amap; 676 struct vm_anon *oanon, *anon = anons[flt->centeridx]; 677 struct vm_page *pg = NULL; 678 int error, ret; 679 680 KASSERT(rw_write_held(amap->am_lock)); 681 KASSERT(anon->an_lock == amap->am_lock); 682 683 /* 684 * no matter if we have case 1A or case 1B we are going to need to 685 * have the anon's memory resident. ensure that now. 686 */ 687 /* 688 * let uvmfault_anonget do the dirty work. 689 * also, if it is OK, then the anon's page is on the queues. 690 */ 691 error = uvmfault_anonget(ufi, amap, anon); 692 switch (error) { 693 case VM_PAGER_OK: 694 break; 695 696 case VM_PAGER_REFAULT: 697 return ERESTART; 698 699 case VM_PAGER_ERROR: 700 /* 701 * An error occured while trying to bring in the 702 * page -- this is the only error we return right 703 * now. 704 */ 705 return EACCES; /* XXX */ 706 default: 707 #ifdef DIAGNOSTIC 708 panic("uvm_fault: uvmfault_anonget -> %d", error); 709 #else 710 return EACCES; 711 #endif 712 } 713 714 KASSERT(rw_write_held(amap->am_lock)); 715 KASSERT(anon->an_lock == amap->am_lock); 716 717 /* 718 * if we are case 1B then we will need to allocate a new blank 719 * anon to transfer the data into. note that we have a lock 720 * on anon, so no one can busy or release the page until we are done. 721 * also note that the ref count can't drop to zero here because 722 * it is > 1 and we are only dropping one ref. 723 * 724 * in the (hopefully very rare) case that we are out of RAM we 725 * will wait for more RAM, and refault. 726 * 727 * if we are out of anon VM we wait for RAM to become available. 728 */ 729 730 if ((flt->access_type & PROT_WRITE) != 0 && anon->an_ref > 1) { 731 counters_inc(uvmexp_counters, flt_acow); 732 oanon = anon; /* oanon = old */ 733 anon = uvm_analloc(); 734 if (anon) { 735 anon->an_lock = amap->am_lock; 736 pg = uvm_pagealloc(NULL, 0, anon, 0); 737 } 738 739 /* check for out of RAM */ 740 if (anon == NULL || pg == NULL) { 741 uvmfault_unlockall(ufi, amap, NULL); 742 if (anon == NULL) 743 counters_inc(uvmexp_counters, flt_noanon); 744 else { 745 anon->an_lock = NULL; 746 anon->an_ref--; 747 uvm_anfree(anon); 748 counters_inc(uvmexp_counters, flt_noram); 749 } 750 751 if (uvm_swapisfull()) 752 return ENOMEM; 753 754 /* out of RAM, wait for more */ 755 if (anon == NULL) 756 uvm_anwait(); 757 else 758 uvm_wait("flt_noram3"); 759 return ERESTART; 760 } 761 762 /* got all resources, replace anon with nanon */ 763 uvm_pagecopy(oanon->an_page, pg); /* pg now !PG_CLEAN */ 764 /* un-busy! new page */ 765 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE); 766 UVM_PAGE_OWN(pg, NULL); 767 ret = amap_add(&ufi->entry->aref, 768 ufi->orig_rvaddr - ufi->entry->start, anon, 1); 769 KASSERT(ret == 0); 770 771 /* deref: can not drop to zero here by defn! */ 772 oanon->an_ref--; 773 774 /* 775 * note: anon is _not_ locked, but we have the sole references 776 * to in from amap. 777 * thus, no one can get at it until we are done with it. 778 */ 779 } else { 780 counters_inc(uvmexp_counters, flt_anon); 781 oanon = anon; 782 pg = anon->an_page; 783 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */ 784 flt->enter_prot = flt->enter_prot & ~PROT_WRITE; 785 } 786 787 /* 788 * now map the page in ... 789 * XXX: old fault unlocks object before pmap_enter. this seems 790 * suspect since some other thread could blast the page out from 791 * under us between the unlock and the pmap_enter. 792 */ 793 if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr, 794 VM_PAGE_TO_PHYS(pg) | flt->pa_flags, flt->enter_prot, 795 flt->access_type | PMAP_CANFAIL | (flt->wired ? PMAP_WIRED : 0)) != 0) { 796 /* 797 * No need to undo what we did; we can simply think of 798 * this as the pmap throwing away the mapping information. 799 * 800 * We do, however, have to go through the ReFault path, 801 * as the map may change while we're asleep. 802 */ 803 uvmfault_unlockall(ufi, amap, NULL); 804 if (uvm_swapisfull()) { 805 /* XXX instrumentation */ 806 return ENOMEM; 807 } 808 /* XXX instrumentation */ 809 uvm_wait("flt_pmfail1"); 810 return ERESTART; 811 } 812 813 /* ... update the page queues. */ 814 uvm_lock_pageq(); 815 816 if (fault_type == VM_FAULT_WIRE) { 817 uvm_pagewire(pg); 818 /* 819 * since the now-wired page cannot be paged out, 820 * release its swap resources for others to use. 821 * since an anon with no swap cannot be PG_CLEAN, 822 * clear its clean flag now. 823 */ 824 atomic_clearbits_int(&pg->pg_flags, PG_CLEAN); 825 uvm_anon_dropswap(anon); 826 } else { 827 /* activate it */ 828 uvm_pageactivate(pg); 829 } 830 831 uvm_unlock_pageq(); 832 833 /* done case 1! finish up by unlocking everything and returning success */ 834 uvmfault_unlockall(ufi, amap, NULL); 835 pmap_update(ufi->orig_map->pmap); 836 return 0; 837 } 838 839 /* 840 * uvm_fault_upper_lookup: look up existing h/w mapping and amap. 841 * 842 * iterate range of interest: 843 * 1. check if h/w mapping exists. if yes, we don't care 844 * 2. check if anon exists. if not, page is lower. 845 * 3. if anon exists, enter h/w mapping for neighbors. 846 */ 847 boolean_t 848 uvm_fault_upper_lookup(struct uvm_faultinfo *ufi, 849 const struct uvm_faultctx *flt, struct vm_anon **anons, 850 struct vm_page **pages) 851 { 852 struct vm_amap *amap = ufi->entry->aref.ar_amap; 853 struct vm_anon *anon; 854 boolean_t shadowed; 855 vaddr_t currva; 856 paddr_t pa; 857 int lcv; 858 859 /* 860 * map in the backpages and frontpages we found in the amap in hopes 861 * of preventing future faults. we also init the pages[] array as 862 * we go. 863 */ 864 currva = flt->startva; 865 shadowed = FALSE; 866 for (lcv = 0 ; lcv < flt->npages ; lcv++, currva += PAGE_SIZE) { 867 /* 868 * dont play with VAs that are already mapped 869 * except for center) 870 */ 871 if (lcv != flt->centeridx && 872 pmap_extract(ufi->orig_map->pmap, currva, &pa)) { 873 pages[lcv] = PGO_DONTCARE; 874 continue; 875 } 876 877 /* unmapped or center page. check if any anon at this level. */ 878 if (amap == NULL || anons[lcv] == NULL) { 879 pages[lcv] = NULL; 880 continue; 881 } 882 883 /* check for present page and map if possible. re-activate it */ 884 pages[lcv] = PGO_DONTCARE; 885 if (lcv == flt->centeridx) { /* save center for later! */ 886 shadowed = TRUE; 887 continue; 888 } 889 anon = anons[lcv]; 890 KASSERT(anon->an_lock == amap->am_lock); 891 if (anon->an_page && 892 (anon->an_page->pg_flags & (PG_RELEASED|PG_BUSY)) == 0) { 893 uvm_lock_pageq(); 894 uvm_pageactivate(anon->an_page); /* reactivate */ 895 uvm_unlock_pageq(); 896 counters_inc(uvmexp_counters, flt_namap); 897 898 /* 899 * Since this isn't the page that's actually faulting, 900 * ignore pmap_enter() failures; it's not critical 901 * that we enter these right now. 902 */ 903 (void) pmap_enter(ufi->orig_map->pmap, currva, 904 VM_PAGE_TO_PHYS(anon->an_page) | flt->pa_flags, 905 (anon->an_ref > 1) ? 906 (flt->enter_prot & ~PROT_WRITE) : flt->enter_prot, 907 PMAP_CANFAIL | 908 (VM_MAPENT_ISWIRED(ufi->entry) ? PMAP_WIRED : 0)); 909 } 910 } 911 if (flt->npages > 1) 912 pmap_update(ufi->orig_map->pmap); 913 914 return shadowed; 915 } 916 917 /* 918 * F A U L T - m a i n e n t r y p o i n t 919 */ 920 921 /* 922 * uvm_fault: page fault handler 923 * 924 * => called from MD code to resolve a page fault 925 * => VM data structures usually should be unlocked. however, it is 926 * possible to call here with the main map locked if the caller 927 * gets a write lock, sets it recursive, and then calls us (c.f. 928 * uvm_map_pageable). this should be avoided because it keeps 929 * the map locked off during I/O. 930 */ 931 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 932 ~PROT_WRITE : PROT_MASK) 933 int 934 uvm_fault(vm_map_t orig_map, vaddr_t vaddr, vm_fault_t fault_type, 935 vm_prot_t access_type) 936 { 937 struct uvm_faultinfo ufi; 938 struct uvm_faultctx flt; 939 boolean_t shadowed; 940 struct vm_anon *anons_store[UVM_MAXRANGE], **anons; 941 struct vm_page *pages[UVM_MAXRANGE]; 942 int error = ERESTART; 943 944 counters_inc(uvmexp_counters, faults); 945 TRACEPOINT(uvm, fault, vaddr, fault_type, access_type, NULL); 946 947 /* init the IN parameters in the ufi */ 948 ufi.orig_map = orig_map; 949 ufi.orig_rvaddr = trunc_page(vaddr); 950 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ 951 if (fault_type == VM_FAULT_WIRE) 952 flt.narrow = TRUE; /* don't look for neighborhood 953 * pages on wire */ 954 else 955 flt.narrow = FALSE; /* normal fault */ 956 flt.access_type = access_type; 957 958 959 /* 960 * ReFault 961 */ 962 while (error == ERESTART) { 963 anons = anons_store; 964 965 error = uvm_fault_check(&ufi, &flt, &anons); 966 if (error != 0) 967 continue; 968 969 /* True if there is an anon at the faulting address */ 970 shadowed = uvm_fault_upper_lookup(&ufi, &flt, anons, pages); 971 if (shadowed == TRUE) { 972 /* case 1: fault on an anon in our amap */ 973 error = uvm_fault_upper(&ufi, &flt, anons, fault_type); 974 } else { 975 /* case 2: fault on backing object or zero fill */ 976 KERNEL_LOCK(); 977 error = uvm_fault_lower(&ufi, &flt, pages, fault_type); 978 KERNEL_UNLOCK(); 979 } 980 } 981 982 return error; 983 } 984 985 int 986 uvm_fault_lower(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 987 struct vm_page **pages, vm_fault_t fault_type) 988 { 989 struct vm_amap *amap = ufi->entry->aref.ar_amap; 990 struct uvm_object *uobj = ufi->entry->object.uvm_obj; 991 boolean_t promote, locked; 992 int result, lcv, gotpages; 993 struct vm_page *uobjpage, *pg = NULL; 994 struct vm_anon *anon = NULL; 995 vaddr_t currva; 996 voff_t uoff; 997 998 /* 999 * if the desired page is not shadowed by the amap and we have a 1000 * backing object, then we check to see if the backing object would 1001 * prefer to handle the fault itself (rather than letting us do it 1002 * with the usual pgo_get hook). the backing object signals this by 1003 * providing a pgo_fault routine. 1004 */ 1005 if (uobj != NULL && uobj->pgops->pgo_fault != NULL) { 1006 result = uobj->pgops->pgo_fault(ufi, flt->startva, pages, 1007 flt->npages, flt->centeridx, fault_type, flt->access_type, 1008 PGO_LOCKED); 1009 1010 if (result == VM_PAGER_OK) 1011 return (0); /* pgo_fault did pmap enter */ 1012 else if (result == VM_PAGER_REFAULT) 1013 return ERESTART; /* try again! */ 1014 else 1015 return (EACCES); 1016 } 1017 1018 /* 1019 * now, if the desired page is not shadowed by the amap and we have 1020 * a backing object that does not have a special fault routine, then 1021 * we ask (with pgo_get) the object for resident pages that we care 1022 * about and attempt to map them in. we do not let pgo_get block 1023 * (PGO_LOCKED). 1024 * 1025 * ("get" has the option of doing a pmap_enter for us) 1026 */ 1027 if (uobj != NULL) { 1028 counters_inc(uvmexp_counters, flt_lget); 1029 gotpages = flt->npages; 1030 (void) uobj->pgops->pgo_get(uobj, ufi->entry->offset + 1031 (flt->startva - ufi->entry->start), 1032 pages, &gotpages, flt->centeridx, 1033 flt->access_type & MASK(ufi->entry), 1034 ufi->entry->advice, PGO_LOCKED); 1035 1036 /* check for pages to map, if we got any */ 1037 uobjpage = NULL; 1038 if (gotpages) { 1039 currva = flt->startva; 1040 for (lcv = 0 ; lcv < flt->npages ; 1041 lcv++, currva += PAGE_SIZE) { 1042 if (pages[lcv] == NULL || 1043 pages[lcv] == PGO_DONTCARE) 1044 continue; 1045 1046 KASSERT((pages[lcv]->pg_flags & PG_RELEASED) == 0); 1047 1048 /* 1049 * if center page is resident and not 1050 * PG_BUSY, then pgo_get made it PG_BUSY 1051 * for us and gave us a handle to it. 1052 * remember this page as "uobjpage." 1053 * (for later use). 1054 */ 1055 if (lcv == flt->centeridx) { 1056 uobjpage = pages[lcv]; 1057 continue; 1058 } 1059 1060 /* 1061 * note: calling pgo_get with locked data 1062 * structures returns us pages which are 1063 * neither busy nor released, so we don't 1064 * need to check for this. we can just 1065 * directly enter the page (after moving it 1066 * to the head of the active queue [useful?]). 1067 */ 1068 1069 uvm_lock_pageq(); 1070 uvm_pageactivate(pages[lcv]); /* reactivate */ 1071 uvm_unlock_pageq(); 1072 counters_inc(uvmexp_counters, flt_nomap); 1073 1074 /* 1075 * Since this page isn't the page that's 1076 * actually faulting, ignore pmap_enter() 1077 * failures; it's not critical that we 1078 * enter these right now. 1079 */ 1080 (void) pmap_enter(ufi->orig_map->pmap, currva, 1081 VM_PAGE_TO_PHYS(pages[lcv]) | flt->pa_flags, 1082 flt->enter_prot & MASK(ufi->entry), 1083 PMAP_CANFAIL | 1084 (flt->wired ? PMAP_WIRED : 0)); 1085 1086 /* 1087 * NOTE: page can't be PG_WANTED because 1088 * we've held the lock the whole time 1089 * we've had the handle. 1090 */ 1091 atomic_clearbits_int(&pages[lcv]->pg_flags, 1092 PG_BUSY); 1093 UVM_PAGE_OWN(pages[lcv], NULL); 1094 } /* for "lcv" loop */ 1095 pmap_update(ufi->orig_map->pmap); 1096 } /* "gotpages" != 0 */ 1097 /* note: object still _locked_ */ 1098 } else { 1099 uobjpage = NULL; 1100 } 1101 1102 /* 1103 * note that at this point we are done with any front or back pages. 1104 * we are now going to focus on the center page (i.e. the one we've 1105 * faulted on). if we have faulted on the bottom (uobj) 1106 * layer [i.e. case 2] and the page was both present and available, 1107 * then we've got a pointer to it as "uobjpage" and we've already 1108 * made it BUSY. 1109 */ 1110 1111 /* 1112 * note that uobjpage can not be PGO_DONTCARE at this point. we now 1113 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we 1114 * have a backing object, check and see if we are going to promote 1115 * the data up to an anon during the fault. 1116 */ 1117 if (uobj == NULL) { 1118 uobjpage = PGO_DONTCARE; 1119 promote = TRUE; /* always need anon here */ 1120 } else { 1121 KASSERT(uobjpage != PGO_DONTCARE); 1122 promote = (flt->access_type & PROT_WRITE) && 1123 UVM_ET_ISCOPYONWRITE(ufi->entry); 1124 } 1125 1126 /* 1127 * if uobjpage is not null then we do not need to do I/O to get the 1128 * uobjpage. 1129 * 1130 * if uobjpage is null, then we need to ask the pager to 1131 * get the data for us. once we have the data, we need to reverify 1132 * the state the world. we are currently not holding any resources. 1133 */ 1134 if (uobjpage) { 1135 /* update rusage counters */ 1136 curproc->p_ru.ru_minflt++; 1137 } else { 1138 /* update rusage counters */ 1139 curproc->p_ru.ru_majflt++; 1140 1141 uvmfault_unlockall(ufi, amap, NULL); 1142 1143 counters_inc(uvmexp_counters, flt_get); 1144 gotpages = 1; 1145 uoff = (ufi->orig_rvaddr - ufi->entry->start) + ufi->entry->offset; 1146 result = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages, 1147 0, flt->access_type & MASK(ufi->entry), ufi->entry->advice, 1148 PGO_SYNCIO); 1149 1150 /* recover from I/O */ 1151 if (result != VM_PAGER_OK) { 1152 KASSERT(result != VM_PAGER_PEND); 1153 1154 if (result == VM_PAGER_AGAIN) { 1155 tsleep_nsec(&nowake, PVM, "fltagain2", 1156 SEC_TO_NSEC(1)); 1157 return ERESTART; 1158 } 1159 1160 if (!UVM_ET_ISNOFAULT(ufi->entry)) 1161 return (EIO); 1162 1163 uobjpage = PGO_DONTCARE; 1164 promote = TRUE; 1165 } 1166 1167 /* re-verify the state of the world. */ 1168 locked = uvmfault_relock(ufi); 1169 if (locked && amap != NULL) 1170 amap_lock(amap); 1171 1172 /* 1173 * Re-verify that amap slot is still free. if there is 1174 * a problem, we clean up. 1175 */ 1176 if (locked && amap && amap_lookup(&ufi->entry->aref, 1177 ufi->orig_rvaddr - ufi->entry->start)) { 1178 if (locked) 1179 uvmfault_unlockall(ufi, amap, NULL); 1180 locked = FALSE; 1181 } 1182 1183 /* didn't get the lock? release the page and retry. */ 1184 if (locked == FALSE && uobjpage != PGO_DONTCARE) { 1185 uvm_lock_pageq(); 1186 /* make sure it is in queues */ 1187 uvm_pageactivate(uobjpage); 1188 uvm_unlock_pageq(); 1189 1190 if (uobjpage->pg_flags & PG_WANTED) 1191 /* still holding object lock */ 1192 wakeup(uobjpage); 1193 atomic_clearbits_int(&uobjpage->pg_flags, 1194 PG_BUSY|PG_WANTED); 1195 UVM_PAGE_OWN(uobjpage, NULL); 1196 return ERESTART; 1197 } 1198 if (locked == FALSE) 1199 return ERESTART; 1200 1201 /* 1202 * we have the data in uobjpage which is PG_BUSY 1203 */ 1204 } 1205 1206 /* 1207 * notes: 1208 * - at this point uobjpage can not be NULL 1209 * - at this point uobjpage could be PG_WANTED (handle later) 1210 */ 1211 if (promote == FALSE) { 1212 /* 1213 * we are not promoting. if the mapping is COW ensure that we 1214 * don't give more access than we should (e.g. when doing a read 1215 * fault on a COPYONWRITE mapping we want to map the COW page in 1216 * R/O even though the entry protection could be R/W). 1217 * 1218 * set "pg" to the page we want to map in (uobjpage, usually) 1219 */ 1220 counters_inc(uvmexp_counters, flt_obj); 1221 if (UVM_ET_ISCOPYONWRITE(ufi->entry)) 1222 flt->enter_prot &= ~PROT_WRITE; 1223 pg = uobjpage; /* map in the actual object */ 1224 1225 /* assert(uobjpage != PGO_DONTCARE) */ 1226 1227 /* 1228 * we are faulting directly on the page. 1229 */ 1230 } else { 1231 /* 1232 * if we are going to promote the data to an anon we 1233 * allocate a blank anon here and plug it into our amap. 1234 */ 1235 #ifdef DIAGNOSTIC 1236 if (amap == NULL) 1237 panic("uvm_fault: want to promote data, but no anon"); 1238 #endif 1239 1240 anon = uvm_analloc(); 1241 if (anon) { 1242 /* 1243 * In `Fill in data...' below, if 1244 * uobjpage == PGO_DONTCARE, we want 1245 * a zero'd, dirty page, so have 1246 * uvm_pagealloc() do that for us. 1247 */ 1248 anon->an_lock = amap->am_lock; 1249 pg = uvm_pagealloc(NULL, 0, anon, 1250 (uobjpage == PGO_DONTCARE) ? UVM_PGA_ZERO : 0); 1251 } 1252 1253 /* 1254 * out of memory resources? 1255 */ 1256 if (anon == NULL || pg == NULL) { 1257 /* arg! must unbusy our page and fail or sleep. */ 1258 if (uobjpage != PGO_DONTCARE) { 1259 uvm_lock_pageq(); 1260 uvm_pageactivate(uobjpage); 1261 uvm_unlock_pageq(); 1262 1263 if (uobjpage->pg_flags & PG_WANTED) 1264 wakeup(uobjpage); 1265 atomic_clearbits_int(&uobjpage->pg_flags, 1266 PG_BUSY|PG_WANTED); 1267 UVM_PAGE_OWN(uobjpage, NULL); 1268 } 1269 1270 /* unlock and fail ... */ 1271 uvmfault_unlockall(ufi, amap, uobj); 1272 if (anon == NULL) 1273 counters_inc(uvmexp_counters, flt_noanon); 1274 else { 1275 anon->an_lock = NULL; 1276 anon->an_ref--; 1277 uvm_anfree(anon); 1278 counters_inc(uvmexp_counters, flt_noram); 1279 } 1280 1281 if (uvm_swapisfull()) 1282 return (ENOMEM); 1283 1284 /* out of RAM, wait for more */ 1285 if (anon == NULL) 1286 uvm_anwait(); 1287 else 1288 uvm_wait("flt_noram5"); 1289 return ERESTART; 1290 } 1291 1292 /* fill in the data */ 1293 if (uobjpage != PGO_DONTCARE) { 1294 counters_inc(uvmexp_counters, flt_prcopy); 1295 /* copy page [pg now dirty] */ 1296 uvm_pagecopy(uobjpage, pg); 1297 1298 /* 1299 * promote to shared amap? make sure all sharing 1300 * procs see it 1301 */ 1302 if ((amap_flags(amap) & AMAP_SHARED) != 0) { 1303 pmap_page_protect(uobjpage, PROT_NONE); 1304 } 1305 1306 /* dispose of uobjpage. drop handle to uobj as well. */ 1307 if (uobjpage->pg_flags & PG_WANTED) 1308 wakeup(uobjpage); 1309 atomic_clearbits_int(&uobjpage->pg_flags, 1310 PG_BUSY|PG_WANTED); 1311 UVM_PAGE_OWN(uobjpage, NULL); 1312 uvm_lock_pageq(); 1313 uvm_pageactivate(uobjpage); 1314 uvm_unlock_pageq(); 1315 uobj = NULL; 1316 } else { 1317 counters_inc(uvmexp_counters, flt_przero); 1318 /* 1319 * Page is zero'd and marked dirty by uvm_pagealloc() 1320 * above. 1321 */ 1322 } 1323 1324 if (amap_add(&ufi->entry->aref, 1325 ufi->orig_rvaddr - ufi->entry->start, anon, 0)) { 1326 uvmfault_unlockall(ufi, amap, NULL); 1327 uvm_anfree(anon); 1328 counters_inc(uvmexp_counters, flt_noamap); 1329 1330 if (uvm_swapisfull()) 1331 return (ENOMEM); 1332 1333 amap_populate(&ufi->entry->aref, 1334 ufi->orig_rvaddr - ufi->entry->start); 1335 return ERESTART; 1336 } 1337 } 1338 1339 /* note: pg is either the uobjpage or the new page in the new anon */ 1340 /* 1341 * all resources are present. we can now map it in and free our 1342 * resources. 1343 */ 1344 if (amap == NULL) 1345 KASSERT(anon == NULL); 1346 else { 1347 KASSERT(rw_write_held(amap->am_lock)); 1348 KASSERT(anon == NULL || anon->an_lock == amap->am_lock); 1349 } 1350 if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr, 1351 VM_PAGE_TO_PHYS(pg) | flt->pa_flags, flt->enter_prot, 1352 flt->access_type | PMAP_CANFAIL | (flt->wired ? PMAP_WIRED : 0)) != 0) { 1353 /* 1354 * No need to undo what we did; we can simply think of 1355 * this as the pmap throwing away the mapping information. 1356 * 1357 * We do, however, have to go through the ReFault path, 1358 * as the map may change while we're asleep. 1359 */ 1360 if (pg->pg_flags & PG_WANTED) 1361 wakeup(pg); 1362 1363 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED); 1364 UVM_PAGE_OWN(pg, NULL); 1365 uvmfault_unlockall(ufi, amap, uobj); 1366 if (uvm_swapisfull()) { 1367 /* XXX instrumentation */ 1368 return (ENOMEM); 1369 } 1370 /* XXX instrumentation */ 1371 uvm_wait("flt_pmfail2"); 1372 return ERESTART; 1373 } 1374 1375 uvm_lock_pageq(); 1376 1377 if (fault_type == VM_FAULT_WIRE) { 1378 uvm_pagewire(pg); 1379 if (pg->pg_flags & PQ_AOBJ) { 1380 /* 1381 * since the now-wired page cannot be paged out, 1382 * release its swap resources for others to use. 1383 * since an aobj page with no swap cannot be PG_CLEAN, 1384 * clear its clean flag now. 1385 */ 1386 atomic_clearbits_int(&pg->pg_flags, PG_CLEAN); 1387 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT); 1388 } 1389 } else { 1390 /* activate it */ 1391 uvm_pageactivate(pg); 1392 } 1393 uvm_unlock_pageq(); 1394 1395 if (pg->pg_flags & PG_WANTED) 1396 wakeup(pg); 1397 1398 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED); 1399 UVM_PAGE_OWN(pg, NULL); 1400 uvmfault_unlockall(ufi, amap, uobj); 1401 pmap_update(ufi->orig_map->pmap); 1402 1403 return (0); 1404 } 1405 1406 1407 /* 1408 * uvm_fault_wire: wire down a range of virtual addresses in a map. 1409 * 1410 * => map may be read-locked by caller, but MUST NOT be write-locked. 1411 * => if map is read-locked, any operations which may cause map to 1412 * be write-locked in uvm_fault() must be taken care of by 1413 * the caller. See uvm_map_pageable(). 1414 */ 1415 int 1416 uvm_fault_wire(vm_map_t map, vaddr_t start, vaddr_t end, vm_prot_t access_type) 1417 { 1418 vaddr_t va; 1419 int rv; 1420 1421 /* 1422 * now fault it in a page at a time. if the fault fails then we have 1423 * to undo what we have done. note that in uvm_fault PROT_NONE 1424 * is replaced with the max protection if fault_type is VM_FAULT_WIRE. 1425 */ 1426 for (va = start ; va < end ; va += PAGE_SIZE) { 1427 rv = uvm_fault(map, va, VM_FAULT_WIRE, access_type); 1428 if (rv) { 1429 if (va != start) { 1430 uvm_fault_unwire(map, start, va); 1431 } 1432 return (rv); 1433 } 1434 } 1435 1436 return (0); 1437 } 1438 1439 /* 1440 * uvm_fault_unwire(): unwire range of virtual space. 1441 */ 1442 void 1443 uvm_fault_unwire(vm_map_t map, vaddr_t start, vaddr_t end) 1444 { 1445 1446 vm_map_lock_read(map); 1447 uvm_fault_unwire_locked(map, start, end); 1448 vm_map_unlock_read(map); 1449 } 1450 1451 /* 1452 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire(). 1453 * 1454 * => map must be at least read-locked. 1455 */ 1456 void 1457 uvm_fault_unwire_locked(vm_map_t map, vaddr_t start, vaddr_t end) 1458 { 1459 vm_map_entry_t entry, next; 1460 pmap_t pmap = vm_map_pmap(map); 1461 vaddr_t va; 1462 paddr_t pa; 1463 struct vm_page *pg; 1464 1465 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0); 1466 1467 /* 1468 * we assume that the area we are unwiring has actually been wired 1469 * in the first place. this means that we should be able to extract 1470 * the PAs from the pmap. we also lock out the page daemon so that 1471 * we can call uvm_pageunwire. 1472 */ 1473 uvm_lock_pageq(); 1474 1475 /* find the beginning map entry for the region. */ 1476 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map)); 1477 if (uvm_map_lookup_entry(map, start, &entry) == FALSE) 1478 panic("uvm_fault_unwire_locked: address not in map"); 1479 1480 for (va = start; va < end ; va += PAGE_SIZE) { 1481 if (pmap_extract(pmap, va, &pa) == FALSE) 1482 continue; 1483 1484 /* find the map entry for the current address. */ 1485 KASSERT(va >= entry->start); 1486 while (va >= entry->end) { 1487 next = RBT_NEXT(uvm_map_addr, entry); 1488 KASSERT(next != NULL && next->start <= entry->end); 1489 entry = next; 1490 } 1491 1492 /* if the entry is no longer wired, tell the pmap. */ 1493 if (VM_MAPENT_ISWIRED(entry) == 0) 1494 pmap_unwire(pmap, va); 1495 1496 pg = PHYS_TO_VM_PAGE(pa); 1497 if (pg) 1498 uvm_pageunwire(pg); 1499 } 1500 1501 uvm_unlock_pageq(); 1502 } 1503 1504 /* 1505 * uvmfault_unlockmaps: unlock the maps 1506 */ 1507 void 1508 uvmfault_unlockmaps(struct uvm_faultinfo *ufi, boolean_t write_locked) 1509 { 1510 /* 1511 * ufi can be NULL when this isn't really a fault, 1512 * but merely paging in anon data. 1513 */ 1514 if (ufi == NULL) { 1515 return; 1516 } 1517 1518 uvmfault_update_stats(ufi); 1519 if (write_locked) { 1520 vm_map_unlock(ufi->map); 1521 } else { 1522 vm_map_unlock_read(ufi->map); 1523 } 1524 } 1525 1526 /* 1527 * uvmfault_unlockall: unlock everything passed in. 1528 * 1529 * => maps must be read-locked (not write-locked). 1530 */ 1531 void 1532 uvmfault_unlockall(struct uvm_faultinfo *ufi, struct vm_amap *amap, 1533 struct uvm_object *uobj) 1534 { 1535 if (amap != NULL) 1536 amap_unlock(amap); 1537 uvmfault_unlockmaps(ufi, FALSE); 1538 } 1539 1540 /* 1541 * uvmfault_lookup: lookup a virtual address in a map 1542 * 1543 * => caller must provide a uvm_faultinfo structure with the IN 1544 * params properly filled in 1545 * => we will lookup the map entry (handling submaps) as we go 1546 * => if the lookup is a success we will return with the maps locked 1547 * => if "write_lock" is TRUE, we write_lock the map, otherwise we only 1548 * get a read lock. 1549 * => note that submaps can only appear in the kernel and they are 1550 * required to use the same virtual addresses as the map they 1551 * are referenced by (thus address translation between the main 1552 * map and the submap is unnecessary). 1553 */ 1554 1555 boolean_t 1556 uvmfault_lookup(struct uvm_faultinfo *ufi, boolean_t write_lock) 1557 { 1558 vm_map_t tmpmap; 1559 1560 /* init ufi values for lookup. */ 1561 ufi->map = ufi->orig_map; 1562 ufi->size = ufi->orig_size; 1563 1564 /* 1565 * keep going down levels until we are done. note that there can 1566 * only be two levels so we won't loop very long. 1567 */ 1568 while (1) { 1569 if (ufi->orig_rvaddr < ufi->map->min_offset || 1570 ufi->orig_rvaddr >= ufi->map->max_offset) 1571 return(FALSE); 1572 1573 /* lock map */ 1574 if (write_lock) { 1575 vm_map_lock(ufi->map); 1576 } else { 1577 vm_map_lock_read(ufi->map); 1578 } 1579 1580 /* lookup */ 1581 if (!uvm_map_lookup_entry(ufi->map, ufi->orig_rvaddr, 1582 &ufi->entry)) { 1583 uvmfault_unlockmaps(ufi, write_lock); 1584 return(FALSE); 1585 } 1586 1587 /* reduce size if necessary */ 1588 if (ufi->entry->end - ufi->orig_rvaddr < ufi->size) 1589 ufi->size = ufi->entry->end - ufi->orig_rvaddr; 1590 1591 /* 1592 * submap? replace map with the submap and lookup again. 1593 * note: VAs in submaps must match VAs in main map. 1594 */ 1595 if (UVM_ET_ISSUBMAP(ufi->entry)) { 1596 tmpmap = ufi->entry->object.sub_map; 1597 uvmfault_unlockmaps(ufi, write_lock); 1598 ufi->map = tmpmap; 1599 continue; 1600 } 1601 1602 /* got it! */ 1603 ufi->mapv = ufi->map->timestamp; 1604 return(TRUE); 1605 1606 } 1607 /*NOTREACHED*/ 1608 } 1609 1610 /* 1611 * uvmfault_relock: attempt to relock the same version of the map 1612 * 1613 * => fault data structures should be unlocked before calling. 1614 * => if a success (TRUE) maps will be locked after call. 1615 */ 1616 boolean_t 1617 uvmfault_relock(struct uvm_faultinfo *ufi) 1618 { 1619 /* 1620 * ufi can be NULL when this isn't really a fault, 1621 * but merely paging in anon data. 1622 */ 1623 if (ufi == NULL) { 1624 return TRUE; 1625 } 1626 1627 counters_inc(uvmexp_counters, flt_relck); 1628 1629 /* 1630 * relock map. fail if version mismatch (in which case nothing 1631 * gets locked). 1632 */ 1633 vm_map_lock_read(ufi->map); 1634 if (ufi->mapv != ufi->map->timestamp) { 1635 vm_map_unlock_read(ufi->map); 1636 return(FALSE); 1637 } 1638 1639 counters_inc(uvmexp_counters, flt_relckok); 1640 return(TRUE); /* got it! */ 1641 } 1642