1 /* $OpenBSD: uvm_fault.c,v 1.111 2021/01/02 02:39:59 cheloha Exp $ */ 2 /* $NetBSD: uvm_fault.c,v 1.51 2000/08/06 00:22:53 thorpej Exp $ */ 3 4 /* 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 * 28 * from: Id: uvm_fault.c,v 1.1.2.23 1998/02/06 05:29:05 chs Exp 29 */ 30 31 /* 32 * uvm_fault.c: fault handler 33 */ 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/percpu.h> 39 #include <sys/proc.h> 40 #include <sys/malloc.h> 41 #include <sys/mman.h> 42 #include <sys/tracepoint.h> 43 44 #include <uvm/uvm.h> 45 46 /* 47 * 48 * a word on page faults: 49 * 50 * types of page faults we handle: 51 * 52 * CASE 1: upper layer faults CASE 2: lower layer faults 53 * 54 * CASE 1A CASE 1B CASE 2A CASE 2B 55 * read/write1 write>1 read/write +-cow_write/zero 56 * | | | | 57 * +--|--+ +--|--+ +-----+ + | + | +-----+ 58 * amap | V | | ----------->new| | | | ^ | 59 * +-----+ +-----+ +-----+ + | + | +--|--+ 60 * | | | 61 * +-----+ +-----+ +--|--+ | +--|--+ 62 * uobj | d/c | | d/c | | V | +----| | 63 * +-----+ +-----+ +-----+ +-----+ 64 * 65 * d/c = don't care 66 * 67 * case [0]: layerless fault 68 * no amap or uobj is present. this is an error. 69 * 70 * case [1]: upper layer fault [anon active] 71 * 1A: [read] or [write with anon->an_ref == 1] 72 * I/O takes place in top level anon and uobj is not touched. 73 * 1B: [write with anon->an_ref > 1] 74 * new anon is alloc'd and data is copied off ["COW"] 75 * 76 * case [2]: lower layer fault [uobj] 77 * 2A: [read on non-NULL uobj] or [write to non-copy_on_write area] 78 * I/O takes place directly in object. 79 * 2B: [write to copy_on_write] or [read on NULL uobj] 80 * data is "promoted" from uobj to a new anon. 81 * if uobj is null, then we zero fill. 82 * 83 * we follow the standard UVM locking protocol ordering: 84 * 85 * MAPS => AMAP => UOBJ => ANON => PAGE QUEUES (PQ) 86 * we hold a PG_BUSY page if we unlock for I/O 87 * 88 * 89 * the code is structured as follows: 90 * 91 * - init the "IN" params in the ufi structure 92 * ReFault: 93 * - do lookups [locks maps], check protection, handle needs_copy 94 * - check for case 0 fault (error) 95 * - establish "range" of fault 96 * - if we have an amap lock it and extract the anons 97 * - if sequential advice deactivate pages behind us 98 * - at the same time check pmap for unmapped areas and anon for pages 99 * that we could map in (and do map it if found) 100 * - check object for resident pages that we could map in 101 * - if (case 2) goto Case2 102 * - >>> handle case 1 103 * - ensure source anon is resident in RAM 104 * - if case 1B alloc new anon and copy from source 105 * - map the correct page in 106 * Case2: 107 * - >>> handle case 2 108 * - ensure source page is resident (if uobj) 109 * - if case 2B alloc new anon and copy from source (could be zero 110 * fill if uobj == NULL) 111 * - map the correct page in 112 * - done! 113 * 114 * note on paging: 115 * if we have to do I/O we place a PG_BUSY page in the correct object, 116 * unlock everything, and do the I/O. when I/O is done we must reverify 117 * the state of the world before assuming that our data structures are 118 * valid. [because mappings could change while the map is unlocked] 119 * 120 * alternative 1: unbusy the page in question and restart the page fault 121 * from the top (ReFault). this is easy but does not take advantage 122 * of the information that we already have from our previous lookup, 123 * although it is possible that the "hints" in the vm_map will help here. 124 * 125 * alternative 2: the system already keeps track of a "version" number of 126 * a map. [i.e. every time you write-lock a map (e.g. to change a 127 * mapping) you bump the version number up by one...] so, we can save 128 * the version number of the map before we release the lock and start I/O. 129 * then when I/O is done we can relock and check the version numbers 130 * to see if anything changed. this might save us some over 1 because 131 * we don't have to unbusy the page and may be less compares(?). 132 * 133 * alternative 3: put in backpointers or a way to "hold" part of a map 134 * in place while I/O is in progress. this could be complex to 135 * implement (especially with structures like amap that can be referenced 136 * by multiple map entries, and figuring out what should wait could be 137 * complex as well...). 138 * 139 * given that we are not currently multiprocessor or multithreaded we might 140 * as well choose alternative 2 now. maybe alternative 3 would be useful 141 * in the future. XXX keep in mind for future consideration//rechecking. 142 */ 143 144 /* 145 * local data structures 146 */ 147 struct uvm_advice { 148 int nback; 149 int nforw; 150 }; 151 152 /* 153 * page range array: set up in uvmfault_init(). 154 */ 155 static struct uvm_advice uvmadvice[MADV_MASK + 1]; 156 157 #define UVM_MAXRANGE 16 /* must be max() of nback+nforw+1 */ 158 159 /* 160 * private prototypes 161 */ 162 static void uvmfault_amapcopy(struct uvm_faultinfo *); 163 static inline void uvmfault_anonflush(struct vm_anon **, int); 164 void uvmfault_unlockmaps(struct uvm_faultinfo *, boolean_t); 165 void uvmfault_update_stats(struct uvm_faultinfo *); 166 167 /* 168 * inline functions 169 */ 170 /* 171 * uvmfault_anonflush: try and deactivate pages in specified anons 172 * 173 * => does not have to deactivate page if it is busy 174 */ 175 static inline void 176 uvmfault_anonflush(struct vm_anon **anons, int n) 177 { 178 int lcv; 179 struct vm_page *pg; 180 181 for (lcv = 0 ; lcv < n ; lcv++) { 182 if (anons[lcv] == NULL) 183 continue; 184 pg = anons[lcv]->an_page; 185 if (pg && (pg->pg_flags & PG_BUSY) == 0) { 186 uvm_lock_pageq(); 187 if (pg->wire_count == 0) { 188 pmap_page_protect(pg, PROT_NONE); 189 uvm_pagedeactivate(pg); 190 } 191 uvm_unlock_pageq(); 192 } 193 } 194 } 195 196 /* 197 * normal functions 198 */ 199 /* 200 * uvmfault_init: compute proper values for the uvmadvice[] array. 201 */ 202 void 203 uvmfault_init(void) 204 { 205 int npages; 206 207 npages = atop(16384); 208 if (npages > 0) { 209 KASSERT(npages <= UVM_MAXRANGE / 2); 210 uvmadvice[MADV_NORMAL].nforw = npages; 211 uvmadvice[MADV_NORMAL].nback = npages - 1; 212 } 213 214 npages = atop(32768); 215 if (npages > 0) { 216 KASSERT(npages <= UVM_MAXRANGE / 2); 217 uvmadvice[MADV_SEQUENTIAL].nforw = npages - 1; 218 uvmadvice[MADV_SEQUENTIAL].nback = npages; 219 } 220 } 221 222 /* 223 * uvmfault_amapcopy: clear "needs_copy" in a map. 224 * 225 * => if we are out of RAM we sleep (waiting for more) 226 */ 227 static void 228 uvmfault_amapcopy(struct uvm_faultinfo *ufi) 229 { 230 231 /* while we haven't done the job */ 232 while (1) { 233 /* no mapping? give up. */ 234 if (uvmfault_lookup(ufi, TRUE) == FALSE) 235 return; 236 237 /* copy if needed. */ 238 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) 239 amap_copy(ufi->map, ufi->entry, M_NOWAIT, 240 UVM_ET_ISSTACK(ufi->entry) ? FALSE : TRUE, 241 ufi->orig_rvaddr, ufi->orig_rvaddr + 1); 242 243 /* didn't work? must be out of RAM. sleep. */ 244 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 245 uvmfault_unlockmaps(ufi, TRUE); 246 uvm_wait("fltamapcopy"); 247 continue; 248 } 249 250 /* got it! */ 251 uvmfault_unlockmaps(ufi, TRUE); 252 return; 253 } 254 /*NOTREACHED*/ 255 } 256 257 /* 258 * uvmfault_anonget: get data in an anon into a non-busy, non-released 259 * page in that anon. 260 * 261 * => we don't move the page on the queues [gets moved later] 262 * => if we allocate a new page [we_own], it gets put on the queues. 263 * either way, the result is that the page is on the queues at return time 264 */ 265 int 266 uvmfault_anonget(struct uvm_faultinfo *ufi, struct vm_amap *amap, 267 struct vm_anon *anon) 268 { 269 boolean_t we_own; /* we own anon's page? */ 270 boolean_t locked; /* did we relock? */ 271 struct vm_page *pg; 272 int result; 273 274 result = 0; /* XXX shut up gcc */ 275 counters_inc(uvmexp_counters, flt_anget); 276 /* bump rusage counters */ 277 if (anon->an_page) 278 curproc->p_ru.ru_minflt++; 279 else 280 curproc->p_ru.ru_majflt++; 281 282 /* loop until we get it, or fail. */ 283 while (1) { 284 we_own = FALSE; /* TRUE if we set PG_BUSY on a page */ 285 pg = anon->an_page; 286 287 /* page there? make sure it is not busy/released. */ 288 if (pg) { 289 KASSERT(pg->pg_flags & PQ_ANON); 290 KASSERT(pg->uanon == anon); 291 292 /* 293 * if the page is busy, we drop all the locks and 294 * try again. 295 */ 296 if ((pg->pg_flags & (PG_BUSY|PG_RELEASED)) == 0) 297 return (VM_PAGER_OK); 298 atomic_setbits_int(&pg->pg_flags, PG_WANTED); 299 counters_inc(uvmexp_counters, flt_pgwait); 300 301 /* 302 * the last unlock must be an atomic unlock+wait on 303 * the owner of page 304 */ 305 uvmfault_unlockall(ufi, amap, NULL); 306 tsleep_nsec(pg, PVM, "anonget2", INFSLP); 307 /* ready to relock and try again */ 308 } else { 309 /* no page, we must try and bring it in. */ 310 pg = uvm_pagealloc(NULL, 0, anon, 0); 311 312 if (pg == NULL) { /* out of RAM. */ 313 uvmfault_unlockall(ufi, amap, NULL); 314 counters_inc(uvmexp_counters, flt_noram); 315 uvm_wait("flt_noram1"); 316 /* ready to relock and try again */ 317 } else { 318 /* we set the PG_BUSY bit */ 319 we_own = TRUE; 320 uvmfault_unlockall(ufi, amap, NULL); 321 322 /* 323 * we are passing a PG_BUSY+PG_FAKE+PG_CLEAN 324 * page into the uvm_swap_get function with 325 * all data structures unlocked. note that 326 * it is ok to read an_swslot here because 327 * we hold PG_BUSY on the page. 328 */ 329 counters_inc(uvmexp_counters, pageins); 330 result = uvm_swap_get(pg, anon->an_swslot, 331 PGO_SYNCIO); 332 333 /* 334 * we clean up after the i/o below in the 335 * "we_own" case 336 */ 337 /* ready to relock and try again */ 338 } 339 } 340 341 /* now relock and try again */ 342 locked = uvmfault_relock(ufi); 343 344 /* 345 * if we own the page (i.e. we set PG_BUSY), then we need 346 * to clean up after the I/O. there are three cases to 347 * consider: 348 * [1] page released during I/O: free anon and ReFault. 349 * [2] I/O not OK. free the page and cause the fault 350 * to fail. 351 * [3] I/O OK! activate the page and sync with the 352 * non-we_own case (i.e. drop anon lock if not locked). 353 */ 354 if (we_own) { 355 if (pg->pg_flags & PG_WANTED) { 356 wakeup(pg); 357 } 358 /* un-busy! */ 359 atomic_clearbits_int(&pg->pg_flags, 360 PG_WANTED|PG_BUSY|PG_FAKE); 361 UVM_PAGE_OWN(pg, NULL); 362 363 /* 364 * if we were RELEASED during I/O, then our anon is 365 * no longer part of an amap. we need to free the 366 * anon and try again. 367 */ 368 if (pg->pg_flags & PG_RELEASED) { 369 pmap_page_protect(pg, PROT_NONE); 370 uvm_anfree(anon); /* frees page for us */ 371 if (locked) 372 uvmfault_unlockall(ufi, amap, NULL); 373 counters_inc(uvmexp_counters, flt_pgrele); 374 return (VM_PAGER_REFAULT); /* refault! */ 375 } 376 377 if (result != VM_PAGER_OK) { 378 KASSERT(result != VM_PAGER_PEND); 379 380 /* remove page from anon */ 381 anon->an_page = NULL; 382 383 /* 384 * remove the swap slot from the anon 385 * and mark the anon as having no real slot. 386 * don't free the swap slot, thus preventing 387 * it from being used again. 388 */ 389 uvm_swap_markbad(anon->an_swslot, 1); 390 anon->an_swslot = SWSLOT_BAD; 391 392 /* 393 * note: page was never !PG_BUSY, so it 394 * can't be mapped and thus no need to 395 * pmap_page_protect it... 396 */ 397 uvm_lock_pageq(); 398 uvm_pagefree(pg); 399 uvm_unlock_pageq(); 400 401 if (locked) 402 uvmfault_unlockall(ufi, amap, NULL); 403 return (VM_PAGER_ERROR); 404 } 405 406 /* 407 * must be OK, clear modify (already PG_CLEAN) 408 * and activate 409 */ 410 pmap_clear_modify(pg); 411 uvm_lock_pageq(); 412 uvm_pageactivate(pg); 413 uvm_unlock_pageq(); 414 } 415 416 /* we were not able to relock. restart fault. */ 417 if (!locked) 418 return (VM_PAGER_REFAULT); 419 420 /* verify no one touched the amap and moved the anon on us. */ 421 if (ufi != NULL && 422 amap_lookup(&ufi->entry->aref, 423 ufi->orig_rvaddr - ufi->entry->start) != anon) { 424 425 uvmfault_unlockall(ufi, amap, NULL); 426 return (VM_PAGER_REFAULT); 427 } 428 429 /* try it again! */ 430 counters_inc(uvmexp_counters, flt_anretry); 431 continue; 432 433 } /* while (1) */ 434 /*NOTREACHED*/ 435 } 436 437 /* 438 * Update statistics after fault resolution. 439 * - maxrss 440 */ 441 void 442 uvmfault_update_stats(struct uvm_faultinfo *ufi) 443 { 444 struct vm_map *map; 445 struct proc *p; 446 vsize_t res; 447 448 map = ufi->orig_map; 449 450 /* 451 * If this is a nested pmap (eg, a virtual machine pmap managed 452 * by vmm(4) on amd64/i386), don't do any updating, just return. 453 * 454 * pmap_nested() on other archs is #defined to 0, so this is a 455 * no-op. 456 */ 457 if (pmap_nested(map->pmap)) 458 return; 459 460 /* Update the maxrss for the process. */ 461 if (map->flags & VM_MAP_ISVMSPACE) { 462 p = curproc; 463 KASSERT(p != NULL && &p->p_vmspace->vm_map == map); 464 465 res = pmap_resident_count(map->pmap); 466 /* Convert res from pages to kilobytes. */ 467 res <<= (PAGE_SHIFT - 10); 468 469 if (p->p_ru.ru_maxrss < res) 470 p->p_ru.ru_maxrss = res; 471 } 472 } 473 474 struct uvm_faultctx { 475 /* 476 * the following members are set up by uvm_fault_check() and 477 * read-only after that. 478 */ 479 vm_prot_t enter_prot; 480 vaddr_t startva; 481 int npages; 482 int centeridx; 483 boolean_t narrow; 484 boolean_t wired; 485 paddr_t pa_flags; 486 }; 487 488 int uvm_fault_lower(struct uvm_faultinfo *, struct uvm_faultctx *, 489 struct vm_page **, vm_fault_t, vm_prot_t); 490 491 /* 492 * uvm_fault_check: check prot, handle needs-copy, etc. 493 * 494 * 1. lookup entry. 495 * 2. check protection. 496 * 3. adjust fault condition (mainly for simulated fault). 497 * 4. handle needs-copy (lazy amap copy). 498 * 5. establish range of interest for neighbor fault (aka pre-fault). 499 * 6. look up anons (if amap exists). 500 * 7. flush pages (if MADV_SEQUENTIAL) 501 * 502 * => called with nothing locked. 503 * => if we fail (result != 0) we unlock everything. 504 * => initialize/adjust many members of flt. 505 */ 506 int 507 uvm_fault_check(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 508 struct vm_anon ***ranons, vm_prot_t access_type) 509 { 510 struct vm_amap *amap; 511 struct uvm_object *uobj; 512 int nback, nforw; 513 514 /* lookup and lock the maps */ 515 if (uvmfault_lookup(ufi, FALSE) == FALSE) { 516 return (EFAULT); 517 } 518 519 #ifdef DIAGNOSTIC 520 if ((ufi->map->flags & VM_MAP_PAGEABLE) == 0) 521 panic("uvm_fault: fault on non-pageable map (%p, 0x%lx)", 522 ufi->map, ufi->orig_rvaddr); 523 #endif 524 525 /* check protection */ 526 if ((ufi->entry->protection & access_type) != access_type) { 527 uvmfault_unlockmaps(ufi, FALSE); 528 return (EACCES); 529 } 530 531 /* 532 * "enter_prot" is the protection we want to enter the page in at. 533 * for certain pages (e.g. copy-on-write pages) this protection can 534 * be more strict than ufi->entry->protection. "wired" means either 535 * the entry is wired or we are fault-wiring the pg. 536 */ 537 538 flt->enter_prot = ufi->entry->protection; 539 flt->pa_flags = UVM_ET_ISWC(ufi->entry) ? PMAP_WC : 0; 540 flt->wired = VM_MAPENT_ISWIRED(ufi->entry) || (flt->narrow == TRUE); 541 if (flt->wired) 542 access_type = flt->enter_prot; /* full access for wired */ 543 544 /* handle "needs_copy" case. */ 545 if (UVM_ET_ISNEEDSCOPY(ufi->entry)) { 546 if ((access_type & PROT_WRITE) || 547 (ufi->entry->object.uvm_obj == NULL)) { 548 /* need to clear */ 549 uvmfault_unlockmaps(ufi, FALSE); 550 uvmfault_amapcopy(ufi); 551 counters_inc(uvmexp_counters, flt_amcopy); 552 return (ERESTART); 553 } else { 554 /* 555 * ensure that we pmap_enter page R/O since 556 * needs_copy is still true 557 */ 558 flt->enter_prot &= ~PROT_WRITE; 559 } 560 } 561 562 /* identify the players */ 563 amap = ufi->entry->aref.ar_amap; /* top layer */ 564 uobj = ufi->entry->object.uvm_obj; /* bottom layer */ 565 566 /* 567 * check for a case 0 fault. if nothing backing the entry then 568 * error now. 569 */ 570 if (amap == NULL && uobj == NULL) { 571 uvmfault_unlockmaps(ufi, FALSE); 572 return (EFAULT); 573 } 574 575 /* 576 * establish range of interest based on advice from mapper 577 * and then clip to fit map entry. note that we only want 578 * to do this the first time through the fault. if we 579 * ReFault we will disable this by setting "narrow" to true. 580 */ 581 if (flt->narrow == FALSE) { 582 583 /* wide fault (!narrow) */ 584 nback = min(uvmadvice[ufi->entry->advice].nback, 585 (ufi->orig_rvaddr - ufi->entry->start) >> PAGE_SHIFT); 586 flt->startva = ufi->orig_rvaddr - ((vsize_t)nback << PAGE_SHIFT); 587 nforw = min(uvmadvice[ufi->entry->advice].nforw, 588 ((ufi->entry->end - ufi->orig_rvaddr) >> PAGE_SHIFT) - 1); 589 /* 590 * note: "-1" because we don't want to count the 591 * faulting page as forw 592 */ 593 flt->npages = nback + nforw + 1; 594 flt->centeridx = nback; 595 596 flt->narrow = TRUE; /* ensure only once per-fault */ 597 } else { 598 /* narrow fault! */ 599 nback = nforw = 0; 600 flt->startva = ufi->orig_rvaddr; 601 flt->npages = 1; 602 flt->centeridx = 0; 603 } 604 605 /* if we've got an amap, extract current anons. */ 606 if (amap) { 607 amap_lookups(&ufi->entry->aref, 608 flt->startva - ufi->entry->start, *ranons, flt->npages); 609 } else { 610 *ranons = NULL; /* to be safe */ 611 } 612 613 /* 614 * for MADV_SEQUENTIAL mappings we want to deactivate the back pages 615 * now and then forget about them (for the rest of the fault). 616 */ 617 if (ufi->entry->advice == MADV_SEQUENTIAL && nback != 0) { 618 /* flush back-page anons? */ 619 if (amap) 620 uvmfault_anonflush(*ranons, nback); 621 622 /* flush object? */ 623 if (uobj) { 624 voff_t uoff; 625 626 uoff = (flt->startva - ufi->entry->start) + ufi->entry->offset; 627 (void) uobj->pgops->pgo_flush(uobj, uoff, uoff + 628 ((vsize_t)nback << PAGE_SHIFT), PGO_DEACTIVATE); 629 } 630 631 /* now forget about the backpages */ 632 if (amap) 633 *ranons += nback; 634 flt->startva += ((vsize_t)nback << PAGE_SHIFT); 635 flt->npages -= nback; 636 flt->centeridx = 0; 637 } 638 639 return 0; 640 } 641 642 /* 643 * uvm_fault_upper: handle upper fault (case 1A & 1B) 644 * 645 * 1. get anon. let uvmfault_anonget do the dirty work. 646 * 2. if COW, promote data to new anon 647 * 3. enter h/w mapping 648 */ 649 int 650 uvm_fault_upper(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 651 struct vm_anon **anons, vm_fault_t fault_type, vm_prot_t access_type) 652 { 653 struct vm_amap *amap = ufi->entry->aref.ar_amap; 654 struct vm_anon *oanon, *anon = anons[flt->centeridx]; 655 struct vm_page *pg = NULL; 656 int error, ret; 657 658 /* 659 * no matter if we have case 1A or case 1B we are going to need to 660 * have the anon's memory resident. ensure that now. 661 */ 662 /* 663 * let uvmfault_anonget do the dirty work. 664 * also, if it is OK, then the anon's page is on the queues. 665 */ 666 error = uvmfault_anonget(ufi, amap, anon); 667 switch (error) { 668 case VM_PAGER_OK: 669 break; 670 671 case VM_PAGER_REFAULT: 672 return ERESTART; 673 674 case VM_PAGER_ERROR: 675 /* 676 * An error occured while trying to bring in the 677 * page -- this is the only error we return right 678 * now. 679 */ 680 return EACCES; /* XXX */ 681 default: 682 #ifdef DIAGNOSTIC 683 panic("uvm_fault: uvmfault_anonget -> %d", error); 684 #else 685 return EACCES; 686 #endif 687 } 688 689 /* 690 * if we are case 1B then we will need to allocate a new blank 691 * anon to transfer the data into. note that we have a lock 692 * on anon, so no one can busy or release the page until we are done. 693 * also note that the ref count can't drop to zero here because 694 * it is > 1 and we are only dropping one ref. 695 * 696 * in the (hopefully very rare) case that we are out of RAM we 697 * will wait for more RAM, and refault. 698 * 699 * if we are out of anon VM we wait for RAM to become available. 700 */ 701 702 if ((access_type & PROT_WRITE) != 0 && anon->an_ref > 1) { 703 counters_inc(uvmexp_counters, flt_acow); 704 oanon = anon; /* oanon = old */ 705 anon = uvm_analloc(); 706 if (anon) { 707 pg = uvm_pagealloc(NULL, 0, anon, 0); 708 } 709 710 /* check for out of RAM */ 711 if (anon == NULL || pg == NULL) { 712 uvmfault_unlockall(ufi, amap, NULL); 713 if (anon == NULL) 714 counters_inc(uvmexp_counters, flt_noanon); 715 else { 716 uvm_anfree(anon); 717 counters_inc(uvmexp_counters, flt_noram); 718 } 719 720 if (uvm_swapisfull()) 721 return ENOMEM; 722 723 /* out of RAM, wait for more */ 724 if (anon == NULL) 725 uvm_anwait(); 726 else 727 uvm_wait("flt_noram3"); 728 return ERESTART; 729 } 730 731 /* got all resources, replace anon with nanon */ 732 uvm_pagecopy(oanon->an_page, pg); /* pg now !PG_CLEAN */ 733 /* un-busy! new page */ 734 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE); 735 UVM_PAGE_OWN(pg, NULL); 736 ret = amap_add(&ufi->entry->aref, 737 ufi->orig_rvaddr - ufi->entry->start, anon, 1); 738 KASSERT(ret == 0); 739 740 /* deref: can not drop to zero here by defn! */ 741 oanon->an_ref--; 742 743 /* 744 * note: anon is _not_ locked, but we have the sole references 745 * to in from amap. 746 * thus, no one can get at it until we are done with it. 747 */ 748 } else { 749 counters_inc(uvmexp_counters, flt_anon); 750 oanon = anon; 751 pg = anon->an_page; 752 if (anon->an_ref > 1) /* disallow writes to ref > 1 anons */ 753 flt->enter_prot = flt->enter_prot & ~PROT_WRITE; 754 } 755 756 /* 757 * now map the page in ... 758 * XXX: old fault unlocks object before pmap_enter. this seems 759 * suspect since some other thread could blast the page out from 760 * under us between the unlock and the pmap_enter. 761 */ 762 if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr, 763 VM_PAGE_TO_PHYS(pg) | flt->pa_flags, flt->enter_prot, 764 access_type | PMAP_CANFAIL | (flt->wired ? PMAP_WIRED : 0)) != 0) { 765 /* 766 * No need to undo what we did; we can simply think of 767 * this as the pmap throwing away the mapping information. 768 * 769 * We do, however, have to go through the ReFault path, 770 * as the map may change while we're asleep. 771 */ 772 uvmfault_unlockall(ufi, amap, NULL); 773 if (uvm_swapisfull()) { 774 /* XXX instrumentation */ 775 return ENOMEM; 776 } 777 /* XXX instrumentation */ 778 uvm_wait("flt_pmfail1"); 779 return ERESTART; 780 } 781 782 /* ... update the page queues. */ 783 uvm_lock_pageq(); 784 785 if (fault_type == VM_FAULT_WIRE) { 786 uvm_pagewire(pg); 787 /* 788 * since the now-wired page cannot be paged out, 789 * release its swap resources for others to use. 790 * since an anon with no swap cannot be PG_CLEAN, 791 * clear its clean flag now. 792 */ 793 atomic_clearbits_int(&pg->pg_flags, PG_CLEAN); 794 uvm_anon_dropswap(anon); 795 } else { 796 /* activate it */ 797 uvm_pageactivate(pg); 798 } 799 800 uvm_unlock_pageq(); 801 802 /* done case 1! finish up by unlocking everything and returning success */ 803 uvmfault_unlockall(ufi, amap, NULL); 804 pmap_update(ufi->orig_map->pmap); 805 return 0; 806 } 807 808 809 /* 810 * uvm_fault_upper_lookup: look up existing h/w mapping and amap. 811 * 812 * iterate range of interest: 813 * 1. check if h/w mapping exists. if yes, we don't care 814 * 2. check if anon exists. if not, page is lower. 815 * 3. if anon exists, enter h/w mapping for neighbors. 816 */ 817 boolean_t 818 uvm_fault_upper_lookup(struct uvm_faultinfo *ufi, 819 const struct uvm_faultctx *flt, struct vm_anon **anons, 820 struct vm_page **pages) 821 { 822 struct vm_amap *amap = ufi->entry->aref.ar_amap; 823 struct vm_anon *anon; 824 boolean_t shadowed; 825 vaddr_t currva; 826 paddr_t pa; 827 int lcv; 828 829 /* 830 * map in the backpages and frontpages we found in the amap in hopes 831 * of preventing future faults. we also init the pages[] array as 832 * we go. 833 */ 834 currva = flt->startva; 835 shadowed = FALSE; 836 for (lcv = 0 ; lcv < flt->npages ; lcv++, currva += PAGE_SIZE) { 837 /* 838 * dont play with VAs that are already mapped 839 * except for center) 840 */ 841 if (lcv != flt->centeridx && 842 pmap_extract(ufi->orig_map->pmap, currva, &pa)) { 843 pages[lcv] = PGO_DONTCARE; 844 continue; 845 } 846 847 /* unmapped or center page. check if any anon at this level. */ 848 if (amap == NULL || anons[lcv] == NULL) { 849 pages[lcv] = NULL; 850 continue; 851 } 852 853 /* check for present page and map if possible. re-activate it */ 854 pages[lcv] = PGO_DONTCARE; 855 if (lcv == flt->centeridx) { /* save center for later! */ 856 shadowed = TRUE; 857 continue; 858 } 859 anon = anons[lcv]; 860 if (anon->an_page && 861 (anon->an_page->pg_flags & (PG_RELEASED|PG_BUSY)) == 0) { 862 uvm_lock_pageq(); 863 uvm_pageactivate(anon->an_page); /* reactivate */ 864 uvm_unlock_pageq(); 865 counters_inc(uvmexp_counters, flt_namap); 866 867 /* 868 * Since this isn't the page that's actually faulting, 869 * ignore pmap_enter() failures; it's not critical 870 * that we enter these right now. 871 */ 872 (void) pmap_enter(ufi->orig_map->pmap, currva, 873 VM_PAGE_TO_PHYS(anon->an_page) | flt->pa_flags, 874 (anon->an_ref > 1) ? 875 (flt->enter_prot & ~PROT_WRITE) : flt->enter_prot, 876 PMAP_CANFAIL | 877 (VM_MAPENT_ISWIRED(ufi->entry) ? PMAP_WIRED : 0)); 878 } 879 } 880 if (flt->npages > 1) 881 pmap_update(ufi->orig_map->pmap); 882 883 return shadowed; 884 } 885 886 /* 887 * F A U L T - m a i n e n t r y p o i n t 888 */ 889 890 /* 891 * uvm_fault: page fault handler 892 * 893 * => called from MD code to resolve a page fault 894 * => VM data structures usually should be unlocked. however, it is 895 * possible to call here with the main map locked if the caller 896 * gets a write lock, sets it recursive, and then calls us (c.f. 897 * uvm_map_pageable). this should be avoided because it keeps 898 * the map locked off during I/O. 899 */ 900 #define MASK(entry) (UVM_ET_ISCOPYONWRITE(entry) ? \ 901 ~PROT_WRITE : PROT_MASK) 902 int 903 uvm_fault(vm_map_t orig_map, vaddr_t vaddr, vm_fault_t fault_type, 904 vm_prot_t access_type) 905 { 906 struct uvm_faultinfo ufi; 907 struct uvm_faultctx flt; 908 boolean_t shadowed; 909 struct vm_anon *anons_store[UVM_MAXRANGE], **anons; 910 struct vm_page *pages[UVM_MAXRANGE]; 911 int error = ERESTART; 912 913 counters_inc(uvmexp_counters, faults); 914 TRACEPOINT(uvm, fault, vaddr, fault_type, access_type, NULL); 915 916 /* init the IN parameters in the ufi */ 917 ufi.orig_map = orig_map; 918 ufi.orig_rvaddr = trunc_page(vaddr); 919 ufi.orig_size = PAGE_SIZE; /* can't get any smaller than this */ 920 if (fault_type == VM_FAULT_WIRE) 921 flt.narrow = TRUE; /* don't look for neighborhood 922 * pages on wire */ 923 else 924 flt.narrow = FALSE; /* normal fault */ 925 926 927 /* 928 * ReFault 929 */ 930 while (error == ERESTART) { 931 anons = anons_store; 932 933 error = uvm_fault_check(&ufi, &flt, &anons, access_type); 934 if (error != 0) 935 continue; 936 937 /* True if there is an anon at the faulting address */ 938 shadowed = uvm_fault_upper_lookup(&ufi, &flt, anons, pages); 939 if (shadowed == TRUE) { 940 /* case 1: fault on an anon in our amap */ 941 error = uvm_fault_upper(&ufi, &flt, anons, fault_type, 942 access_type); 943 } else { 944 /* case 2: fault on backing object or zero fill */ 945 KERNEL_LOCK(); 946 error = uvm_fault_lower(&ufi, &flt, pages, fault_type, 947 access_type); 948 KERNEL_UNLOCK(); 949 } 950 } 951 952 return error; 953 } 954 955 int 956 uvm_fault_lower(struct uvm_faultinfo *ufi, struct uvm_faultctx *flt, 957 struct vm_page **pages, vm_fault_t fault_type, vm_prot_t access_type) 958 { 959 struct vm_amap *amap = ufi->entry->aref.ar_amap; 960 struct uvm_object *uobj = ufi->entry->object.uvm_obj; 961 boolean_t promote, locked; 962 int result, lcv, gotpages; 963 struct vm_page *uobjpage, *pg = NULL; 964 struct vm_anon *anon = NULL; 965 vaddr_t currva; 966 voff_t uoff; 967 968 /* 969 * if the desired page is not shadowed by the amap and we have a 970 * backing object, then we check to see if the backing object would 971 * prefer to handle the fault itself (rather than letting us do it 972 * with the usual pgo_get hook). the backing object signals this by 973 * providing a pgo_fault routine. 974 */ 975 if (uobj != NULL && uobj->pgops->pgo_fault != NULL) { 976 result = uobj->pgops->pgo_fault(ufi, flt->startva, pages, 977 flt->npages, flt->centeridx, fault_type, access_type, 978 PGO_LOCKED); 979 980 if (result == VM_PAGER_OK) 981 return (0); /* pgo_fault did pmap enter */ 982 else if (result == VM_PAGER_REFAULT) 983 return ERESTART; /* try again! */ 984 else 985 return (EACCES); 986 } 987 988 /* 989 * now, if the desired page is not shadowed by the amap and we have 990 * a backing object that does not have a special fault routine, then 991 * we ask (with pgo_get) the object for resident pages that we care 992 * about and attempt to map them in. we do not let pgo_get block 993 * (PGO_LOCKED). 994 * 995 * ("get" has the option of doing a pmap_enter for us) 996 */ 997 if (uobj != NULL) { 998 counters_inc(uvmexp_counters, flt_lget); 999 gotpages = flt->npages; 1000 (void) uobj->pgops->pgo_get(uobj, ufi->entry->offset + 1001 (flt->startva - ufi->entry->start), 1002 pages, &gotpages, flt->centeridx, 1003 access_type & MASK(ufi->entry), 1004 ufi->entry->advice, PGO_LOCKED); 1005 1006 /* check for pages to map, if we got any */ 1007 uobjpage = NULL; 1008 if (gotpages) { 1009 currva = flt->startva; 1010 for (lcv = 0 ; lcv < flt->npages ; 1011 lcv++, currva += PAGE_SIZE) { 1012 if (pages[lcv] == NULL || 1013 pages[lcv] == PGO_DONTCARE) 1014 continue; 1015 1016 KASSERT((pages[lcv]->pg_flags & PG_RELEASED) == 0); 1017 1018 /* 1019 * if center page is resident and not 1020 * PG_BUSY, then pgo_get made it PG_BUSY 1021 * for us and gave us a handle to it. 1022 * remember this page as "uobjpage." 1023 * (for later use). 1024 */ 1025 if (lcv == flt->centeridx) { 1026 uobjpage = pages[lcv]; 1027 continue; 1028 } 1029 1030 /* 1031 * note: calling pgo_get with locked data 1032 * structures returns us pages which are 1033 * neither busy nor released, so we don't 1034 * need to check for this. we can just 1035 * directly enter the page (after moving it 1036 * to the head of the active queue [useful?]). 1037 */ 1038 1039 uvm_lock_pageq(); 1040 uvm_pageactivate(pages[lcv]); /* reactivate */ 1041 uvm_unlock_pageq(); 1042 counters_inc(uvmexp_counters, flt_nomap); 1043 1044 /* 1045 * Since this page isn't the page that's 1046 * actually faulting, ignore pmap_enter() 1047 * failures; it's not critical that we 1048 * enter these right now. 1049 */ 1050 (void) pmap_enter(ufi->orig_map->pmap, currva, 1051 VM_PAGE_TO_PHYS(pages[lcv]) | flt->pa_flags, 1052 flt->enter_prot & MASK(ufi->entry), 1053 PMAP_CANFAIL | 1054 (flt->wired ? PMAP_WIRED : 0)); 1055 1056 /* 1057 * NOTE: page can't be PG_WANTED because 1058 * we've held the lock the whole time 1059 * we've had the handle. 1060 */ 1061 atomic_clearbits_int(&pages[lcv]->pg_flags, 1062 PG_BUSY); 1063 UVM_PAGE_OWN(pages[lcv], NULL); 1064 } /* for "lcv" loop */ 1065 pmap_update(ufi->orig_map->pmap); 1066 } /* "gotpages" != 0 */ 1067 /* note: object still _locked_ */ 1068 } else { 1069 uobjpage = NULL; 1070 } 1071 1072 /* 1073 * note that at this point we are done with any front or back pages. 1074 * we are now going to focus on the center page (i.e. the one we've 1075 * faulted on). if we have faulted on the bottom (uobj) 1076 * layer [i.e. case 2] and the page was both present and available, 1077 * then we've got a pointer to it as "uobjpage" and we've already 1078 * made it BUSY. 1079 */ 1080 1081 /* 1082 * note that uobjpage can not be PGO_DONTCARE at this point. we now 1083 * set uobjpage to PGO_DONTCARE if we are doing a zero fill. if we 1084 * have a backing object, check and see if we are going to promote 1085 * the data up to an anon during the fault. 1086 */ 1087 if (uobj == NULL) { 1088 uobjpage = PGO_DONTCARE; 1089 promote = TRUE; /* always need anon here */ 1090 } else { 1091 KASSERT(uobjpage != PGO_DONTCARE); 1092 promote = (access_type & PROT_WRITE) && 1093 UVM_ET_ISCOPYONWRITE(ufi->entry); 1094 } 1095 1096 /* 1097 * if uobjpage is not null then we do not need to do I/O to get the 1098 * uobjpage. 1099 * 1100 * if uobjpage is null, then we need to ask the pager to 1101 * get the data for us. once we have the data, we need to reverify 1102 * the state the world. we are currently not holding any resources. 1103 */ 1104 if (uobjpage) { 1105 /* update rusage counters */ 1106 curproc->p_ru.ru_minflt++; 1107 } else { 1108 /* update rusage counters */ 1109 curproc->p_ru.ru_majflt++; 1110 1111 uvmfault_unlockall(ufi, amap, NULL); 1112 1113 counters_inc(uvmexp_counters, flt_get); 1114 gotpages = 1; 1115 uoff = (ufi->orig_rvaddr - ufi->entry->start) + ufi->entry->offset; 1116 result = uobj->pgops->pgo_get(uobj, uoff, &uobjpage, &gotpages, 1117 0, access_type & MASK(ufi->entry), ufi->entry->advice, 1118 PGO_SYNCIO); 1119 1120 /* recover from I/O */ 1121 if (result != VM_PAGER_OK) { 1122 KASSERT(result != VM_PAGER_PEND); 1123 1124 if (result == VM_PAGER_AGAIN) { 1125 tsleep_nsec(&nowake, PVM, "fltagain2", 1126 SEC_TO_NSEC(1)); 1127 return ERESTART; 1128 } 1129 1130 if (!UVM_ET_ISNOFAULT(ufi->entry)) 1131 return (EIO); 1132 1133 uobjpage = PGO_DONTCARE; 1134 promote = TRUE; 1135 } 1136 1137 /* re-verify the state of the world. */ 1138 locked = uvmfault_relock(ufi); 1139 1140 /* 1141 * Re-verify that amap slot is still free. if there is 1142 * a problem, we clean up. 1143 */ 1144 if (locked && amap && amap_lookup(&ufi->entry->aref, 1145 ufi->orig_rvaddr - ufi->entry->start)) { 1146 if (locked) 1147 uvmfault_unlockall(ufi, amap, NULL); 1148 locked = FALSE; 1149 } 1150 1151 /* didn't get the lock? release the page and retry. */ 1152 if (locked == FALSE && uobjpage != PGO_DONTCARE) { 1153 uvm_lock_pageq(); 1154 /* make sure it is in queues */ 1155 uvm_pageactivate(uobjpage); 1156 uvm_unlock_pageq(); 1157 1158 if (uobjpage->pg_flags & PG_WANTED) 1159 /* still holding object lock */ 1160 wakeup(uobjpage); 1161 atomic_clearbits_int(&uobjpage->pg_flags, 1162 PG_BUSY|PG_WANTED); 1163 UVM_PAGE_OWN(uobjpage, NULL); 1164 return ERESTART; 1165 } 1166 if (locked == FALSE) 1167 return ERESTART; 1168 1169 /* 1170 * we have the data in uobjpage which is PG_BUSY 1171 */ 1172 } 1173 1174 /* 1175 * notes: 1176 * - at this point uobjpage can not be NULL 1177 * - at this point uobjpage could be PG_WANTED (handle later) 1178 */ 1179 if (promote == FALSE) { 1180 /* 1181 * we are not promoting. if the mapping is COW ensure that we 1182 * don't give more access than we should (e.g. when doing a read 1183 * fault on a COPYONWRITE mapping we want to map the COW page in 1184 * R/O even though the entry protection could be R/W). 1185 * 1186 * set "pg" to the page we want to map in (uobjpage, usually) 1187 */ 1188 counters_inc(uvmexp_counters, flt_obj); 1189 if (UVM_ET_ISCOPYONWRITE(ufi->entry)) 1190 flt->enter_prot &= ~PROT_WRITE; 1191 pg = uobjpage; /* map in the actual object */ 1192 1193 /* assert(uobjpage != PGO_DONTCARE) */ 1194 1195 /* 1196 * we are faulting directly on the page. 1197 */ 1198 } else { 1199 /* 1200 * if we are going to promote the data to an anon we 1201 * allocate a blank anon here and plug it into our amap. 1202 */ 1203 #ifdef DIAGNOSTIC 1204 if (amap == NULL) 1205 panic("uvm_fault: want to promote data, but no anon"); 1206 #endif 1207 1208 anon = uvm_analloc(); 1209 if (anon) { 1210 /* 1211 * In `Fill in data...' below, if 1212 * uobjpage == PGO_DONTCARE, we want 1213 * a zero'd, dirty page, so have 1214 * uvm_pagealloc() do that for us. 1215 */ 1216 pg = uvm_pagealloc(NULL, 0, anon, 1217 (uobjpage == PGO_DONTCARE) ? UVM_PGA_ZERO : 0); 1218 } 1219 1220 /* 1221 * out of memory resources? 1222 */ 1223 if (anon == NULL || pg == NULL) { 1224 /* arg! must unbusy our page and fail or sleep. */ 1225 if (uobjpage != PGO_DONTCARE) { 1226 uvm_lock_pageq(); 1227 uvm_pageactivate(uobjpage); 1228 uvm_unlock_pageq(); 1229 1230 if (uobjpage->pg_flags & PG_WANTED) 1231 wakeup(uobjpage); 1232 atomic_clearbits_int(&uobjpage->pg_flags, 1233 PG_BUSY|PG_WANTED); 1234 UVM_PAGE_OWN(uobjpage, NULL); 1235 } 1236 1237 /* unlock and fail ... */ 1238 uvmfault_unlockall(ufi, amap, uobj); 1239 if (anon == NULL) 1240 counters_inc(uvmexp_counters, flt_noanon); 1241 else { 1242 uvm_anfree(anon); 1243 counters_inc(uvmexp_counters, flt_noram); 1244 } 1245 1246 if (uvm_swapisfull()) 1247 return (ENOMEM); 1248 1249 /* out of RAM, wait for more */ 1250 if (anon == NULL) 1251 uvm_anwait(); 1252 else 1253 uvm_wait("flt_noram5"); 1254 return ERESTART; 1255 } 1256 1257 /* fill in the data */ 1258 if (uobjpage != PGO_DONTCARE) { 1259 counters_inc(uvmexp_counters, flt_prcopy); 1260 /* copy page [pg now dirty] */ 1261 uvm_pagecopy(uobjpage, pg); 1262 1263 /* 1264 * promote to shared amap? make sure all sharing 1265 * procs see it 1266 */ 1267 if ((amap_flags(amap) & AMAP_SHARED) != 0) { 1268 pmap_page_protect(uobjpage, PROT_NONE); 1269 } 1270 1271 /* dispose of uobjpage. drop handle to uobj as well. */ 1272 if (uobjpage->pg_flags & PG_WANTED) 1273 wakeup(uobjpage); 1274 atomic_clearbits_int(&uobjpage->pg_flags, 1275 PG_BUSY|PG_WANTED); 1276 UVM_PAGE_OWN(uobjpage, NULL); 1277 uvm_lock_pageq(); 1278 uvm_pageactivate(uobjpage); 1279 uvm_unlock_pageq(); 1280 uobj = NULL; 1281 } else { 1282 counters_inc(uvmexp_counters, flt_przero); 1283 /* 1284 * Page is zero'd and marked dirty by uvm_pagealloc() 1285 * above. 1286 */ 1287 } 1288 1289 if (amap_add(&ufi->entry->aref, 1290 ufi->orig_rvaddr - ufi->entry->start, anon, 0)) { 1291 uvmfault_unlockall(ufi, amap, NULL); 1292 uvm_anfree(anon); 1293 counters_inc(uvmexp_counters, flt_noamap); 1294 1295 if (uvm_swapisfull()) 1296 return (ENOMEM); 1297 1298 amap_populate(&ufi->entry->aref, 1299 ufi->orig_rvaddr - ufi->entry->start); 1300 return ERESTART; 1301 } 1302 } 1303 1304 /* note: pg is either the uobjpage or the new page in the new anon */ 1305 /* 1306 * all resources are present. we can now map it in and free our 1307 * resources. 1308 */ 1309 if (pmap_enter(ufi->orig_map->pmap, ufi->orig_rvaddr, 1310 VM_PAGE_TO_PHYS(pg) | flt->pa_flags, flt->enter_prot, 1311 access_type | PMAP_CANFAIL | (flt->wired ? PMAP_WIRED : 0)) != 0) { 1312 /* 1313 * No need to undo what we did; we can simply think of 1314 * this as the pmap throwing away the mapping information. 1315 * 1316 * We do, however, have to go through the ReFault path, 1317 * as the map may change while we're asleep. 1318 */ 1319 if (pg->pg_flags & PG_WANTED) 1320 wakeup(pg); 1321 1322 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED); 1323 UVM_PAGE_OWN(pg, NULL); 1324 uvmfault_unlockall(ufi, amap, uobj); 1325 if (uvm_swapisfull()) { 1326 /* XXX instrumentation */ 1327 return (ENOMEM); 1328 } 1329 /* XXX instrumentation */ 1330 uvm_wait("flt_pmfail2"); 1331 return ERESTART; 1332 } 1333 1334 uvm_lock_pageq(); 1335 1336 if (fault_type == VM_FAULT_WIRE) { 1337 uvm_pagewire(pg); 1338 if (pg->pg_flags & PQ_AOBJ) { 1339 /* 1340 * since the now-wired page cannot be paged out, 1341 * release its swap resources for others to use. 1342 * since an aobj page with no swap cannot be PG_CLEAN, 1343 * clear its clean flag now. 1344 */ 1345 atomic_clearbits_int(&pg->pg_flags, PG_CLEAN); 1346 uao_dropswap(uobj, pg->offset >> PAGE_SHIFT); 1347 } 1348 } else { 1349 /* activate it */ 1350 uvm_pageactivate(pg); 1351 } 1352 uvm_unlock_pageq(); 1353 1354 if (pg->pg_flags & PG_WANTED) 1355 wakeup(pg); 1356 1357 atomic_clearbits_int(&pg->pg_flags, PG_BUSY|PG_FAKE|PG_WANTED); 1358 UVM_PAGE_OWN(pg, NULL); 1359 uvmfault_unlockall(ufi, amap, uobj); 1360 pmap_update(ufi->orig_map->pmap); 1361 1362 return (0); 1363 } 1364 1365 1366 /* 1367 * uvm_fault_wire: wire down a range of virtual addresses in a map. 1368 * 1369 * => map may be read-locked by caller, but MUST NOT be write-locked. 1370 * => if map is read-locked, any operations which may cause map to 1371 * be write-locked in uvm_fault() must be taken care of by 1372 * the caller. See uvm_map_pageable(). 1373 */ 1374 int 1375 uvm_fault_wire(vm_map_t map, vaddr_t start, vaddr_t end, vm_prot_t access_type) 1376 { 1377 vaddr_t va; 1378 int rv; 1379 1380 /* 1381 * now fault it in a page at a time. if the fault fails then we have 1382 * to undo what we have done. note that in uvm_fault PROT_NONE 1383 * is replaced with the max protection if fault_type is VM_FAULT_WIRE. 1384 */ 1385 for (va = start ; va < end ; va += PAGE_SIZE) { 1386 rv = uvm_fault(map, va, VM_FAULT_WIRE, access_type); 1387 if (rv) { 1388 if (va != start) { 1389 uvm_fault_unwire(map, start, va); 1390 } 1391 return (rv); 1392 } 1393 } 1394 1395 return (0); 1396 } 1397 1398 /* 1399 * uvm_fault_unwire(): unwire range of virtual space. 1400 */ 1401 void 1402 uvm_fault_unwire(vm_map_t map, vaddr_t start, vaddr_t end) 1403 { 1404 1405 vm_map_lock_read(map); 1406 uvm_fault_unwire_locked(map, start, end); 1407 vm_map_unlock_read(map); 1408 } 1409 1410 /* 1411 * uvm_fault_unwire_locked(): the guts of uvm_fault_unwire(). 1412 * 1413 * => map must be at least read-locked. 1414 */ 1415 void 1416 uvm_fault_unwire_locked(vm_map_t map, vaddr_t start, vaddr_t end) 1417 { 1418 vm_map_entry_t entry, next; 1419 pmap_t pmap = vm_map_pmap(map); 1420 vaddr_t va; 1421 paddr_t pa; 1422 struct vm_page *pg; 1423 1424 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0); 1425 1426 /* 1427 * we assume that the area we are unwiring has actually been wired 1428 * in the first place. this means that we should be able to extract 1429 * the PAs from the pmap. we also lock out the page daemon so that 1430 * we can call uvm_pageunwire. 1431 */ 1432 uvm_lock_pageq(); 1433 1434 /* find the beginning map entry for the region. */ 1435 KASSERT(start >= vm_map_min(map) && end <= vm_map_max(map)); 1436 if (uvm_map_lookup_entry(map, start, &entry) == FALSE) 1437 panic("uvm_fault_unwire_locked: address not in map"); 1438 1439 for (va = start; va < end ; va += PAGE_SIZE) { 1440 if (pmap_extract(pmap, va, &pa) == FALSE) 1441 continue; 1442 1443 /* find the map entry for the current address. */ 1444 KASSERT(va >= entry->start); 1445 while (va >= entry->end) { 1446 next = RBT_NEXT(uvm_map_addr, entry); 1447 KASSERT(next != NULL && next->start <= entry->end); 1448 entry = next; 1449 } 1450 1451 /* if the entry is no longer wired, tell the pmap. */ 1452 if (VM_MAPENT_ISWIRED(entry) == 0) 1453 pmap_unwire(pmap, va); 1454 1455 pg = PHYS_TO_VM_PAGE(pa); 1456 if (pg) 1457 uvm_pageunwire(pg); 1458 } 1459 1460 uvm_unlock_pageq(); 1461 } 1462 1463 /* 1464 * uvmfault_unlockmaps: unlock the maps 1465 */ 1466 void 1467 uvmfault_unlockmaps(struct uvm_faultinfo *ufi, boolean_t write_locked) 1468 { 1469 /* 1470 * ufi can be NULL when this isn't really a fault, 1471 * but merely paging in anon data. 1472 */ 1473 if (ufi == NULL) { 1474 return; 1475 } 1476 1477 uvmfault_update_stats(ufi); 1478 if (write_locked) { 1479 vm_map_unlock(ufi->map); 1480 } else { 1481 vm_map_unlock_read(ufi->map); 1482 } 1483 } 1484 1485 /* 1486 * uvmfault_unlockall: unlock everything passed in. 1487 * 1488 * => maps must be read-locked (not write-locked). 1489 */ 1490 void 1491 uvmfault_unlockall(struct uvm_faultinfo *ufi, struct vm_amap *amap, 1492 struct uvm_object *uobj) 1493 { 1494 1495 uvmfault_unlockmaps(ufi, FALSE); 1496 } 1497 1498 /* 1499 * uvmfault_lookup: lookup a virtual address in a map 1500 * 1501 * => caller must provide a uvm_faultinfo structure with the IN 1502 * params properly filled in 1503 * => we will lookup the map entry (handling submaps) as we go 1504 * => if the lookup is a success we will return with the maps locked 1505 * => if "write_lock" is TRUE, we write_lock the map, otherwise we only 1506 * get a read lock. 1507 * => note that submaps can only appear in the kernel and they are 1508 * required to use the same virtual addresses as the map they 1509 * are referenced by (thus address translation between the main 1510 * map and the submap is unnecessary). 1511 */ 1512 1513 boolean_t 1514 uvmfault_lookup(struct uvm_faultinfo *ufi, boolean_t write_lock) 1515 { 1516 vm_map_t tmpmap; 1517 1518 /* init ufi values for lookup. */ 1519 ufi->map = ufi->orig_map; 1520 ufi->size = ufi->orig_size; 1521 1522 /* 1523 * keep going down levels until we are done. note that there can 1524 * only be two levels so we won't loop very long. 1525 */ 1526 while (1) { 1527 if (ufi->orig_rvaddr < ufi->map->min_offset || 1528 ufi->orig_rvaddr >= ufi->map->max_offset) 1529 return(FALSE); 1530 1531 /* lock map */ 1532 if (write_lock) { 1533 vm_map_lock(ufi->map); 1534 } else { 1535 vm_map_lock_read(ufi->map); 1536 } 1537 1538 /* lookup */ 1539 if (!uvm_map_lookup_entry(ufi->map, ufi->orig_rvaddr, 1540 &ufi->entry)) { 1541 uvmfault_unlockmaps(ufi, write_lock); 1542 return(FALSE); 1543 } 1544 1545 /* reduce size if necessary */ 1546 if (ufi->entry->end - ufi->orig_rvaddr < ufi->size) 1547 ufi->size = ufi->entry->end - ufi->orig_rvaddr; 1548 1549 /* 1550 * submap? replace map with the submap and lookup again. 1551 * note: VAs in submaps must match VAs in main map. 1552 */ 1553 if (UVM_ET_ISSUBMAP(ufi->entry)) { 1554 tmpmap = ufi->entry->object.sub_map; 1555 uvmfault_unlockmaps(ufi, write_lock); 1556 ufi->map = tmpmap; 1557 continue; 1558 } 1559 1560 /* got it! */ 1561 ufi->mapv = ufi->map->timestamp; 1562 return(TRUE); 1563 1564 } 1565 /*NOTREACHED*/ 1566 } 1567 1568 /* 1569 * uvmfault_relock: attempt to relock the same version of the map 1570 * 1571 * => fault data structures should be unlocked before calling. 1572 * => if a success (TRUE) maps will be locked after call. 1573 */ 1574 boolean_t 1575 uvmfault_relock(struct uvm_faultinfo *ufi) 1576 { 1577 /* 1578 * ufi can be NULL when this isn't really a fault, 1579 * but merely paging in anon data. 1580 */ 1581 if (ufi == NULL) { 1582 return TRUE; 1583 } 1584 1585 counters_inc(uvmexp_counters, flt_relck); 1586 1587 /* 1588 * relock map. fail if version mismatch (in which case nothing 1589 * gets locked). 1590 */ 1591 vm_map_lock_read(ufi->map); 1592 if (ufi->mapv != ufi->map->timestamp) { 1593 vm_map_unlock_read(ufi->map); 1594 return(FALSE); 1595 } 1596 1597 counters_inc(uvmexp_counters, flt_relckok); 1598 return(TRUE); /* got it! */ 1599 } 1600