1 /* $NetBSD: uvm_loan.c,v 1.42 2003/05/03 17:54:32 yamt Exp $ */ 2 3 /* 4 * 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by Charles D. Cranor and 19 * Washington University. 20 * 4. The name of the author may not be used to endorse or promote products 21 * derived from this software without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 24 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 25 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 26 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 27 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 28 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 32 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 * 34 * from: Id: uvm_loan.c,v 1.1.6.4 1998/02/06 05:08:43 chs Exp 35 */ 36 37 /* 38 * uvm_loan.c: page loanout handler 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: uvm_loan.c,v 1.42 2003/05/03 17:54:32 yamt Exp $"); 43 44 #include <sys/param.h> 45 #include <sys/systm.h> 46 #include <sys/kernel.h> 47 #include <sys/proc.h> 48 #include <sys/malloc.h> 49 #include <sys/mman.h> 50 51 #include <uvm/uvm.h> 52 53 /* 54 * "loaned" pages are pages which are (read-only, copy-on-write) loaned 55 * from the VM system to other parts of the kernel. this allows page 56 * copying to be avoided (e.g. you can loan pages from objs/anons to 57 * the mbuf system). 58 * 59 * there are 3 types of loans possible: 60 * O->K uvm_object page to wired kernel page (e.g. mbuf data area) 61 * A->K anon page to wired kernel page (e.g. mbuf data area) 62 * O->A uvm_object to anon loan (e.g. vnode page to an anon) 63 * note that it possible to have an O page loaned to both an A and K 64 * at the same time. 65 * 66 * loans are tracked by pg->loan_count. an O->A page will have both 67 * a uvm_object and a vm_anon, but PQ_ANON will not be set. this sort 68 * of page is considered "owned" by the uvm_object (not the anon). 69 * 70 * each loan of a page to the kernel bumps the pg->wire_count. the 71 * kernel mappings for these pages will be read-only and wired. since 72 * the page will also be wired, it will not be a candidate for pageout, 73 * and thus will never be pmap_page_protect()'d with VM_PROT_NONE. a 74 * write fault in the kernel to one of these pages will not cause 75 * copy-on-write. instead, the page fault is considered fatal. this 76 * is because the kernel mapping will have no way to look up the 77 * object/anon which the page is owned by. this is a good side-effect, 78 * since a kernel write to a loaned page is an error. 79 * 80 * owners that want to free their pages and discover that they are 81 * loaned out simply "disown" them (the page becomes an orphan). these 82 * pages should be freed when the last loan is dropped. in some cases 83 * an anon may "adopt" an orphaned page. 84 * 85 * locking: to read pg->loan_count either the owner or the page queues 86 * must be locked. to modify pg->loan_count, both the owner of the page 87 * and the PQs must be locked. pg->flags is (as always) locked by 88 * the owner of the page. 89 * 90 * note that locking from the "loaned" side is tricky since the object 91 * getting the loaned page has no reference to the page's owner and thus 92 * the owner could "die" at any time. in order to prevent the owner 93 * from dying the page queues should be locked. this forces us to sometimes 94 * use "try" locking. 95 * 96 * loans are typically broken by the following events: 97 * 1. user-level xwrite fault to a loaned page 98 * 2. pageout of clean+inactive O->A loaned page 99 * 3. owner frees page (e.g. pager flush) 100 * 101 * note that loaning a page causes all mappings of the page to become 102 * read-only (via pmap_page_protect). this could have an unexpected 103 * effect on normal "wired" pages if one is not careful (XXX). 104 */ 105 106 /* 107 * local prototypes 108 */ 109 110 static int uvm_loananon __P((struct uvm_faultinfo *, void ***, 111 int, struct vm_anon *)); 112 static int uvm_loanentry __P((struct uvm_faultinfo *, void ***, int)); 113 static int uvm_loanuobj __P((struct uvm_faultinfo *, void ***, 114 int, vaddr_t)); 115 static int uvm_loanzero __P((struct uvm_faultinfo *, void ***, int)); 116 static void uvm_unloananon __P((struct vm_anon **, int)); 117 static void uvm_unloanpage __P((struct vm_page **, int)); 118 119 120 /* 121 * inlines 122 */ 123 124 /* 125 * uvm_loanentry: loan out pages in a map entry (helper fn for uvm_loan()) 126 * 127 * => "ufi" is the result of a successful map lookup (meaning that 128 * on entry the map is locked by the caller) 129 * => we may unlock and then relock the map if needed (for I/O) 130 * => we put our output result in "output" 131 * => we always return with the map unlocked 132 * => possible return values: 133 * -1 == error, map is unlocked 134 * 0 == map relock error (try again!), map is unlocked 135 * >0 == number of pages we loaned, map is unlocked 136 */ 137 138 static __inline int 139 uvm_loanentry(ufi, output, flags) 140 struct uvm_faultinfo *ufi; 141 void ***output; 142 int flags; 143 { 144 vaddr_t curaddr = ufi->orig_rvaddr; 145 vsize_t togo = ufi->size; 146 struct vm_aref *aref = &ufi->entry->aref; 147 struct uvm_object *uobj = ufi->entry->object.uvm_obj; 148 struct vm_anon *anon; 149 int rv, result = 0; 150 151 /* 152 * lock us the rest of the way down (we unlock before return) 153 */ 154 if (aref->ar_amap) 155 amap_lock(aref->ar_amap); 156 if (uobj) 157 simple_lock(&uobj->vmobjlock); 158 159 /* 160 * loop until done 161 */ 162 while (togo) { 163 164 /* 165 * find the page we want. check the anon layer first. 166 */ 167 168 if (aref->ar_amap) { 169 anon = amap_lookup(aref, curaddr - ufi->entry->start); 170 } else { 171 anon = NULL; 172 } 173 174 /* locked: map, amap, uobj */ 175 if (anon) { 176 rv = uvm_loananon(ufi, output, flags, anon); 177 } else if (uobj) { 178 rv = uvm_loanuobj(ufi, output, flags, curaddr); 179 } else if (UVM_ET_ISCOPYONWRITE(ufi->entry)) { 180 rv = uvm_loanzero(ufi, output, flags); 181 } else { 182 rv = -1; 183 } 184 /* locked: if (rv > 0) => map, amap, uobj [o.w. unlocked] */ 185 186 /* total failure */ 187 if (rv < 0) 188 return (-1); 189 190 /* relock failed, need to do another lookup */ 191 if (rv == 0) 192 return (result); 193 194 /* 195 * got it... advance to next page 196 */ 197 198 result++; 199 togo -= PAGE_SIZE; 200 curaddr += PAGE_SIZE; 201 } 202 203 /* 204 * unlock what we locked, unlock the maps and return 205 */ 206 207 if (aref->ar_amap) 208 amap_unlock(aref->ar_amap); 209 if (uobj) 210 simple_unlock(&uobj->vmobjlock); 211 uvmfault_unlockmaps(ufi, FALSE); 212 return (result); 213 } 214 215 /* 216 * normal functions 217 */ 218 219 /* 220 * uvm_loan: loan pages in a map out to anons or to the kernel 221 * 222 * => map should be unlocked 223 * => start and len should be multiples of PAGE_SIZE 224 * => result is either an array of anon's or vm_pages (depending on flags) 225 * => flag values: UVM_LOAN_TOANON - loan to anons 226 * UVM_LOAN_TOPAGE - loan to wired kernel page 227 * one and only one of these flags must be set! 228 * => returns 0 (success), or an appropriate error number 229 */ 230 231 int 232 uvm_loan(map, start, len, v, flags) 233 struct vm_map *map; 234 vaddr_t start; 235 vsize_t len; 236 void *v; 237 int flags; 238 { 239 struct uvm_faultinfo ufi; 240 void **result, **output; 241 int rv, error; 242 243 /* 244 * ensure that one and only one of the flags is set 245 */ 246 247 KASSERT(((flags & UVM_LOAN_TOANON) == 0) ^ 248 ((flags & UVM_LOAN_TOPAGE) == 0)); 249 KASSERT((map->flags & VM_MAP_INTRSAFE) == 0); 250 251 /* 252 * "output" is a pointer to the current place to put the loaned page. 253 */ 254 255 result = v; 256 output = &result[0]; /* start at the beginning ... */ 257 258 /* 259 * while we've got pages to do 260 */ 261 262 while (len > 0) { 263 264 /* 265 * fill in params for a call to uvmfault_lookup 266 */ 267 268 ufi.orig_map = map; 269 ufi.orig_rvaddr = start; 270 ufi.orig_size = len; 271 272 /* 273 * do the lookup, the only time this will fail is if we hit on 274 * an unmapped region (an error) 275 */ 276 277 if (!uvmfault_lookup(&ufi, FALSE)) { 278 error = ENOENT; 279 goto fail; 280 } 281 282 /* 283 * map now locked. now do the loanout... 284 */ 285 286 rv = uvm_loanentry(&ufi, &output, flags); 287 if (rv < 0) { 288 /* all unlocked due to error */ 289 error = EINVAL; 290 goto fail; 291 } 292 293 /* 294 * done! the map is unlocked. advance, if possible. 295 * 296 * XXXCDC: could be recoded to hold the map lock with 297 * smarter code (but it only happens on map entry 298 * boundaries, so it isn't that bad). 299 */ 300 301 if (rv) { 302 rv <<= PAGE_SHIFT; 303 len -= rv; 304 start += rv; 305 } 306 } 307 return 0; 308 309 fail: 310 /* 311 * failed to complete loans. drop any loans and return failure code. 312 * map is already unlocked. 313 */ 314 315 if (output - result) { 316 if (flags & UVM_LOAN_TOANON) { 317 uvm_unloananon((struct vm_anon **)result, 318 output - result); 319 } else { 320 uvm_unloanpage((struct vm_page **)result, 321 output - result); 322 } 323 } 324 return (error); 325 } 326 327 /* 328 * uvm_loananon: loan a page from an anon out 329 * 330 * => called with map, amap, uobj locked 331 * => return value: 332 * -1 = fatal error, everything is unlocked, abort. 333 * 0 = lookup in ufi went stale, everything unlocked, relookup and 334 * try again 335 * 1 = got it, everything still locked 336 */ 337 338 int 339 uvm_loananon(ufi, output, flags, anon) 340 struct uvm_faultinfo *ufi; 341 void ***output; 342 int flags; 343 struct vm_anon *anon; 344 { 345 struct vm_page *pg; 346 int error; 347 348 /* 349 * if we are loaning to "another" anon then it is easy, we just 350 * bump the reference count on the current anon and return a 351 * pointer to it (it becomes copy-on-write shared). 352 */ 353 354 if (flags & UVM_LOAN_TOANON) { 355 simple_lock(&anon->an_lock); 356 pg = anon->u.an_page; 357 if (pg && (pg->pqflags & PQ_ANON) != 0 && anon->an_ref == 1) { 358 pmap_page_protect(pg, VM_PROT_READ); 359 } 360 anon->an_ref++; 361 **output = anon; 362 (*output)++; 363 simple_unlock(&anon->an_lock); 364 return (1); 365 } 366 367 /* 368 * we are loaning to a kernel-page. we need to get the page 369 * resident so we can wire it. uvmfault_anonget will handle 370 * this for us. 371 */ 372 373 simple_lock(&anon->an_lock); 374 error = uvmfault_anonget(ufi, ufi->entry->aref.ar_amap, anon); 375 376 /* 377 * if we were unable to get the anon, then uvmfault_anonget has 378 * unlocked everything and returned an error code. 379 */ 380 381 if (error) { 382 383 /* need to refault (i.e. refresh our lookup) ? */ 384 if (error == ERESTART) { 385 return (0); 386 } 387 388 /* "try again"? sleep a bit and retry ... */ 389 if (error == EAGAIN) { 390 tsleep(&lbolt, PVM, "loanagain", 0); 391 return (0); 392 } 393 394 /* otherwise flag it as an error */ 395 return (-1); 396 } 397 398 /* 399 * we have the page and its owner locked: do the loan now. 400 */ 401 402 pg = anon->u.an_page; 403 uvm_lock_pageq(); 404 KASSERT(pg->wire_count == 0); 405 if (pg->loan_count == 0) { 406 pmap_page_protect(pg, VM_PROT_READ); 407 } 408 pg->loan_count++; 409 uvm_pagedequeue(pg); 410 uvm_unlock_pageq(); 411 **output = pg; 412 (*output)++; 413 414 /* unlock anon and return success */ 415 if (pg->uobject) /* XXXCDC: what if this is our uobj? bad */ 416 simple_unlock(&pg->uobject->vmobjlock); 417 simple_unlock(&anon->an_lock); 418 return (1); 419 } 420 421 /* 422 * uvm_loanuobjpages: loan pages from a uobj out (O->K) 423 * 424 * => called with uobj locked. 425 * => caller should own the pages. 426 */ 427 void 428 uvm_loanuobjpages(pgpp, npages) 429 struct vm_page **pgpp; 430 int npages; 431 { 432 int i; 433 434 for (i = 0; i < npages; i++) { 435 struct vm_page *pg = pgpp[i]; 436 437 KASSERT(pg->uobject != NULL); 438 KASSERT(!(pg->flags & (PG_RELEASED|PG_PAGEOUT))); 439 LOCK_ASSERT(simple_lock_held(&pg->uobject->vmobjlock)); 440 KASSERT(pg->flags & PG_BUSY); 441 442 uvm_lock_pageq(); 443 if (pg->loan_count == 0) { 444 pmap_page_protect(pg, VM_PROT_READ); 445 } 446 pg->loan_count++; 447 uvm_pagedequeue(pg); 448 uvm_unlock_pageq(); 449 if (pg->flags & PG_WANTED) { 450 wakeup(pg); 451 } 452 pg->flags &= ~(PG_WANTED|PG_BUSY); 453 UVM_PAGE_OWN(pg, NULL); 454 } 455 } 456 457 /* 458 * uvm_loanuobj: loan a page from a uobj out 459 * 460 * => called with map, amap, uobj locked 461 * => return value: 462 * -1 = fatal error, everything is unlocked, abort. 463 * 0 = lookup in ufi went stale, everything unlocked, relookup and 464 * try again 465 * 1 = got it, everything still locked 466 */ 467 468 static int 469 uvm_loanuobj(ufi, output, flags, va) 470 struct uvm_faultinfo *ufi; 471 void ***output; 472 int flags; 473 vaddr_t va; 474 { 475 struct vm_amap *amap = ufi->entry->aref.ar_amap; 476 struct uvm_object *uobj = ufi->entry->object.uvm_obj; 477 struct vm_page *pg; 478 struct vm_anon *anon; 479 int error, npages; 480 boolean_t locked; 481 482 /* 483 * first we must make sure the page is resident. 484 * 485 * XXXCDC: duplicate code with uvm_fault(). 486 */ 487 488 if (uobj->pgops->pgo_get) { /* try locked pgo_get */ 489 npages = 1; 490 pg = NULL; 491 error = (*uobj->pgops->pgo_get)(uobj, 492 va - ufi->entry->start + ufi->entry->offset, 493 &pg, &npages, 0, VM_PROT_READ, MADV_NORMAL, PGO_LOCKED); 494 } else { 495 error = EIO; /* must have pgo_get op */ 496 } 497 498 /* 499 * check the result of the locked pgo_get. if there is a problem, 500 * then we fail the loan. 501 */ 502 503 if (error && error != EBUSY) { 504 uvmfault_unlockall(ufi, amap, uobj, NULL); 505 return (-1); 506 } 507 508 /* 509 * if we need to unlock for I/O, do so now. 510 */ 511 512 if (error == EBUSY) { 513 uvmfault_unlockall(ufi, amap, NULL, NULL); 514 515 /* locked: uobj */ 516 npages = 1; 517 error = (*uobj->pgops->pgo_get)(uobj, 518 va - ufi->entry->start + ufi->entry->offset, 519 &pg, &npages, 0, VM_PROT_READ, MADV_NORMAL, PGO_SYNCIO); 520 /* locked: <nothing> */ 521 522 if (error) { 523 if (error == EAGAIN) { 524 tsleep(&lbolt, PVM, "fltagain2", 0); 525 return (0); 526 } 527 return (-1); 528 } 529 530 /* 531 * pgo_get was a success. attempt to relock everything. 532 */ 533 534 locked = uvmfault_relock(ufi); 535 if (locked && amap) 536 amap_lock(amap); 537 simple_lock(&uobj->vmobjlock); 538 539 /* 540 * verify that the page has not be released and re-verify 541 * that amap slot is still free. if there is a problem we 542 * drop our lock (thus force a lookup refresh/retry). 543 */ 544 545 if ((pg->flags & PG_RELEASED) != 0 || 546 (locked && amap && amap_lookup(&ufi->entry->aref, 547 ufi->orig_rvaddr - ufi->entry->start))) { 548 if (locked) 549 uvmfault_unlockall(ufi, amap, NULL, NULL); 550 locked = FALSE; 551 } 552 553 /* 554 * didn't get the lock? release the page and retry. 555 */ 556 557 if (locked == FALSE) { 558 if (pg->flags & PG_WANTED) { 559 wakeup(pg); 560 } 561 if (pg->flags & PG_RELEASED) { 562 uvm_lock_pageq(); 563 uvm_pagefree(pg); 564 uvm_unlock_pageq(); 565 return (0); 566 } 567 uvm_lock_pageq(); 568 uvm_pageactivate(pg); 569 uvm_unlock_pageq(); 570 pg->flags &= ~(PG_BUSY|PG_WANTED); 571 UVM_PAGE_OWN(pg, NULL); 572 simple_unlock(&uobj->vmobjlock); 573 return (0); 574 } 575 } 576 577 /* 578 * at this point we have the page we want ("pg") marked PG_BUSY for us 579 * and we have all data structures locked. do the loanout. page can 580 * not be PG_RELEASED (we caught this above). 581 */ 582 583 if ((flags & UVM_LOAN_TOANON) == 0) { 584 uvm_loanuobjpages(&pg, 1); 585 **output = pg; 586 (*output)++; 587 return (1); 588 } 589 590 /* 591 * must be a loan to an anon. check to see if there is already 592 * an anon associated with this page. if so, then just return 593 * a reference to this object. the page should already be 594 * mapped read-only because it is already on loan. 595 */ 596 597 if (pg->uanon) { 598 anon = pg->uanon; 599 simple_lock(&anon->an_lock); 600 anon->an_ref++; 601 simple_unlock(&anon->an_lock); 602 if (pg->flags & PG_WANTED) { 603 wakeup(pg); 604 } 605 pg->flags &= ~(PG_WANTED|PG_BUSY); 606 UVM_PAGE_OWN(pg, NULL); 607 **output = anon; 608 (*output)++; 609 return (1); 610 } 611 612 /* 613 * need to allocate a new anon 614 */ 615 616 anon = uvm_analloc(); 617 if (anon == NULL) { 618 if (pg->flags & PG_WANTED) { 619 wakeup(pg); 620 } 621 pg->flags &= ~(PG_WANTED|PG_BUSY); 622 UVM_PAGE_OWN(pg, NULL); 623 uvmfault_unlockall(ufi, amap, uobj, NULL); 624 return (-1); 625 } 626 anon->u.an_page = pg; 627 pg->uanon = anon; 628 uvm_lock_pageq(); 629 if (pg->loan_count == 0) { 630 pmap_page_protect(pg, VM_PROT_READ); 631 } 632 pg->loan_count++; 633 uvm_pageactivate(pg); 634 uvm_unlock_pageq(); 635 if (pg->flags & PG_WANTED) { 636 wakeup(pg); 637 } 638 pg->flags &= ~(PG_WANTED|PG_BUSY); 639 UVM_PAGE_OWN(pg, NULL); 640 simple_unlock(&anon->an_lock); 641 **output = anon; 642 (*output)++; 643 return (1); 644 } 645 646 /* 647 * uvm_loanzero: loan a zero-fill page out 648 * 649 * => called with map, amap, uobj locked 650 * => return value: 651 * -1 = fatal error, everything is unlocked, abort. 652 * 0 = lookup in ufi went stale, everything unlocked, relookup and 653 * try again 654 * 1 = got it, everything still locked 655 */ 656 657 static struct uvm_object uvm_loanzero_object; 658 659 static int 660 uvm_loanzero(ufi, output, flags) 661 struct uvm_faultinfo *ufi; 662 void ***output; 663 int flags; 664 { 665 struct vm_anon *anon; 666 struct vm_page *pg; 667 struct uvm_object *uobj = ufi->entry->object.uvm_obj; 668 struct vm_amap *amap = ufi->entry->aref.ar_amap; 669 670 simple_lock(&uvm_loanzero_object.vmobjlock); 671 672 /* 673 * first, get ahold of our single zero page. 674 */ 675 676 if (__predict_false((pg = 677 TAILQ_FIRST(&uvm_loanzero_object.memq)) == NULL)) { 678 while ((pg = uvm_pagealloc(&uvm_loanzero_object, 0, NULL, 679 UVM_PGA_ZERO)) == NULL) { 680 simple_unlock(&uvm_loanzero_object.vmobjlock); 681 uvmfault_unlockall(ufi, amap, uobj, NULL); 682 uvm_wait("loanzero"); 683 if (!uvmfault_relock(ufi)) { 684 return (0); 685 } 686 if (amap) { 687 amap_lock(amap); 688 } 689 if (uobj) { 690 simple_lock(&uobj->vmobjlock); 691 } 692 simple_lock(&uvm_loanzero_object.vmobjlock); 693 } 694 695 /* got a zero'd page. */ 696 pg->flags &= ~(PG_WANTED|PG_BUSY|PG_FAKE); 697 pg->flags |= PG_RDONLY; 698 UVM_PAGE_OWN(pg, NULL); 699 } 700 701 if ((flags & UVM_LOAN_TOANON) == 0) { /* loaning to kernel-page */ 702 uvm_lock_pageq(); 703 pg->loan_count++; 704 uvm_unlock_pageq(); 705 simple_unlock(&uvm_loanzero_object.vmobjlock); 706 **output = pg; 707 (*output)++; 708 return (1); 709 } 710 711 /* 712 * loaning to an anon. check to see if there is already an anon 713 * associated with this page. if so, then just return a reference 714 * to this object. 715 */ 716 717 if (pg->uanon) { 718 anon = pg->uanon; 719 simple_lock(&anon->an_lock); 720 anon->an_ref++; 721 simple_unlock(&anon->an_lock); 722 simple_unlock(&uvm_loanzero_object.vmobjlock); 723 **output = anon; 724 (*output)++; 725 return (1); 726 } 727 728 /* 729 * need to allocate a new anon 730 */ 731 732 anon = uvm_analloc(); 733 if (anon == NULL) { 734 /* out of swap causes us to fail */ 735 simple_unlock(&uvm_loanzero_object.vmobjlock); 736 uvmfault_unlockall(ufi, amap, uobj, NULL); 737 return (-1); 738 } 739 anon->u.an_page = pg; 740 pg->uanon = anon; 741 uvm_lock_pageq(); 742 pg->loan_count++; 743 uvm_pageactivate(pg); 744 uvm_unlock_pageq(); 745 simple_unlock(&uvm_loanzero_object.vmobjlock); 746 **output = anon; 747 (*output)++; 748 return (1); 749 } 750 751 752 /* 753 * uvm_unloananon: kill loans on anons (basically a normal ref drop) 754 * 755 * => we expect all our resources to be unlocked 756 */ 757 758 static void 759 uvm_unloananon(aloans, nanons) 760 struct vm_anon **aloans; 761 int nanons; 762 { 763 struct vm_anon *anon; 764 765 while (nanons-- > 0) { 766 int refs; 767 768 anon = *aloans++; 769 simple_lock(&anon->an_lock); 770 refs = --anon->an_ref; 771 simple_unlock(&anon->an_lock); 772 773 if (refs == 0) { 774 uvm_anfree(anon); 775 } 776 } 777 } 778 779 /* 780 * uvm_unloanpage: kill loans on pages loaned out to the kernel 781 * 782 * => we expect all our resources to be unlocked 783 */ 784 785 static void 786 uvm_unloanpage(ploans, npages) 787 struct vm_page **ploans; 788 int npages; 789 { 790 struct vm_page *pg; 791 struct simplelock *slock; 792 793 uvm_lock_pageq(); 794 while (npages-- > 0) { 795 pg = *ploans++; 796 797 /* 798 * do a little dance to acquire the object or anon lock 799 * as appropriate. we are locking in the wrong order, 800 * so we have to do a try-lock here. 801 */ 802 803 slock = NULL; 804 while (pg->uobject != NULL || pg->uanon != NULL) { 805 if (pg->uobject != NULL) { 806 slock = &pg->uobject->vmobjlock; 807 } else { 808 slock = &pg->uanon->an_lock; 809 } 810 if (simple_lock_try(slock)) { 811 break; 812 } 813 uvm_unlock_pageq(); 814 uvm_lock_pageq(); 815 slock = NULL; 816 } 817 818 /* 819 * drop our loan. if page is owned by an anon but 820 * PQ_ANON is not set, the page was loaned to the anon 821 * from an object which dropped ownership, so resolve 822 * this by turning the anon's loan into real ownership 823 * (ie. decrement loan_count again and set PQ_ANON). 824 * after all this, if there are no loans left, put the 825 * page back a paging queue (if the page is owned by 826 * an anon) or free it (if the page is now unowned). 827 */ 828 829 KASSERT(pg->loan_count > 0); 830 pg->loan_count--; 831 if (pg->uobject == NULL && pg->uanon != NULL && 832 (pg->pqflags & PQ_ANON) == 0) { 833 KASSERT(pg->loan_count > 0); 834 pg->loan_count--; 835 pg->pqflags |= PQ_ANON; 836 } 837 if (pg->loan_count == 0) { 838 if (pg->uobject == NULL && pg->uanon == NULL) { 839 KASSERT((pg->flags & PG_BUSY) == 0); 840 uvm_pagefree(pg); 841 } else { 842 uvm_pageactivate(pg); 843 } 844 } else if (pg->loan_count == 1 && pg->uobject != NULL && 845 pg->uanon != NULL) { 846 uvm_pageactivate(pg); 847 } 848 if (slock != NULL) { 849 simple_unlock(slock); 850 } 851 } 852 uvm_unlock_pageq(); 853 } 854 855 /* 856 * uvm_unloan: kill loans on pages or anons. 857 */ 858 859 void 860 uvm_unloan(void *v, int npages, int flags) 861 { 862 if (flags & UVM_LOAN_TOANON) { 863 uvm_unloananon(v, npages); 864 } else { 865 uvm_unloanpage(v, npages); 866 } 867 } 868 869 /* 870 * Minimal pager for uvm_loanzero_object. We need to provide a "put" 871 * method, because the page can end up on a paging queue, and the 872 * page daemon will want to call pgo_put when it encounters the page 873 * on the inactive list. 874 */ 875 876 static int 877 ulz_put(struct uvm_object *uobj, voff_t start, voff_t stop, int flags) 878 { 879 struct vm_page *pg; 880 881 KDASSERT(uobj == &uvm_loanzero_object); 882 883 /* 884 * Don't need to do any work here if we're not freeing pages. 885 */ 886 887 if ((flags & PGO_FREE) == 0) { 888 simple_unlock(&uobj->vmobjlock); 889 return 0; 890 } 891 892 /* 893 * we don't actually want to ever free the uvm_loanzero_page, so 894 * just reactivate or dequeue it. 895 */ 896 897 pg = TAILQ_FIRST(&uobj->memq); 898 KASSERT(pg != NULL); 899 KASSERT(TAILQ_NEXT(pg, listq) == NULL); 900 901 uvm_lock_pageq(); 902 if (pg->uanon) 903 uvm_pageactivate(pg); 904 else 905 uvm_pagedequeue(pg); 906 uvm_unlock_pageq(); 907 908 simple_unlock(&uobj->vmobjlock); 909 return 0; 910 } 911 912 static struct uvm_pagerops ulz_pager = { 913 NULL, /* init */ 914 NULL, /* reference */ 915 NULL, /* detach */ 916 NULL, /* fault */ 917 NULL, /* get */ 918 ulz_put, /* put */ 919 }; 920 921 /* 922 * uvm_loan_init(): initialize the uvm_loan() facility. 923 */ 924 925 void 926 uvm_loan_init(void) 927 { 928 929 simple_lock_init(&uvm_loanzero_object.vmobjlock); 930 TAILQ_INIT(&uvm_loanzero_object.memq); 931 uvm_loanzero_object.pgops = &ulz_pager; 932 } 933 934 /* 935 * uvm_loanbreak: break loan on a uobj page 936 * 937 * => called with uobj locked 938 * => the page should be busy 939 * => return value: 940 * newly allocated page if succeeded 941 */ 942 struct vm_page * 943 uvm_loanbreak(struct vm_page *uobjpage) 944 { 945 struct vm_page *pg; 946 struct uvm_object *uobj = uobjpage->uobject; 947 voff_t offset; 948 949 KASSERT(uobj != NULL); 950 LOCK_ASSERT(simple_lock_held(&uobj->vmobjlock)); 951 KASSERT(uobjpage->flags & PG_BUSY); 952 953 /* alloc new un-owned page */ 954 pg = uvm_pagealloc(NULL, 0, NULL, 0); 955 if (pg == NULL) 956 return NULL; 957 958 /* 959 * copy the data from the old page to the new 960 * one and clear the fake/clean flags on the 961 * new page (keep it busy). force a reload 962 * of the old page by clearing it from all 963 * pmaps. then lock the page queues to 964 * rename the pages. 965 */ 966 967 uvm_pagecopy(uobjpage, pg); /* old -> new */ 968 pg->flags &= ~(PG_FAKE|PG_CLEAN); 969 pmap_page_protect(uobjpage, VM_PROT_NONE); 970 if (uobjpage->flags & PG_WANTED) 971 wakeup(uobjpage); 972 /* uobj still locked */ 973 uobjpage->flags &= ~(PG_WANTED|PG_BUSY); 974 UVM_PAGE_OWN(uobjpage, NULL); 975 976 uvm_lock_pageq(); 977 offset = uobjpage->offset; 978 uvm_pagerealloc(uobjpage, NULL, 0); 979 980 /* 981 * if the page is no longer referenced by 982 * an anon (i.e. we are breaking an O->K 983 * loan), then remove it from any pageq's. 984 */ 985 if (uobjpage->uanon == NULL) 986 uvm_pagedequeue(uobjpage); 987 988 /* 989 * at this point we have absolutely no 990 * control over uobjpage 991 */ 992 993 /* install new page */ 994 uvm_pageactivate(pg); 995 uvm_pagerealloc(pg, uobj, offset); 996 uvm_unlock_pageq(); 997 998 /* 999 * done! loan is broken and "pg" is 1000 * PG_BUSY. it can now replace uobjpage. 1001 */ 1002 1003 return pg; 1004 } 1005