1 /* $OpenBSD: uvm_aobj.c,v 1.15 2001/09/11 20:05:26 miod Exp $ */ 2 /* $NetBSD: uvm_aobj.c,v 1.31 2000/05/19 04:34:45 thorpej Exp $ */ 3 4 /* 5 * Copyright (c) 1998 Chuck Silvers, Charles D. Cranor and 6 * Washington University. 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed by Charles D. Cranor and 20 * Washington University. 21 * 4. The name of the author may not be used to endorse or promote products 22 * derived from this software without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 25 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 26 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 27 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 28 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 29 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 30 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 31 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 32 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 33 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 34 * 35 * from: Id: uvm_aobj.c,v 1.1.2.5 1998/02/06 05:14:38 chs Exp 36 */ 37 /* 38 * uvm_aobj.c: anonymous memory uvm_object pager 39 * 40 * author: Chuck Silvers <chuq@chuq.com> 41 * started: Jan-1998 42 * 43 * - design mostly from Chuck Cranor 44 */ 45 46 #include <sys/param.h> 47 #include <sys/systm.h> 48 #include <sys/proc.h> 49 #include <sys/malloc.h> 50 #include <sys/pool.h> 51 #include <sys/kernel.h> 52 53 #include <vm/vm.h> 54 #include <vm/vm_page.h> 55 56 #include <uvm/uvm.h> 57 58 /* 59 * an aobj manages anonymous-memory backed uvm_objects. in addition 60 * to keeping the list of resident pages, it also keeps a list of 61 * allocated swap blocks. depending on the size of the aobj this list 62 * of allocated swap blocks is either stored in an array (small objects) 63 * or in a hash table (large objects). 64 */ 65 66 /* 67 * local structures 68 */ 69 70 /* 71 * for hash tables, we break the address space of the aobj into blocks 72 * of UAO_SWHASH_CLUSTER_SIZE pages. we require the cluster size to 73 * be a power of two. 74 */ 75 76 #define UAO_SWHASH_CLUSTER_SHIFT 4 77 #define UAO_SWHASH_CLUSTER_SIZE (1 << UAO_SWHASH_CLUSTER_SHIFT) 78 79 /* get the "tag" for this page index */ 80 #define UAO_SWHASH_ELT_TAG(PAGEIDX) \ 81 ((PAGEIDX) >> UAO_SWHASH_CLUSTER_SHIFT) 82 83 /* given an ELT and a page index, find the swap slot */ 84 #define UAO_SWHASH_ELT_PAGESLOT(ELT, PAGEIDX) \ 85 ((ELT)->slots[(PAGEIDX) & (UAO_SWHASH_CLUSTER_SIZE - 1)]) 86 87 /* given an ELT, return its pageidx base */ 88 #define UAO_SWHASH_ELT_PAGEIDX_BASE(ELT) \ 89 ((ELT)->tag << UAO_SWHASH_CLUSTER_SHIFT) 90 91 /* 92 * the swhash hash function 93 */ 94 #define UAO_SWHASH_HASH(AOBJ, PAGEIDX) \ 95 (&(AOBJ)->u_swhash[(((PAGEIDX) >> UAO_SWHASH_CLUSTER_SHIFT) \ 96 & (AOBJ)->u_swhashmask)]) 97 98 /* 99 * the swhash threshhold determines if we will use an array or a 100 * hash table to store the list of allocated swap blocks. 101 */ 102 103 #define UAO_SWHASH_THRESHOLD (UAO_SWHASH_CLUSTER_SIZE * 4) 104 #define UAO_USES_SWHASH(AOBJ) \ 105 ((AOBJ)->u_pages > UAO_SWHASH_THRESHOLD) /* use hash? */ 106 107 /* 108 * the number of buckets in a swhash, with an upper bound 109 */ 110 #define UAO_SWHASH_MAXBUCKETS 256 111 #define UAO_SWHASH_BUCKETS(AOBJ) \ 112 (min((AOBJ)->u_pages >> UAO_SWHASH_CLUSTER_SHIFT, \ 113 UAO_SWHASH_MAXBUCKETS)) 114 115 116 /* 117 * uao_swhash_elt: when a hash table is being used, this structure defines 118 * the format of an entry in the bucket list. 119 */ 120 121 struct uao_swhash_elt { 122 LIST_ENTRY(uao_swhash_elt) list; /* the hash list */ 123 voff_t tag; /* our 'tag' */ 124 int count; /* our number of active slots */ 125 int slots[UAO_SWHASH_CLUSTER_SIZE]; /* the slots */ 126 }; 127 128 /* 129 * uao_swhash: the swap hash table structure 130 */ 131 132 LIST_HEAD(uao_swhash, uao_swhash_elt); 133 134 /* 135 * uao_swhash_elt_pool: pool of uao_swhash_elt structures 136 */ 137 138 struct pool uao_swhash_elt_pool; 139 140 /* 141 * uvm_aobj: the actual anon-backed uvm_object 142 * 143 * => the uvm_object is at the top of the structure, this allows 144 * (struct uvm_device *) == (struct uvm_object *) 145 * => only one of u_swslots and u_swhash is used in any given aobj 146 */ 147 148 struct uvm_aobj { 149 struct uvm_object u_obj; /* has: lock, pgops, memq, #pages, #refs */ 150 int u_pages; /* number of pages in entire object */ 151 int u_flags; /* the flags (see uvm_aobj.h) */ 152 int *u_swslots; /* array of offset->swapslot mappings */ 153 /* 154 * hashtable of offset->swapslot mappings 155 * (u_swhash is an array of bucket heads) 156 */ 157 struct uao_swhash *u_swhash; 158 u_long u_swhashmask; /* mask for hashtable */ 159 LIST_ENTRY(uvm_aobj) u_list; /* global list of aobjs */ 160 }; 161 162 /* 163 * uvm_aobj_pool: pool of uvm_aobj structures 164 */ 165 166 struct pool uvm_aobj_pool; 167 168 /* 169 * local functions 170 */ 171 172 static struct uao_swhash_elt *uao_find_swhash_elt __P((struct uvm_aobj *, 173 int, boolean_t)); 174 static int uao_find_swslot __P((struct uvm_aobj *, int)); 175 static boolean_t uao_flush __P((struct uvm_object *, 176 voff_t, voff_t, int)); 177 static void uao_free __P((struct uvm_aobj *)); 178 static int uao_get __P((struct uvm_object *, voff_t, 179 vm_page_t *, int *, int, 180 vm_prot_t, int, int)); 181 static boolean_t uao_releasepg __P((struct vm_page *, 182 struct vm_page **)); 183 static boolean_t uao_pagein __P((struct uvm_aobj *, int, int)); 184 static boolean_t uao_pagein_page __P((struct uvm_aobj *, int)); 185 186 187 188 /* 189 * aobj_pager 190 * 191 * note that some functions (e.g. put) are handled elsewhere 192 */ 193 194 struct uvm_pagerops aobj_pager = { 195 NULL, /* init */ 196 uao_reference, /* reference */ 197 uao_detach, /* detach */ 198 NULL, /* fault */ 199 uao_flush, /* flush */ 200 uao_get, /* get */ 201 NULL, /* asyncget */ 202 NULL, /* put (done by pagedaemon) */ 203 NULL, /* cluster */ 204 NULL, /* mk_pcluster */ 205 NULL, /* aiodone */ 206 uao_releasepg /* releasepg */ 207 }; 208 209 /* 210 * uao_list: global list of active aobjs, locked by uao_list_lock 211 */ 212 213 static LIST_HEAD(aobjlist, uvm_aobj) uao_list; 214 static simple_lock_data_t uao_list_lock; 215 216 217 /* 218 * functions 219 */ 220 221 /* 222 * hash table/array related functions 223 */ 224 225 /* 226 * uao_find_swhash_elt: find (or create) a hash table entry for a page 227 * offset. 228 * 229 * => the object should be locked by the caller 230 */ 231 232 static struct uao_swhash_elt * 233 uao_find_swhash_elt(aobj, pageidx, create) 234 struct uvm_aobj *aobj; 235 int pageidx; 236 boolean_t create; 237 { 238 struct uao_swhash *swhash; 239 struct uao_swhash_elt *elt; 240 voff_t page_tag; 241 242 swhash = UAO_SWHASH_HASH(aobj, pageidx); /* first hash to get bucket */ 243 page_tag = UAO_SWHASH_ELT_TAG(pageidx); /* tag to search for */ 244 245 /* 246 * now search the bucket for the requested tag 247 */ 248 for (elt = swhash->lh_first; elt != NULL; elt = elt->list.le_next) { 249 if (elt->tag == page_tag) 250 return(elt); 251 } 252 253 /* fail now if we are not allowed to create a new entry in the bucket */ 254 if (!create) 255 return NULL; 256 257 258 /* 259 * allocate a new entry for the bucket and init/insert it in 260 */ 261 elt = pool_get(&uao_swhash_elt_pool, PR_WAITOK); 262 LIST_INSERT_HEAD(swhash, elt, list); 263 elt->tag = page_tag; 264 elt->count = 0; 265 memset(elt->slots, 0, sizeof(elt->slots)); 266 267 return(elt); 268 } 269 270 /* 271 * uao_find_swslot: find the swap slot number for an aobj/pageidx 272 * 273 * => object must be locked by caller 274 */ 275 __inline static int 276 uao_find_swslot(aobj, pageidx) 277 struct uvm_aobj *aobj; 278 int pageidx; 279 { 280 281 /* 282 * if noswap flag is set, then we never return a slot 283 */ 284 285 if (aobj->u_flags & UAO_FLAG_NOSWAP) 286 return(0); 287 288 /* 289 * if hashing, look in hash table. 290 */ 291 292 if (UAO_USES_SWHASH(aobj)) { 293 struct uao_swhash_elt *elt = 294 uao_find_swhash_elt(aobj, pageidx, FALSE); 295 296 if (elt) 297 return(UAO_SWHASH_ELT_PAGESLOT(elt, pageidx)); 298 else 299 return(0); 300 } 301 302 /* 303 * otherwise, look in the array 304 */ 305 return(aobj->u_swslots[pageidx]); 306 } 307 308 /* 309 * uao_set_swslot: set the swap slot for a page in an aobj. 310 * 311 * => setting a slot to zero frees the slot 312 * => object must be locked by caller 313 */ 314 int 315 uao_set_swslot(uobj, pageidx, slot) 316 struct uvm_object *uobj; 317 int pageidx, slot; 318 { 319 struct uvm_aobj *aobj = (struct uvm_aobj *)uobj; 320 int oldslot; 321 UVMHIST_FUNC("uao_set_swslot"); UVMHIST_CALLED(pdhist); 322 UVMHIST_LOG(pdhist, "aobj %p pageidx %d slot %d", 323 aobj, pageidx, slot, 0); 324 325 /* 326 * if noswap flag is set, then we can't set a slot 327 */ 328 329 if (aobj->u_flags & UAO_FLAG_NOSWAP) { 330 331 if (slot == 0) 332 return(0); /* a clear is ok */ 333 334 /* but a set is not */ 335 printf("uao_set_swslot: uobj = %p\n", uobj); 336 panic("uao_set_swslot: attempt to set a slot on a NOSWAP object"); 337 } 338 339 /* 340 * are we using a hash table? if so, add it in the hash. 341 */ 342 343 if (UAO_USES_SWHASH(aobj)) { 344 /* 345 * Avoid allocating an entry just to free it again if 346 * the page had not swap slot in the first place, and 347 * we are freeing. 348 */ 349 struct uao_swhash_elt *elt = 350 uao_find_swhash_elt(aobj, pageidx, slot ? TRUE : FALSE); 351 if (elt == NULL) { 352 #ifdef DIAGNOSTIC 353 if (slot) 354 panic("uao_set_swslot: didn't create elt"); 355 #endif 356 return (0); 357 } 358 359 oldslot = UAO_SWHASH_ELT_PAGESLOT(elt, pageidx); 360 UAO_SWHASH_ELT_PAGESLOT(elt, pageidx) = slot; 361 362 /* 363 * now adjust the elt's reference counter and free it if we've 364 * dropped it to zero. 365 */ 366 367 /* an allocation? */ 368 if (slot) { 369 if (oldslot == 0) 370 elt->count++; 371 } else { /* freeing slot ... */ 372 if (oldslot) /* to be safe */ 373 elt->count--; 374 375 if (elt->count == 0) { 376 LIST_REMOVE(elt, list); 377 pool_put(&uao_swhash_elt_pool, elt); 378 } 379 } 380 381 } else { 382 /* we are using an array */ 383 oldslot = aobj->u_swslots[pageidx]; 384 aobj->u_swslots[pageidx] = slot; 385 } 386 return (oldslot); 387 } 388 389 /* 390 * end of hash/array functions 391 */ 392 393 /* 394 * uao_free: free all resources held by an aobj, and then free the aobj 395 * 396 * => the aobj should be dead 397 */ 398 static void 399 uao_free(aobj) 400 struct uvm_aobj *aobj; 401 { 402 403 simple_unlock(&aobj->u_obj.vmobjlock); 404 405 if (UAO_USES_SWHASH(aobj)) { 406 int i, hashbuckets = aobj->u_swhashmask + 1; 407 408 /* 409 * free the swslots from each hash bucket, 410 * then the hash bucket, and finally the hash table itself. 411 */ 412 for (i = 0; i < hashbuckets; i++) { 413 struct uao_swhash_elt *elt, *next; 414 415 for (elt = LIST_FIRST(&aobj->u_swhash[i]); 416 elt != NULL; 417 elt = next) { 418 int j; 419 420 for (j = 0; j < UAO_SWHASH_CLUSTER_SIZE; j++) { 421 int slot = elt->slots[j]; 422 423 if (slot) { 424 uvm_swap_free(slot, 1); 425 426 /* 427 * this page is no longer 428 * only in swap. 429 */ 430 simple_lock(&uvm.swap_data_lock); 431 uvmexp.swpgonly--; 432 simple_unlock(&uvm.swap_data_lock); 433 } 434 } 435 436 next = LIST_NEXT(elt, list); 437 pool_put(&uao_swhash_elt_pool, elt); 438 } 439 } 440 FREE(aobj->u_swhash, M_UVMAOBJ); 441 } else { 442 int i; 443 444 /* 445 * free the array 446 */ 447 448 for (i = 0; i < aobj->u_pages; i++) { 449 int slot = aobj->u_swslots[i]; 450 451 if (slot) { 452 uvm_swap_free(slot, 1); 453 454 /* this page is no longer only in swap. */ 455 simple_lock(&uvm.swap_data_lock); 456 uvmexp.swpgonly--; 457 simple_unlock(&uvm.swap_data_lock); 458 } 459 } 460 FREE(aobj->u_swslots, M_UVMAOBJ); 461 } 462 463 /* 464 * finally free the aobj itself 465 */ 466 pool_put(&uvm_aobj_pool, aobj); 467 } 468 469 /* 470 * pager functions 471 */ 472 473 /* 474 * uao_create: create an aobj of the given size and return its uvm_object. 475 * 476 * => for normal use, flags are always zero 477 * => for the kernel object, the flags are: 478 * UAO_FLAG_KERNOBJ - allocate the kernel object (can only happen once) 479 * UAO_FLAG_KERNSWAP - enable swapping of kernel object (" ") 480 */ 481 struct uvm_object * 482 uao_create(size, flags) 483 vsize_t size; 484 int flags; 485 { 486 static struct uvm_aobj kernel_object_store; /* home of kernel_object */ 487 static int kobj_alloced = 0; /* not allocated yet */ 488 int pages = round_page(size) >> PAGE_SHIFT; 489 struct uvm_aobj *aobj; 490 491 /* 492 * malloc a new aobj unless we are asked for the kernel object 493 */ 494 if (flags & UAO_FLAG_KERNOBJ) { /* want kernel object? */ 495 if (kobj_alloced) 496 panic("uao_create: kernel object already allocated"); 497 498 aobj = &kernel_object_store; 499 aobj->u_pages = pages; 500 aobj->u_flags = UAO_FLAG_NOSWAP; /* no swap to start */ 501 /* we are special, we never die */ 502 aobj->u_obj.uo_refs = UVM_OBJ_KERN; 503 kobj_alloced = UAO_FLAG_KERNOBJ; 504 } else if (flags & UAO_FLAG_KERNSWAP) { 505 aobj = &kernel_object_store; 506 if (kobj_alloced != UAO_FLAG_KERNOBJ) 507 panic("uao_create: asked to enable swap on kernel object"); 508 kobj_alloced = UAO_FLAG_KERNSWAP; 509 } else { /* normal object */ 510 aobj = pool_get(&uvm_aobj_pool, PR_WAITOK); 511 aobj->u_pages = pages; 512 aobj->u_flags = 0; /* normal object */ 513 aobj->u_obj.uo_refs = 1; /* start with 1 reference */ 514 } 515 516 /* 517 * allocate hash/array if necessary 518 * 519 * note: in the KERNSWAP case no need to worry about locking since 520 * we are still booting we should be the only thread around. 521 */ 522 if (flags == 0 || (flags & UAO_FLAG_KERNSWAP) != 0) { 523 int mflags = (flags & UAO_FLAG_KERNSWAP) != 0 ? 524 M_NOWAIT : M_WAITOK; 525 526 /* allocate hash table or array depending on object size */ 527 if (UAO_USES_SWHASH(aobj)) { 528 aobj->u_swhash = hashinit(UAO_SWHASH_BUCKETS(aobj), 529 M_UVMAOBJ, mflags, &aobj->u_swhashmask); 530 if (aobj->u_swhash == NULL) 531 panic("uao_create: hashinit swhash failed"); 532 } else { 533 MALLOC(aobj->u_swslots, int *, pages * sizeof(int), 534 M_UVMAOBJ, mflags); 535 if (aobj->u_swslots == NULL) 536 panic("uao_create: malloc swslots failed"); 537 memset(aobj->u_swslots, 0, pages * sizeof(int)); 538 } 539 540 if (flags) { 541 aobj->u_flags &= ~UAO_FLAG_NOSWAP; /* clear noswap */ 542 return(&aobj->u_obj); 543 /* done! */ 544 } 545 } 546 547 /* 548 * init aobj fields 549 */ 550 simple_lock_init(&aobj->u_obj.vmobjlock); 551 aobj->u_obj.pgops = &aobj_pager; 552 TAILQ_INIT(&aobj->u_obj.memq); 553 aobj->u_obj.uo_npages = 0; 554 555 /* 556 * now that aobj is ready, add it to the global list 557 */ 558 simple_lock(&uao_list_lock); 559 LIST_INSERT_HEAD(&uao_list, aobj, u_list); 560 simple_unlock(&uao_list_lock); 561 562 /* 563 * done! 564 */ 565 return(&aobj->u_obj); 566 } 567 568 569 570 /* 571 * uao_init: set up aobj pager subsystem 572 * 573 * => called at boot time from uvm_pager_init() 574 */ 575 void 576 uao_init() 577 { 578 static int uao_initialized; 579 580 if (uao_initialized) 581 return; 582 uao_initialized = TRUE; 583 584 LIST_INIT(&uao_list); 585 simple_lock_init(&uao_list_lock); 586 587 /* 588 * NOTE: Pages fror this pool must not come from a pageable 589 * kernel map! 590 */ 591 pool_init(&uao_swhash_elt_pool, sizeof(struct uao_swhash_elt), 592 0, 0, 0, "uaoeltpl", 0, NULL, NULL, M_UVMAOBJ); 593 594 pool_init(&uvm_aobj_pool, sizeof(struct uvm_aobj), 0, 0, 0, 595 "aobjpl", 0, 596 pool_page_alloc_nointr, pool_page_free_nointr, M_UVMAOBJ); 597 } 598 599 /* 600 * uao_reference: add a ref to an aobj 601 * 602 * => aobj must be unlocked 603 * => just lock it and call the locked version 604 */ 605 void 606 uao_reference(uobj) 607 struct uvm_object *uobj; 608 { 609 simple_lock(&uobj->vmobjlock); 610 uao_reference_locked(uobj); 611 simple_unlock(&uobj->vmobjlock); 612 } 613 614 /* 615 * uao_reference_locked: add a ref to an aobj that is already locked 616 * 617 * => aobj must be locked 618 * this needs to be separate from the normal routine 619 * since sometimes we need to add a reference to an aobj when 620 * it's already locked. 621 */ 622 void 623 uao_reference_locked(uobj) 624 struct uvm_object *uobj; 625 { 626 UVMHIST_FUNC("uao_reference"); UVMHIST_CALLED(maphist); 627 628 /* 629 * kernel_object already has plenty of references, leave it alone. 630 */ 631 632 if (UVM_OBJ_IS_KERN_OBJECT(uobj)) 633 return; 634 635 uobj->uo_refs++; /* bump! */ 636 UVMHIST_LOG(maphist, "<- done (uobj=0x%x, ref = %d)", 637 uobj, uobj->uo_refs,0,0); 638 } 639 640 641 /* 642 * uao_detach: drop a reference to an aobj 643 * 644 * => aobj must be unlocked 645 * => just lock it and call the locked version 646 */ 647 void 648 uao_detach(uobj) 649 struct uvm_object *uobj; 650 { 651 simple_lock(&uobj->vmobjlock); 652 uao_detach_locked(uobj); 653 } 654 655 656 /* 657 * uao_detach_locked: drop a reference to an aobj 658 * 659 * => aobj must be locked, and is unlocked (or freed) upon return. 660 * this needs to be separate from the normal routine 661 * since sometimes we need to detach from an aobj when 662 * it's already locked. 663 */ 664 void 665 uao_detach_locked(uobj) 666 struct uvm_object *uobj; 667 { 668 struct uvm_aobj *aobj = (struct uvm_aobj *)uobj; 669 struct vm_page *pg; 670 boolean_t busybody; 671 UVMHIST_FUNC("uao_detach"); UVMHIST_CALLED(maphist); 672 673 /* 674 * detaching from kernel_object is a noop. 675 */ 676 if (UVM_OBJ_IS_KERN_OBJECT(uobj)) { 677 simple_unlock(&uobj->vmobjlock); 678 return; 679 } 680 681 UVMHIST_LOG(maphist," (uobj=0x%x) ref=%d", uobj,uobj->uo_refs,0,0); 682 uobj->uo_refs--; /* drop ref! */ 683 if (uobj->uo_refs) { /* still more refs? */ 684 simple_unlock(&uobj->vmobjlock); 685 UVMHIST_LOG(maphist, "<- done (rc>0)", 0,0,0,0); 686 return; 687 } 688 689 /* 690 * remove the aobj from the global list. 691 */ 692 simple_lock(&uao_list_lock); 693 LIST_REMOVE(aobj, u_list); 694 simple_unlock(&uao_list_lock); 695 696 /* 697 * free all the pages that aren't PG_BUSY, 698 * mark for release any that are. 699 */ 700 busybody = FALSE; 701 for (pg = TAILQ_FIRST(&uobj->memq); 702 pg != NULL; 703 pg = TAILQ_NEXT(pg, listq)) { 704 if (pg->flags & PG_BUSY) { 705 pg->flags |= PG_RELEASED; 706 busybody = TRUE; 707 continue; 708 } 709 710 /* zap the mappings, free the swap slot, free the page */ 711 pmap_page_protect(pg, VM_PROT_NONE); 712 uao_dropswap(&aobj->u_obj, pg->offset >> PAGE_SHIFT); 713 uvm_lock_pageq(); 714 uvm_pagefree(pg); 715 uvm_unlock_pageq(); 716 } 717 718 /* 719 * if we found any busy pages, we're done for now. 720 * mark the aobj for death, releasepg will finish up for us. 721 */ 722 if (busybody) { 723 aobj->u_flags |= UAO_FLAG_KILLME; 724 simple_unlock(&aobj->u_obj.vmobjlock); 725 return; 726 } 727 728 /* 729 * finally, free the rest. 730 */ 731 uao_free(aobj); 732 } 733 734 /* 735 * uao_flush: "flush" pages out of a uvm object 736 * 737 * => object should be locked by caller. we may _unlock_ the object 738 * if (and only if) we need to clean a page (PGO_CLEANIT). 739 * XXXJRT Currently, however, we don't. In the case of cleaning 740 * XXXJRT a page, we simply just deactivate it. Should probably 741 * XXXJRT handle this better, in the future (although "flushing" 742 * XXXJRT anonymous memory isn't terribly important). 743 * => if PGO_CLEANIT is not set, then we will neither unlock the object 744 * or block. 745 * => if PGO_ALLPAGE is set, then all pages in the object are valid targets 746 * for flushing. 747 * => NOTE: we rely on the fact that the object's memq is a TAILQ and 748 * that new pages are inserted on the tail end of the list. thus, 749 * we can make a complete pass through the object in one go by starting 750 * at the head and working towards the tail (new pages are put in 751 * front of us). 752 * => NOTE: we are allowed to lock the page queues, so the caller 753 * must not be holding the lock on them [e.g. pagedaemon had 754 * better not call us with the queues locked] 755 * => we return TRUE unless we encountered some sort of I/O error 756 * XXXJRT currently never happens, as we never directly initiate 757 * XXXJRT I/O 758 * 759 * comment on "cleaning" object and PG_BUSY pages: 760 * this routine is holding the lock on the object. the only time 761 * that is can run into a PG_BUSY page that it does not own is if 762 * some other process has started I/O on the page (e.g. either 763 * a pagein or a pageout). if the PG_BUSY page is being paged 764 * in, then it can not be dirty (!PG_CLEAN) because no one has 765 * had a change to modify it yet. if the PG_BUSY page is being 766 * paged out then it means that someone else has already started 767 * cleaning the page for us (how nice!). in this case, if we 768 * have syncio specified, then after we make our pass through the 769 * object we need to wait for the other PG_BUSY pages to clear 770 * off (i.e. we need to do an iosync). also note that once a 771 * page is PG_BUSY is must stary in its object until it is un-busyed. 772 * XXXJRT We never actually do this, as we are "flushing" anonymous 773 * XXXJRT memory, which doesn't have persistent backing store. 774 * 775 * note on page traversal: 776 * we can traverse the pages in an object either by going down the 777 * linked list in "uobj->memq", or we can go over the address range 778 * by page doing hash table lookups for each address. depending 779 * on how many pages are in the object it may be cheaper to do one 780 * or the other. we set "by_list" to true if we are using memq. 781 * if the cost of a hash lookup was equal to the cost of the list 782 * traversal we could compare the number of pages in the start->stop 783 * range to the total number of pages in the object. however, it 784 * seems that a hash table lookup is more expensive than the linked 785 * list traversal, so we multiply the number of pages in the 786 * start->stop range by a penalty which we define below. 787 */ 788 789 #define UAO_HASH_PENALTY 4 /* XXX: a guess */ 790 791 boolean_t 792 uao_flush(uobj, start, stop, flags) 793 struct uvm_object *uobj; 794 voff_t start, stop; 795 int flags; 796 { 797 struct uvm_aobj *aobj = (struct uvm_aobj *) uobj; 798 struct vm_page *pp, *ppnext; 799 boolean_t retval, by_list; 800 voff_t curoff; 801 UVMHIST_FUNC("uao_flush"); UVMHIST_CALLED(maphist); 802 803 curoff = 0; /* XXX: shut up gcc */ 804 805 retval = TRUE; /* default to success */ 806 807 if (flags & PGO_ALLPAGES) { 808 start = 0; 809 stop = aobj->u_pages << PAGE_SHIFT; 810 by_list = TRUE; /* always go by the list */ 811 } else { 812 start = trunc_page(start); 813 stop = round_page(stop); 814 if (stop > (aobj->u_pages << PAGE_SHIFT)) { 815 printf("uao_flush: strange, got an out of range " 816 "flush (fixed)\n"); 817 stop = aobj->u_pages << PAGE_SHIFT; 818 } 819 by_list = (uobj->uo_npages <= 820 ((stop - start) >> PAGE_SHIFT) * UAO_HASH_PENALTY); 821 } 822 823 UVMHIST_LOG(maphist, 824 " flush start=0x%lx, stop=0x%x, by_list=%d, flags=0x%x", 825 start, stop, by_list, flags); 826 827 /* 828 * Don't need to do any work here if we're not freeing 829 * or deactivating pages. 830 */ 831 if ((flags & (PGO_DEACTIVATE|PGO_FREE)) == 0) { 832 UVMHIST_LOG(maphist, 833 "<- done (no work to do)",0,0,0,0); 834 return (retval); 835 } 836 837 /* 838 * now do it. note: we must update ppnext in the body of loop or we 839 * will get stuck. we need to use ppnext because we may free "pp" 840 * before doing the next loop. 841 */ 842 843 if (by_list) { 844 pp = uobj->memq.tqh_first; 845 } else { 846 curoff = start; 847 pp = uvm_pagelookup(uobj, curoff); 848 } 849 850 ppnext = NULL; /* XXX: shut up gcc */ 851 uvm_lock_pageq(); /* page queues locked */ 852 853 /* locked: both page queues and uobj */ 854 for ( ; (by_list && pp != NULL) || 855 (!by_list && curoff < stop) ; pp = ppnext) { 856 if (by_list) { 857 ppnext = pp->listq.tqe_next; 858 859 /* range check */ 860 if (pp->offset < start || pp->offset >= stop) 861 continue; 862 } else { 863 curoff += PAGE_SIZE; 864 if (curoff < stop) 865 ppnext = uvm_pagelookup(uobj, curoff); 866 867 /* null check */ 868 if (pp == NULL) 869 continue; 870 } 871 872 switch (flags & (PGO_CLEANIT|PGO_FREE|PGO_DEACTIVATE)) { 873 /* 874 * XXX In these first 3 cases, we always just 875 * XXX deactivate the page. We may want to 876 * XXX handle the different cases more specifically 877 * XXX in the future. 878 */ 879 case PGO_CLEANIT|PGO_FREE: 880 case PGO_CLEANIT|PGO_DEACTIVATE: 881 case PGO_DEACTIVATE: 882 deactivate_it: 883 /* skip the page if it's loaned or wired */ 884 if (pp->loan_count != 0 || 885 pp->wire_count != 0) 886 continue; 887 888 /* zap all mappings for the page. */ 889 pmap_page_protect(pp, VM_PROT_NONE); 890 891 /* ...and deactivate the page. */ 892 uvm_pagedeactivate(pp); 893 894 continue; 895 896 case PGO_FREE: 897 /* 898 * If there are multiple references to 899 * the object, just deactivate the page. 900 */ 901 if (uobj->uo_refs > 1) 902 goto deactivate_it; 903 904 /* XXX skip the page if it's loaned or wired */ 905 if (pp->loan_count != 0 || 906 pp->wire_count != 0) 907 continue; 908 909 /* 910 * mark the page as released if its busy. 911 */ 912 if (pp->flags & PG_BUSY) { 913 pp->flags |= PG_RELEASED; 914 continue; 915 } 916 917 /* zap all mappings for the page. */ 918 pmap_page_protect(pp, VM_PROT_NONE); 919 920 uao_dropswap(uobj, pp->offset >> PAGE_SHIFT); 921 uvm_pagefree(pp); 922 923 continue; 924 925 default: 926 panic("uao_flush: weird flags"); 927 } 928 #ifdef DIAGNOSTIC 929 panic("uao_flush: unreachable code"); 930 #endif 931 } 932 933 uvm_unlock_pageq(); 934 935 UVMHIST_LOG(maphist, 936 "<- done, rv=%d",retval,0,0,0); 937 return (retval); 938 } 939 940 /* 941 * uao_get: fetch me a page 942 * 943 * we have three cases: 944 * 1: page is resident -> just return the page. 945 * 2: page is zero-fill -> allocate a new page and zero it. 946 * 3: page is swapped out -> fetch the page from swap. 947 * 948 * cases 1 and 2 can be handled with PGO_LOCKED, case 3 cannot. 949 * so, if the "center" page hits case 3 (or any page, with PGO_ALLPAGES), 950 * then we will need to return VM_PAGER_UNLOCK. 951 * 952 * => prefer map unlocked (not required) 953 * => object must be locked! we will _unlock_ it before starting any I/O. 954 * => flags: PGO_ALLPAGES: get all of the pages 955 * PGO_LOCKED: fault data structures are locked 956 * => NOTE: offset is the offset of pps[0], _NOT_ pps[centeridx] 957 * => NOTE: caller must check for released pages!! 958 */ 959 static int 960 uao_get(uobj, offset, pps, npagesp, centeridx, access_type, advice, flags) 961 struct uvm_object *uobj; 962 voff_t offset; 963 struct vm_page **pps; 964 int *npagesp; 965 int centeridx, advice, flags; 966 vm_prot_t access_type; 967 { 968 struct uvm_aobj *aobj = (struct uvm_aobj *)uobj; 969 voff_t current_offset; 970 vm_page_t ptmp; 971 int lcv, gotpages, maxpages, swslot, rv, pageidx; 972 boolean_t done; 973 UVMHIST_FUNC("uao_get"); UVMHIST_CALLED(pdhist); 974 975 UVMHIST_LOG(pdhist, "aobj=%p offset=%d, flags=%d", 976 aobj, offset, flags,0); 977 978 /* 979 * get number of pages 980 */ 981 maxpages = *npagesp; 982 983 /* 984 * step 1: handled the case where fault data structures are locked. 985 */ 986 987 if (flags & PGO_LOCKED) { 988 /* 989 * step 1a: get pages that are already resident. only do 990 * this if the data structures are locked (i.e. the first 991 * time through). 992 */ 993 994 done = TRUE; /* be optimistic */ 995 gotpages = 0; /* # of pages we got so far */ 996 997 for (lcv = 0, current_offset = offset ; lcv < maxpages ; 998 lcv++, current_offset += PAGE_SIZE) { 999 /* do we care about this page? if not, skip it */ 1000 if (pps[lcv] == PGO_DONTCARE) 1001 continue; 1002 1003 ptmp = uvm_pagelookup(uobj, current_offset); 1004 1005 /* 1006 * if page is new, attempt to allocate the page, 1007 * zero-fill'd. 1008 */ 1009 if (ptmp == NULL && uao_find_swslot(aobj, 1010 current_offset >> PAGE_SHIFT) == 0) { 1011 ptmp = uvm_pagealloc(uobj, current_offset, 1012 NULL, UVM_PGA_ZERO); 1013 if (ptmp) { 1014 /* new page */ 1015 ptmp->flags &= ~(PG_BUSY|PG_FAKE); 1016 ptmp->pqflags |= PQ_AOBJ; 1017 UVM_PAGE_OWN(ptmp, NULL); 1018 } 1019 } 1020 1021 /* 1022 * to be useful must get a non-busy, non-released page 1023 */ 1024 if (ptmp == NULL || 1025 (ptmp->flags & (PG_BUSY|PG_RELEASED)) != 0) { 1026 if (lcv == centeridx || 1027 (flags & PGO_ALLPAGES) != 0) 1028 /* need to do a wait or I/O! */ 1029 done = FALSE; 1030 continue; 1031 } 1032 1033 /* 1034 * useful page: busy/lock it and plug it in our 1035 * result array 1036 */ 1037 /* caller must un-busy this page */ 1038 ptmp->flags |= PG_BUSY; 1039 UVM_PAGE_OWN(ptmp, "uao_get1"); 1040 pps[lcv] = ptmp; 1041 gotpages++; 1042 1043 } /* "for" lcv loop */ 1044 1045 /* 1046 * step 1b: now we've either done everything needed or we 1047 * to unlock and do some waiting or I/O. 1048 */ 1049 1050 UVMHIST_LOG(pdhist, "<- done (done=%d)", done, 0,0,0); 1051 1052 *npagesp = gotpages; 1053 if (done) 1054 /* bingo! */ 1055 return(VM_PAGER_OK); 1056 else 1057 /* EEK! Need to unlock and I/O */ 1058 return(VM_PAGER_UNLOCK); 1059 } 1060 1061 /* 1062 * step 2: get non-resident or busy pages. 1063 * object is locked. data structures are unlocked. 1064 */ 1065 1066 for (lcv = 0, current_offset = offset ; lcv < maxpages ; 1067 lcv++, current_offset += PAGE_SIZE) { 1068 1069 /* 1070 * - skip over pages we've already gotten or don't want 1071 * - skip over pages we don't _have_ to get 1072 */ 1073 1074 if (pps[lcv] != NULL || 1075 (lcv != centeridx && (flags & PGO_ALLPAGES) == 0)) 1076 continue; 1077 1078 pageidx = current_offset >> PAGE_SHIFT; 1079 1080 /* 1081 * we have yet to locate the current page (pps[lcv]). we 1082 * first look for a page that is already at the current offset. 1083 * if we find a page, we check to see if it is busy or 1084 * released. if that is the case, then we sleep on the page 1085 * until it is no longer busy or released and repeat the lookup. 1086 * if the page we found is neither busy nor released, then we 1087 * busy it (so we own it) and plug it into pps[lcv]. this 1088 * 'break's the following while loop and indicates we are 1089 * ready to move on to the next page in the "lcv" loop above. 1090 * 1091 * if we exit the while loop with pps[lcv] still set to NULL, 1092 * then it means that we allocated a new busy/fake/clean page 1093 * ptmp in the object and we need to do I/O to fill in the data. 1094 */ 1095 1096 /* top of "pps" while loop */ 1097 while (pps[lcv] == NULL) { 1098 /* look for a resident page */ 1099 ptmp = uvm_pagelookup(uobj, current_offset); 1100 1101 /* not resident? allocate one now (if we can) */ 1102 if (ptmp == NULL) { 1103 1104 ptmp = uvm_pagealloc(uobj, current_offset, 1105 NULL, 0); 1106 1107 /* out of RAM? */ 1108 if (ptmp == NULL) { 1109 simple_unlock(&uobj->vmobjlock); 1110 UVMHIST_LOG(pdhist, 1111 "sleeping, ptmp == NULL\n",0,0,0,0); 1112 uvm_wait("uao_getpage"); 1113 simple_lock(&uobj->vmobjlock); 1114 /* goto top of pps while loop */ 1115 continue; 1116 } 1117 1118 /* 1119 * safe with PQ's unlocked: because we just 1120 * alloc'd the page 1121 */ 1122 ptmp->pqflags |= PQ_AOBJ; 1123 1124 /* 1125 * got new page ready for I/O. break pps while 1126 * loop. pps[lcv] is still NULL. 1127 */ 1128 break; 1129 } 1130 1131 /* page is there, see if we need to wait on it */ 1132 if ((ptmp->flags & (PG_BUSY|PG_RELEASED)) != 0) { 1133 ptmp->flags |= PG_WANTED; 1134 UVMHIST_LOG(pdhist, 1135 "sleeping, ptmp->flags 0x%x\n", 1136 ptmp->flags,0,0,0); 1137 UVM_UNLOCK_AND_WAIT(ptmp, &uobj->vmobjlock, 1138 FALSE, "uao_get", 0); 1139 simple_lock(&uobj->vmobjlock); 1140 continue; /* goto top of pps while loop */ 1141 } 1142 1143 /* 1144 * if we get here then the page has become resident and 1145 * unbusy between steps 1 and 2. we busy it now (so we 1146 * own it) and set pps[lcv] (so that we exit the while 1147 * loop). 1148 */ 1149 /* we own it, caller must un-busy */ 1150 ptmp->flags |= PG_BUSY; 1151 UVM_PAGE_OWN(ptmp, "uao_get2"); 1152 pps[lcv] = ptmp; 1153 } 1154 1155 /* 1156 * if we own the valid page at the correct offset, pps[lcv] will 1157 * point to it. nothing more to do except go to the next page. 1158 */ 1159 if (pps[lcv]) 1160 continue; /* next lcv */ 1161 1162 /* 1163 * we have a "fake/busy/clean" page that we just allocated. 1164 * do the needed "i/o", either reading from swap or zeroing. 1165 */ 1166 swslot = uao_find_swslot(aobj, pageidx); 1167 1168 /* 1169 * just zero the page if there's nothing in swap. 1170 */ 1171 if (swslot == 0) 1172 { 1173 /* 1174 * page hasn't existed before, just zero it. 1175 */ 1176 uvm_pagezero(ptmp); 1177 } else { 1178 UVMHIST_LOG(pdhist, "pagein from swslot %d", 1179 swslot, 0,0,0); 1180 1181 /* 1182 * page in the swapped-out page. 1183 * unlock object for i/o, relock when done. 1184 */ 1185 simple_unlock(&uobj->vmobjlock); 1186 rv = uvm_swap_get(ptmp, swslot, PGO_SYNCIO); 1187 simple_lock(&uobj->vmobjlock); 1188 1189 /* 1190 * I/O done. check for errors. 1191 */ 1192 if (rv != VM_PAGER_OK) 1193 { 1194 UVMHIST_LOG(pdhist, "<- done (error=%d)", 1195 rv,0,0,0); 1196 if (ptmp->flags & PG_WANTED) 1197 wakeup(ptmp); 1198 1199 /* 1200 * remove the swap slot from the aobj 1201 * and mark the aobj as having no real slot. 1202 * don't free the swap slot, thus preventing 1203 * it from being used again. 1204 */ 1205 swslot = uao_set_swslot(&aobj->u_obj, pageidx, 1206 SWSLOT_BAD); 1207 uvm_swap_markbad(swslot, 1); 1208 1209 ptmp->flags &= ~(PG_WANTED|PG_BUSY); 1210 UVM_PAGE_OWN(ptmp, NULL); 1211 uvm_lock_pageq(); 1212 uvm_pagefree(ptmp); 1213 uvm_unlock_pageq(); 1214 1215 simple_unlock(&uobj->vmobjlock); 1216 return (rv); 1217 } 1218 } 1219 1220 /* 1221 * we got the page! clear the fake flag (indicates valid 1222 * data now in page) and plug into our result array. note 1223 * that page is still busy. 1224 * 1225 * it is the callers job to: 1226 * => check if the page is released 1227 * => unbusy the page 1228 * => activate the page 1229 */ 1230 1231 ptmp->flags &= ~PG_FAKE; /* data is valid ... */ 1232 pmap_clear_modify(ptmp); /* ... and clean */ 1233 pps[lcv] = ptmp; 1234 1235 } /* lcv loop */ 1236 1237 /* 1238 * finally, unlock object and return. 1239 */ 1240 1241 simple_unlock(&uobj->vmobjlock); 1242 UVMHIST_LOG(pdhist, "<- done (OK)",0,0,0,0); 1243 return(VM_PAGER_OK); 1244 } 1245 1246 /* 1247 * uao_releasepg: handle released page in an aobj 1248 * 1249 * => "pg" is a PG_BUSY [caller owns it], PG_RELEASED page that we need 1250 * to dispose of. 1251 * => caller must handle PG_WANTED case 1252 * => called with page's object locked, pageq's unlocked 1253 * => returns TRUE if page's object is still alive, FALSE if we 1254 * killed the page's object. if we return TRUE, then we 1255 * return with the object locked. 1256 * => if (nextpgp != NULL) => we return pageq.tqe_next here, and return 1257 * with the page queues locked [for pagedaemon] 1258 * => if (nextpgp == NULL) => we return with page queues unlocked [normal case] 1259 * => we kill the aobj if it is not referenced and we are suppose to 1260 * kill it ("KILLME"). 1261 */ 1262 static boolean_t 1263 uao_releasepg(pg, nextpgp) 1264 struct vm_page *pg; 1265 struct vm_page **nextpgp; /* OUT */ 1266 { 1267 struct uvm_aobj *aobj = (struct uvm_aobj *) pg->uobject; 1268 1269 #ifdef DIAGNOSTIC 1270 if ((pg->flags & PG_RELEASED) == 0) 1271 panic("uao_releasepg: page not released!"); 1272 #endif 1273 1274 /* 1275 * dispose of the page [caller handles PG_WANTED] and swap slot. 1276 */ 1277 pmap_page_protect(pg, VM_PROT_NONE); 1278 uao_dropswap(&aobj->u_obj, pg->offset >> PAGE_SHIFT); 1279 uvm_lock_pageq(); 1280 if (nextpgp) 1281 *nextpgp = pg->pageq.tqe_next; /* next page for daemon */ 1282 uvm_pagefree(pg); 1283 if (!nextpgp) 1284 uvm_unlock_pageq(); /* keep locked for daemon */ 1285 1286 /* 1287 * if we're not killing the object, we're done. 1288 */ 1289 if ((aobj->u_flags & UAO_FLAG_KILLME) == 0) 1290 return TRUE; 1291 1292 #ifdef DIAGNOSTIC 1293 if (aobj->u_obj.uo_refs) 1294 panic("uvm_km_releasepg: kill flag set on referenced object!"); 1295 #endif 1296 1297 /* 1298 * if there are still pages in the object, we're done for now. 1299 */ 1300 if (aobj->u_obj.uo_npages != 0) 1301 return TRUE; 1302 1303 #ifdef DIAGNOSTIC 1304 if (TAILQ_FIRST(&aobj->u_obj.memq)) 1305 panic("uvn_releasepg: pages in object with npages == 0"); 1306 #endif 1307 1308 /* 1309 * finally, free the rest. 1310 */ 1311 uao_free(aobj); 1312 1313 return FALSE; 1314 } 1315 1316 1317 /* 1318 * uao_dropswap: release any swap resources from this aobj page. 1319 * 1320 * => aobj must be locked or have a reference count of 0. 1321 */ 1322 1323 void 1324 uao_dropswap(uobj, pageidx) 1325 struct uvm_object *uobj; 1326 int pageidx; 1327 { 1328 int slot; 1329 1330 slot = uao_set_swslot(uobj, pageidx, 0); 1331 if (slot) { 1332 uvm_swap_free(slot, 1); 1333 } 1334 } 1335 1336 1337 /* 1338 * page in every page in every aobj that is paged-out to a range of swslots. 1339 * 1340 * => nothing should be locked. 1341 * => returns TRUE if pagein was aborted due to lack of memory. 1342 */ 1343 boolean_t 1344 uao_swap_off(startslot, endslot) 1345 int startslot, endslot; 1346 { 1347 struct uvm_aobj *aobj, *nextaobj; 1348 1349 /* 1350 * walk the list of all aobjs. 1351 */ 1352 1353 restart: 1354 simple_lock(&uao_list_lock); 1355 1356 for (aobj = LIST_FIRST(&uao_list); 1357 aobj != NULL; 1358 aobj = nextaobj) { 1359 boolean_t rv; 1360 1361 /* 1362 * try to get the object lock, 1363 * start all over if we fail. 1364 * most of the time we'll get the aobj lock, 1365 * so this should be a rare case. 1366 */ 1367 if (!simple_lock_try(&aobj->u_obj.vmobjlock)) { 1368 simple_unlock(&uao_list_lock); 1369 goto restart; 1370 } 1371 1372 /* 1373 * add a ref to the aobj so it doesn't disappear 1374 * while we're working. 1375 */ 1376 uao_reference_locked(&aobj->u_obj); 1377 1378 /* 1379 * now it's safe to unlock the uao list. 1380 */ 1381 simple_unlock(&uao_list_lock); 1382 1383 /* 1384 * page in any pages in the swslot range. 1385 * if there's an error, abort and return the error. 1386 */ 1387 rv = uao_pagein(aobj, startslot, endslot); 1388 if (rv) { 1389 uao_detach_locked(&aobj->u_obj); 1390 return rv; 1391 } 1392 1393 /* 1394 * we're done with this aobj. 1395 * relock the list and drop our ref on the aobj. 1396 */ 1397 simple_lock(&uao_list_lock); 1398 nextaobj = LIST_NEXT(aobj, u_list); 1399 uao_detach_locked(&aobj->u_obj); 1400 } 1401 1402 /* 1403 * done with traversal, unlock the list 1404 */ 1405 simple_unlock(&uao_list_lock); 1406 return FALSE; 1407 } 1408 1409 1410 /* 1411 * page in any pages from aobj in the given range. 1412 * 1413 * => aobj must be locked and is returned locked. 1414 * => returns TRUE if pagein was aborted due to lack of memory. 1415 */ 1416 static boolean_t 1417 uao_pagein(aobj, startslot, endslot) 1418 struct uvm_aobj *aobj; 1419 int startslot, endslot; 1420 { 1421 boolean_t rv; 1422 1423 if (UAO_USES_SWHASH(aobj)) { 1424 struct uao_swhash_elt *elt; 1425 int bucket; 1426 1427 restart: 1428 for (bucket = aobj->u_swhashmask; bucket >= 0; bucket--) { 1429 for (elt = LIST_FIRST(&aobj->u_swhash[bucket]); 1430 elt != NULL; 1431 elt = LIST_NEXT(elt, list)) { 1432 int i; 1433 1434 for (i = 0; i < UAO_SWHASH_CLUSTER_SIZE; i++) { 1435 int slot = elt->slots[i]; 1436 1437 /* 1438 * if the slot isn't in range, skip it. 1439 */ 1440 if (slot < startslot || 1441 slot >= endslot) { 1442 continue; 1443 } 1444 1445 /* 1446 * process the page, 1447 * the start over on this object 1448 * since the swhash elt 1449 * may have been freed. 1450 */ 1451 rv = uao_pagein_page(aobj, 1452 UAO_SWHASH_ELT_PAGEIDX_BASE(elt) + i); 1453 if (rv) { 1454 return rv; 1455 } 1456 goto restart; 1457 } 1458 } 1459 } 1460 } else { 1461 int i; 1462 1463 for (i = 0; i < aobj->u_pages; i++) { 1464 int slot = aobj->u_swslots[i]; 1465 1466 /* 1467 * if the slot isn't in range, skip it 1468 */ 1469 if (slot < startslot || slot >= endslot) { 1470 continue; 1471 } 1472 1473 /* 1474 * process the page. 1475 */ 1476 rv = uao_pagein_page(aobj, i); 1477 if (rv) { 1478 return rv; 1479 } 1480 } 1481 } 1482 1483 return FALSE; 1484 } 1485 1486 /* 1487 * page in a page from an aobj. used for swap_off. 1488 * returns TRUE if pagein was aborted due to lack of memory. 1489 * 1490 * => aobj must be locked and is returned locked. 1491 */ 1492 static boolean_t 1493 uao_pagein_page(aobj, pageidx) 1494 struct uvm_aobj *aobj; 1495 int pageidx; 1496 { 1497 struct vm_page *pg; 1498 int rv, slot, npages; 1499 UVMHIST_FUNC("uao_pagein_page"); UVMHIST_CALLED(pdhist); 1500 1501 pg = NULL; 1502 npages = 1; 1503 /* locked: aobj */ 1504 rv = uao_get(&aobj->u_obj, pageidx << PAGE_SHIFT, 1505 &pg, &npages, 0, VM_PROT_READ|VM_PROT_WRITE, 0, 0); 1506 /* unlocked: aobj */ 1507 1508 /* 1509 * relock and finish up. 1510 */ 1511 simple_lock(&aobj->u_obj.vmobjlock); 1512 1513 switch (rv) { 1514 case VM_PAGER_OK: 1515 break; 1516 1517 case VM_PAGER_ERROR: 1518 case VM_PAGER_REFAULT: 1519 /* 1520 * nothing more to do on errors. 1521 * VM_PAGER_REFAULT can only mean that the anon was freed, 1522 * so again there's nothing to do. 1523 */ 1524 return FALSE; 1525 1526 #ifdef DIAGNOSTIC 1527 default: 1528 panic("uao_pagein_page: uao_get -> %d\n", rv); 1529 #endif 1530 } 1531 1532 #ifdef DIAGNOSTIC 1533 /* 1534 * this should never happen, since we have a reference on the aobj. 1535 */ 1536 if (pg->flags & PG_RELEASED) { 1537 panic("uao_pagein_page: found PG_RELEASED page?\n"); 1538 } 1539 #endif 1540 1541 /* 1542 * ok, we've got the page now. 1543 * mark it as dirty, clear its swslot and un-busy it. 1544 */ 1545 slot = uao_set_swslot(&aobj->u_obj, pageidx, 0); 1546 uvm_swap_free(slot, 1); 1547 pg->flags &= ~(PG_BUSY|PG_CLEAN|PG_FAKE); 1548 UVM_PAGE_OWN(pg, NULL); 1549 1550 /* 1551 * deactivate the page (to put it on a page queue). 1552 */ 1553 pmap_clear_reference(pg); 1554 pmap_page_protect(pg, VM_PROT_NONE); 1555 uvm_lock_pageq(); 1556 uvm_pagedeactivate(pg); 1557 uvm_unlock_pageq(); 1558 1559 return FALSE; 1560 } 1561