145749Smckusick /* 245749Smckusick * Copyright (c) 1990 University of Utah. 363379Sbostic * Copyright (c) 1991, 1993 463379Sbostic * The Regents of the University of California. All rights reserved. 545749Smckusick * 645749Smckusick * This code is derived from software contributed to Berkeley by 745749Smckusick * the Systems Programming Group of the University of Utah Computer 845749Smckusick * Science Department. 945749Smckusick * 1045749Smckusick * %sccs.include.redist.c% 1145749Smckusick * 1249289Shibler * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$ 1349289Shibler * 14*65723Shibler * @(#)swap_pager.c 8.8 (Berkeley) 01/13/94 1545749Smckusick */ 1645749Smckusick 1745749Smckusick /* 1845749Smckusick * Quick hack to page to dedicated partition(s). 1945749Smckusick * TODO: 2045749Smckusick * Add multiprocessor locks 2145749Smckusick * Deal with async writes in a better fashion 2245749Smckusick */ 2345749Smckusick 2453341Sbostic #include <sys/param.h> 2553341Sbostic #include <sys/systm.h> 2653341Sbostic #include <sys/proc.h> 2753341Sbostic #include <sys/buf.h> 2853341Sbostic #include <sys/map.h> 2953496Sheideman #include <sys/vnode.h> 3053341Sbostic #include <sys/malloc.h> 3145749Smckusick 3255051Spendry #include <miscfs/specfs/specdev.h> 3355051Spendry 3453341Sbostic #include <vm/vm.h> 3553341Sbostic #include <vm/vm_page.h> 3653341Sbostic #include <vm/vm_pageout.h> 3753341Sbostic #include <vm/swap_pager.h> 3845749Smckusick 3945749Smckusick #define NSWSIZES 16 /* size of swtab */ 4065682Shibler #define MAXDADDRS 64 /* max # of disk addrs for fixed allocations */ 4165682Shibler #ifndef NPENDINGIO 4245749Smckusick #define NPENDINGIO 64 /* max # of pending cleans */ 4365682Shibler #endif 4445749Smckusick 4545749Smckusick #ifdef DEBUG 4645749Smckusick int swpagerdebug = 0x100; 4745749Smckusick #define SDB_FOLLOW 0x001 4845749Smckusick #define SDB_INIT 0x002 4945749Smckusick #define SDB_ALLOC 0x004 5045749Smckusick #define SDB_IO 0x008 5145749Smckusick #define SDB_WRITE 0x010 5245749Smckusick #define SDB_FAIL 0x020 5345749Smckusick #define SDB_ALLOCBLK 0x040 5445749Smckusick #define SDB_FULL 0x080 5545749Smckusick #define SDB_ANOM 0x100 5645749Smckusick #define SDB_ANOMPANIC 0x200 5765682Shibler #define SDB_CLUSTER 0x400 5865682Shibler #define SDB_PARANOIA 0x800 5945749Smckusick #endif 6045749Smckusick 6165231Smckusick TAILQ_HEAD(swpclean, swpagerclean); 6265231Smckusick 6345749Smckusick struct swpagerclean { 6465231Smckusick TAILQ_ENTRY(swpagerclean) spc_list; 6565231Smckusick int spc_flags; 6665231Smckusick struct buf *spc_bp; 6765231Smckusick sw_pager_t spc_swp; 6865231Smckusick vm_offset_t spc_kva; 6965231Smckusick vm_page_t spc_m; 7065682Shibler int spc_npages; 7145749Smckusick } swcleanlist[NPENDINGIO]; 7253341Sbostic typedef struct swpagerclean *swp_clean_t; 7345749Smckusick 7445749Smckusick /* spc_flags values */ 7545749Smckusick #define SPC_FREE 0x00 7645749Smckusick #define SPC_BUSY 0x01 7745749Smckusick #define SPC_DONE 0x02 7845749Smckusick #define SPC_ERROR 0x04 7945749Smckusick 8045749Smckusick struct swtab { 8145749Smckusick vm_size_t st_osize; /* size of object (bytes) */ 8245749Smckusick int st_bsize; /* vs. size of swap block (DEV_BSIZE units) */ 8345749Smckusick #ifdef DEBUG 8445749Smckusick u_long st_inuse; /* number in this range in use */ 8545749Smckusick u_long st_usecnt; /* total used of this size */ 8645749Smckusick #endif 8745749Smckusick } swtab[NSWSIZES+1]; 8845749Smckusick 8945749Smckusick #ifdef DEBUG 9045749Smckusick int swap_pager_poip; /* pageouts in progress */ 9145749Smckusick int swap_pager_piip; /* pageins in progress */ 9245749Smckusick #endif 9345749Smckusick 9465682Shibler int swap_pager_maxcluster; /* maximum cluster size */ 9565682Shibler int swap_pager_npendingio; /* number of pager clean structs */ 9665682Shibler 9765231Smckusick struct swpclean swap_pager_inuse; /* list of pending page cleans */ 9865231Smckusick struct swpclean swap_pager_free; /* list of free pager clean structs */ 9965231Smckusick struct pagerlst swap_pager_list; /* list of "named" anon regions */ 10045749Smckusick 10153341Sbostic static void swap_pager_init __P((void)); 10264827Storek static vm_pager_t swap_pager_alloc 10364827Storek __P((caddr_t, vm_size_t, vm_prot_t, vm_offset_t)); 10465682Shibler static void swap_pager_clean __P((int)); 10565682Shibler #ifdef DEBUG 10665682Shibler static void swap_pager_clean_check __P((vm_page_t *, int, int)); 10765682Shibler #endif 10865682Shibler static void swap_pager_cluster 10965682Shibler __P((vm_pager_t, vm_offset_t, 11065682Shibler vm_offset_t *, vm_offset_t *)); 11153341Sbostic static void swap_pager_dealloc __P((vm_pager_t)); 11253341Sbostic static int swap_pager_getpage 11365682Shibler __P((vm_pager_t, vm_page_t *, int, boolean_t)); 11453341Sbostic static boolean_t swap_pager_haspage __P((vm_pager_t, vm_offset_t)); 11565682Shibler static int swap_pager_io __P((sw_pager_t, vm_page_t *, int, int)); 11653341Sbostic static void swap_pager_iodone __P((struct buf *)); 11753341Sbostic static int swap_pager_putpage 11865682Shibler __P((vm_pager_t, vm_page_t *, int, boolean_t)); 11953341Sbostic 12053341Sbostic struct pagerops swappagerops = { 12153341Sbostic swap_pager_init, 12253341Sbostic swap_pager_alloc, 12353341Sbostic swap_pager_dealloc, 12453341Sbostic swap_pager_getpage, 12553341Sbostic swap_pager_putpage, 12665682Shibler swap_pager_haspage, 12765682Shibler swap_pager_cluster 12853341Sbostic }; 12953341Sbostic 13053341Sbostic static void 13145749Smckusick swap_pager_init() 13245749Smckusick { 13345749Smckusick register swp_clean_t spc; 13445749Smckusick register int i, bsize; 13545749Smckusick extern int dmmin, dmmax; 13645749Smckusick int maxbsize; 13745749Smckusick 13845749Smckusick #ifdef DEBUG 13945749Smckusick if (swpagerdebug & (SDB_FOLLOW|SDB_INIT)) 14045749Smckusick printf("swpg_init()\n"); 14145749Smckusick #endif 14245749Smckusick dfltpagerops = &swappagerops; 14365231Smckusick TAILQ_INIT(&swap_pager_list); 14445749Smckusick 14545749Smckusick /* 14665682Shibler * Allocate async IO structures. 14765682Shibler * 14865682Shibler * XXX it would be nice if we could do this dynamically based on 14965682Shibler * the value of nswbuf (since we are ultimately limited by that) 15065682Shibler * but neither nswbuf or malloc has been initialized yet. So the 15165682Shibler * structs are statically allocated above. 15265682Shibler */ 15365682Shibler swap_pager_npendingio = NPENDINGIO; 15465682Shibler 15565682Shibler /* 15645749Smckusick * Initialize clean lists 15745749Smckusick */ 15865231Smckusick TAILQ_INIT(&swap_pager_inuse); 15965231Smckusick TAILQ_INIT(&swap_pager_free); 16065682Shibler for (i = 0, spc = swcleanlist; i < swap_pager_npendingio; i++, spc++) { 16165231Smckusick TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 16245749Smckusick spc->spc_flags = SPC_FREE; 16345749Smckusick } 16445749Smckusick 16545749Smckusick /* 16645749Smckusick * Calculate the swap allocation constants. 16745749Smckusick */ 16845749Smckusick if (dmmin == 0) { 16945749Smckusick dmmin = DMMIN; 17045749Smckusick if (dmmin < CLBYTES/DEV_BSIZE) 17145749Smckusick dmmin = CLBYTES/DEV_BSIZE; 17245749Smckusick } 17345749Smckusick if (dmmax == 0) 17445749Smckusick dmmax = DMMAX; 17545749Smckusick 17645749Smckusick /* 17745749Smckusick * Fill in our table of object size vs. allocation size 17845749Smckusick */ 17945749Smckusick bsize = btodb(PAGE_SIZE); 18045749Smckusick if (bsize < dmmin) 18145749Smckusick bsize = dmmin; 18245749Smckusick maxbsize = btodb(sizeof(sw_bm_t) * NBBY * PAGE_SIZE); 18345749Smckusick if (maxbsize > dmmax) 18445749Smckusick maxbsize = dmmax; 18545749Smckusick for (i = 0; i < NSWSIZES; i++) { 18645749Smckusick swtab[i].st_osize = (vm_size_t) (MAXDADDRS * dbtob(bsize)); 18745749Smckusick swtab[i].st_bsize = bsize; 18865682Shibler if (bsize <= btodb(MAXPHYS)) 18965682Shibler swap_pager_maxcluster = dbtob(bsize); 19045749Smckusick #ifdef DEBUG 19145749Smckusick if (swpagerdebug & SDB_INIT) 19245749Smckusick printf("swpg_init: ix %d, size %x, bsize %x\n", 19345749Smckusick i, swtab[i].st_osize, swtab[i].st_bsize); 19445749Smckusick #endif 19545749Smckusick if (bsize >= maxbsize) 19645749Smckusick break; 19745749Smckusick bsize *= 2; 19845749Smckusick } 19945749Smckusick swtab[i].st_osize = 0; 20045749Smckusick swtab[i].st_bsize = bsize; 20145749Smckusick } 20245749Smckusick 20345749Smckusick /* 20445749Smckusick * Allocate a pager structure and associated resources. 20545749Smckusick * Note that if we are called from the pageout daemon (handle == NULL) 20645749Smckusick * we should not wait for memory as it could resulting in deadlock. 20745749Smckusick */ 20853341Sbostic static vm_pager_t 20964827Storek swap_pager_alloc(handle, size, prot, foff) 21045749Smckusick caddr_t handle; 21145749Smckusick register vm_size_t size; 21245749Smckusick vm_prot_t prot; 21364827Storek vm_offset_t foff; 21445749Smckusick { 21545749Smckusick register vm_pager_t pager; 21645749Smckusick register sw_pager_t swp; 21745749Smckusick struct swtab *swt; 21845749Smckusick int waitok; 21945749Smckusick 22045749Smckusick #ifdef DEBUG 22145749Smckusick if (swpagerdebug & (SDB_FOLLOW|SDB_ALLOC)) 22245749Smckusick printf("swpg_alloc(%x, %x, %x)\n", handle, size, prot); 22345749Smckusick #endif 22445749Smckusick /* 22545749Smckusick * If this is a "named" anonymous region, look it up and 22645749Smckusick * return the appropriate pager if it exists. 22745749Smckusick */ 22845749Smckusick if (handle) { 22945749Smckusick pager = vm_pager_lookup(&swap_pager_list, handle); 23048386Skarels if (pager != NULL) { 23145749Smckusick /* 23245749Smckusick * Use vm_object_lookup to gain a reference 23345749Smckusick * to the object and also to remove from the 23445749Smckusick * object cache. 23545749Smckusick */ 23648386Skarels if (vm_object_lookup(pager) == NULL) 23745749Smckusick panic("swap_pager_alloc: bad object"); 23845749Smckusick return(pager); 23945749Smckusick } 24045749Smckusick } 24145749Smckusick /* 24245749Smckusick * Pager doesn't exist, allocate swap management resources 24345749Smckusick * and initialize. 24445749Smckusick */ 24545749Smckusick waitok = handle ? M_WAITOK : M_NOWAIT; 24645749Smckusick pager = (vm_pager_t)malloc(sizeof *pager, M_VMPAGER, waitok); 24748386Skarels if (pager == NULL) 24848386Skarels return(NULL); 24945749Smckusick swp = (sw_pager_t)malloc(sizeof *swp, M_VMPGDATA, waitok); 25045749Smckusick if (swp == NULL) { 25145749Smckusick #ifdef DEBUG 25245749Smckusick if (swpagerdebug & SDB_FAIL) 25345749Smckusick printf("swpg_alloc: swpager malloc failed\n"); 25445749Smckusick #endif 25545749Smckusick free((caddr_t)pager, M_VMPAGER); 25648386Skarels return(NULL); 25745749Smckusick } 25845749Smckusick size = round_page(size); 25945749Smckusick for (swt = swtab; swt->st_osize; swt++) 26045749Smckusick if (size <= swt->st_osize) 26145749Smckusick break; 26245749Smckusick #ifdef DEBUG 26345749Smckusick swt->st_inuse++; 26445749Smckusick swt->st_usecnt++; 26545749Smckusick #endif 26645749Smckusick swp->sw_osize = size; 26745749Smckusick swp->sw_bsize = swt->st_bsize; 26845749Smckusick swp->sw_nblocks = (btodb(size) + swp->sw_bsize - 1) / swp->sw_bsize; 26945749Smckusick swp->sw_blocks = (sw_blk_t) 27045749Smckusick malloc(swp->sw_nblocks*sizeof(*swp->sw_blocks), 27145749Smckusick M_VMPGDATA, M_NOWAIT); 27245749Smckusick if (swp->sw_blocks == NULL) { 27345749Smckusick free((caddr_t)swp, M_VMPGDATA); 27445749Smckusick free((caddr_t)pager, M_VMPAGER); 27545749Smckusick #ifdef DEBUG 27645749Smckusick if (swpagerdebug & SDB_FAIL) 27745749Smckusick printf("swpg_alloc: sw_blocks malloc failed\n"); 27845749Smckusick swt->st_inuse--; 27945749Smckusick swt->st_usecnt--; 28045749Smckusick #endif 28145749Smckusick return(FALSE); 28245749Smckusick } 28345749Smckusick bzero((caddr_t)swp->sw_blocks, 28445749Smckusick swp->sw_nblocks * sizeof(*swp->sw_blocks)); 28545749Smckusick swp->sw_poip = 0; 28645749Smckusick if (handle) { 28745749Smckusick vm_object_t object; 28845749Smckusick 28945749Smckusick swp->sw_flags = SW_NAMED; 29065231Smckusick TAILQ_INSERT_TAIL(&swap_pager_list, pager, pg_list); 29145749Smckusick /* 29245749Smckusick * Consistant with other pagers: return with object 29345749Smckusick * referenced. Can't do this with handle == NULL 29445749Smckusick * since it might be the pageout daemon calling. 29545749Smckusick */ 29645749Smckusick object = vm_object_allocate(size); 29745749Smckusick vm_object_enter(object, pager); 29845749Smckusick vm_object_setpager(object, pager, 0, FALSE); 29945749Smckusick } else { 30045749Smckusick swp->sw_flags = 0; 30165231Smckusick pager->pg_list.tqe_next = NULL; 30265231Smckusick pager->pg_list.tqe_prev = NULL; 30345749Smckusick } 30445749Smckusick pager->pg_handle = handle; 30545749Smckusick pager->pg_ops = &swappagerops; 30645749Smckusick pager->pg_type = PG_SWAP; 30765682Shibler pager->pg_flags = PG_CLUSTERPUT; 30864860Shibler pager->pg_data = swp; 30945749Smckusick 31045749Smckusick #ifdef DEBUG 31145749Smckusick if (swpagerdebug & SDB_ALLOC) 31245749Smckusick printf("swpg_alloc: pg_data %x, %x of %x at %x\n", 31345749Smckusick swp, swp->sw_nblocks, swp->sw_bsize, swp->sw_blocks); 31445749Smckusick #endif 31545749Smckusick return(pager); 31645749Smckusick } 31745749Smckusick 31853341Sbostic static void 31945749Smckusick swap_pager_dealloc(pager) 32045749Smckusick vm_pager_t pager; 32145749Smckusick { 32245749Smckusick register int i; 32345749Smckusick register sw_blk_t bp; 32445749Smckusick register sw_pager_t swp; 32545749Smckusick struct swtab *swt; 32645749Smckusick int s; 32745749Smckusick 32845749Smckusick #ifdef DEBUG 32945749Smckusick /* save panic time state */ 33045749Smckusick if ((swpagerdebug & SDB_ANOMPANIC) && panicstr) 33145749Smckusick return; 33245749Smckusick if (swpagerdebug & (SDB_FOLLOW|SDB_ALLOC)) 33345749Smckusick printf("swpg_dealloc(%x)\n", pager); 33445749Smckusick #endif 33545749Smckusick /* 33645749Smckusick * Remove from list right away so lookups will fail if we 33745749Smckusick * block for pageout completion. 33845749Smckusick */ 33945749Smckusick swp = (sw_pager_t) pager->pg_data; 34045749Smckusick if (swp->sw_flags & SW_NAMED) { 34165231Smckusick TAILQ_REMOVE(&swap_pager_list, pager, pg_list); 34245749Smckusick swp->sw_flags &= ~SW_NAMED; 34345749Smckusick } 34445749Smckusick #ifdef DEBUG 34545749Smckusick for (swt = swtab; swt->st_osize; swt++) 34645749Smckusick if (swp->sw_osize <= swt->st_osize) 34745749Smckusick break; 34845749Smckusick swt->st_inuse--; 34945749Smckusick #endif 35045749Smckusick 35145749Smckusick /* 35245749Smckusick * Wait for all pageouts to finish and remove 35345749Smckusick * all entries from cleaning list. 35445749Smckusick */ 35545749Smckusick s = splbio(); 35645749Smckusick while (swp->sw_poip) { 35745749Smckusick swp->sw_flags |= SW_WANTED; 35865682Shibler (void) tsleep(swp, PVM, "swpgdealloc", 0); 35945749Smckusick } 36045749Smckusick splx(s); 36165682Shibler swap_pager_clean(B_WRITE); 36245749Smckusick 36345749Smckusick /* 36445749Smckusick * Free left over swap blocks 36545749Smckusick */ 36645749Smckusick for (i = 0, bp = swp->sw_blocks; i < swp->sw_nblocks; i++, bp++) 36745749Smckusick if (bp->swb_block) { 36845749Smckusick #ifdef DEBUG 36945749Smckusick if (swpagerdebug & (SDB_ALLOCBLK|SDB_FULL)) 37045749Smckusick printf("swpg_dealloc: blk %x\n", 37145749Smckusick bp->swb_block); 37245749Smckusick #endif 37345749Smckusick rmfree(swapmap, swp->sw_bsize, bp->swb_block); 37445749Smckusick } 37545749Smckusick /* 37645749Smckusick * Free swap management resources 37745749Smckusick */ 37845749Smckusick free((caddr_t)swp->sw_blocks, M_VMPGDATA); 37945749Smckusick free((caddr_t)swp, M_VMPGDATA); 38045749Smckusick free((caddr_t)pager, M_VMPAGER); 38145749Smckusick } 38245749Smckusick 38353341Sbostic static int 38465682Shibler swap_pager_getpage(pager, mlist, npages, sync) 38545749Smckusick vm_pager_t pager; 38665682Shibler vm_page_t *mlist; 38765682Shibler int npages; 38845749Smckusick boolean_t sync; 38945749Smckusick { 39045749Smckusick #ifdef DEBUG 39145749Smckusick if (swpagerdebug & SDB_FOLLOW) 39265682Shibler printf("swpg_getpage(%x, %x, %x, %x)\n", 39365682Shibler pager, mlist, npages, sync); 39445749Smckusick #endif 39565682Shibler return(swap_pager_io((sw_pager_t)pager->pg_data, 39665682Shibler mlist, npages, B_READ)); 39745749Smckusick } 39845749Smckusick 39953341Sbostic static int 40065682Shibler swap_pager_putpage(pager, mlist, npages, sync) 40145749Smckusick vm_pager_t pager; 40265682Shibler vm_page_t *mlist; 40365682Shibler int npages; 40445749Smckusick boolean_t sync; 40545749Smckusick { 40645749Smckusick int flags; 40745749Smckusick 40845749Smckusick #ifdef DEBUG 40945749Smckusick if (swpagerdebug & SDB_FOLLOW) 41065682Shibler printf("swpg_putpage(%x, %x, %x, %x)\n", 41165682Shibler pager, mlist, npages, sync); 41245749Smckusick #endif 41348386Skarels if (pager == NULL) { 41465682Shibler swap_pager_clean(B_WRITE); 41554817Storek return (VM_PAGER_OK); /* ??? */ 41645749Smckusick } 41745749Smckusick flags = B_WRITE; 41845749Smckusick if (!sync) 41945749Smckusick flags |= B_ASYNC; 42065682Shibler return(swap_pager_io((sw_pager_t)pager->pg_data, 42165682Shibler mlist, npages, flags)); 42245749Smckusick } 42345749Smckusick 42453341Sbostic static boolean_t 42545749Smckusick swap_pager_haspage(pager, offset) 42645749Smckusick vm_pager_t pager; 42745749Smckusick vm_offset_t offset; 42845749Smckusick { 42945749Smckusick register sw_pager_t swp; 43045749Smckusick register sw_blk_t swb; 43145749Smckusick int ix; 43245749Smckusick 43345749Smckusick #ifdef DEBUG 43445749Smckusick if (swpagerdebug & (SDB_FOLLOW|SDB_ALLOCBLK)) 43545749Smckusick printf("swpg_haspage(%x, %x) ", pager, offset); 43645749Smckusick #endif 43745749Smckusick swp = (sw_pager_t) pager->pg_data; 43845749Smckusick ix = offset / dbtob(swp->sw_bsize); 43945749Smckusick if (swp->sw_blocks == NULL || ix >= swp->sw_nblocks) { 44045749Smckusick #ifdef DEBUG 44145749Smckusick if (swpagerdebug & (SDB_FAIL|SDB_FOLLOW|SDB_ALLOCBLK)) 44245749Smckusick printf("swpg_haspage: %x bad offset %x, ix %x\n", 44345749Smckusick swp->sw_blocks, offset, ix); 44445749Smckusick #endif 44545749Smckusick return(FALSE); 44645749Smckusick } 44745749Smckusick swb = &swp->sw_blocks[ix]; 44845749Smckusick if (swb->swb_block) 44945749Smckusick ix = atop(offset % dbtob(swp->sw_bsize)); 45045749Smckusick #ifdef DEBUG 45145749Smckusick if (swpagerdebug & SDB_ALLOCBLK) 45245749Smckusick printf("%x blk %x+%x ", swp->sw_blocks, swb->swb_block, ix); 45345749Smckusick if (swpagerdebug & (SDB_FOLLOW|SDB_ALLOCBLK)) 45445749Smckusick printf("-> %c\n", 45545749Smckusick "FT"[swb->swb_block && (swb->swb_mask & (1 << ix))]); 45645749Smckusick #endif 45745749Smckusick if (swb->swb_block && (swb->swb_mask & (1 << ix))) 45845749Smckusick return(TRUE); 45945749Smckusick return(FALSE); 46045749Smckusick } 46145749Smckusick 46265682Shibler static void 46365682Shibler swap_pager_cluster(pager, offset, loffset, hoffset) 46465682Shibler vm_pager_t pager; 46565682Shibler vm_offset_t offset; 46665682Shibler vm_offset_t *loffset; 46765682Shibler vm_offset_t *hoffset; 46865682Shibler { 46965682Shibler sw_pager_t swp; 47065682Shibler register int bsize; 47165682Shibler vm_offset_t loff, hoff; 47265682Shibler 47365682Shibler #ifdef DEBUG 47465682Shibler if (swpagerdebug & (SDB_FOLLOW|SDB_CLUSTER)) 47565682Shibler printf("swpg_cluster(%x, %x) ", pager, offset); 47665682Shibler #endif 47765682Shibler swp = (sw_pager_t) pager->pg_data; 47865682Shibler bsize = dbtob(swp->sw_bsize); 47965682Shibler if (bsize > swap_pager_maxcluster) 48065682Shibler bsize = swap_pager_maxcluster; 48165682Shibler 48265682Shibler loff = offset - (offset % bsize); 48365682Shibler if (loff >= swp->sw_osize) 48465682Shibler panic("swap_pager_cluster: bad offset"); 48565682Shibler 48665682Shibler hoff = loff + bsize; 48765682Shibler if (hoff > swp->sw_osize) 48865682Shibler hoff = swp->sw_osize; 48965682Shibler 49065682Shibler *loffset = loff; 49165682Shibler *hoffset = hoff; 49265682Shibler #ifdef DEBUG 49365682Shibler if (swpagerdebug & (SDB_FOLLOW|SDB_CLUSTER)) 49465682Shibler printf("returns [%x-%x]\n", loff, hoff); 49565682Shibler #endif 49665682Shibler } 49765682Shibler 49845749Smckusick /* 49945749Smckusick * Scaled down version of swap(). 50045749Smckusick * Assumes that PAGE_SIZE < MAXPHYS; i.e. only one operation needed. 50145749Smckusick * BOGUS: lower level IO routines expect a KVA so we have to map our 50245749Smckusick * provided physical page into the KVA to keep them happy. 50345749Smckusick */ 50453341Sbostic static int 50565682Shibler swap_pager_io(swp, mlist, npages, flags) 50645749Smckusick register sw_pager_t swp; 50765682Shibler vm_page_t *mlist; 50865682Shibler int npages; 50945749Smckusick int flags; 51045749Smckusick { 51145749Smckusick register struct buf *bp; 51245749Smckusick register sw_blk_t swb; 51345749Smckusick register int s; 51465682Shibler int ix, mask; 51545749Smckusick boolean_t rv; 51645749Smckusick vm_offset_t kva, off; 51745749Smckusick swp_clean_t spc; 51865682Shibler vm_page_t m; 51945749Smckusick 52045749Smckusick #ifdef DEBUG 52145749Smckusick /* save panic time state */ 52245749Smckusick if ((swpagerdebug & SDB_ANOMPANIC) && panicstr) 52353341Sbostic return (VM_PAGER_FAIL); /* XXX: correct return? */ 52445749Smckusick if (swpagerdebug & (SDB_FOLLOW|SDB_IO)) 52565682Shibler printf("swpg_io(%x, %x, %x, %x)\n", swp, mlist, npages, flags); 52665682Shibler if (flags & B_READ) { 52765682Shibler if (flags & B_ASYNC) 52865682Shibler panic("swap_pager_io: cannot do ASYNC reads"); 52965682Shibler if (npages != 1) 53065682Shibler panic("swap_pager_io: cannot do clustered reads"); 53165682Shibler } 53245749Smckusick #endif 53345749Smckusick 53445749Smckusick /* 53564860Shibler * First determine if the page exists in the pager if this is 53664860Shibler * a sync read. This quickly handles cases where we are 53764860Shibler * following shadow chains looking for the top level object 53864860Shibler * with the page. 53964860Shibler */ 54065682Shibler m = *mlist; 54164860Shibler off = m->offset + m->object->paging_offset; 54264860Shibler ix = off / dbtob(swp->sw_bsize); 54365682Shibler if (swp->sw_blocks == NULL || ix >= swp->sw_nblocks) { 54465682Shibler #ifdef DEBUG 54565682Shibler if ((flags & B_READ) == 0 && (swpagerdebug & SDB_ANOM)) { 54665682Shibler printf("swap_pager_io: no swap block on write\n"); 54765682Shibler return(VM_PAGER_BAD); 54865682Shibler } 54965682Shibler #endif 55064860Shibler return(VM_PAGER_FAIL); 55165682Shibler } 55264860Shibler swb = &swp->sw_blocks[ix]; 55364860Shibler off = off % dbtob(swp->sw_bsize); 55464860Shibler if ((flags & B_READ) && 55564860Shibler (swb->swb_block == 0 || (swb->swb_mask & (1 << atop(off))) == 0)) 55664860Shibler return(VM_PAGER_FAIL); 55764860Shibler 55864860Shibler /* 55945749Smckusick * For reads (pageins) and synchronous writes, we clean up 56049289Shibler * all completed async pageouts. 56145749Smckusick */ 56245749Smckusick if ((flags & B_ASYNC) == 0) { 56345749Smckusick s = splbio(); 56465682Shibler swap_pager_clean(flags&B_READ); 56549289Shibler #ifdef DEBUG 56665682Shibler if (swpagerdebug & SDB_PARANOIA) 56765682Shibler swap_pager_clean_check(mlist, npages, flags&B_READ); 56849289Shibler #endif 56945749Smckusick splx(s); 57045749Smckusick } 57145749Smckusick /* 57245749Smckusick * For async writes (pageouts), we cleanup completed pageouts so 57345749Smckusick * that all available resources are freed. Also tells us if this 57445749Smckusick * page is already being cleaned. If it is, or no resources 57545749Smckusick * are available, we try again later. 57645749Smckusick */ 57765682Shibler else { 57865682Shibler swap_pager_clean(B_WRITE); 57949289Shibler #ifdef DEBUG 58065682Shibler if (swpagerdebug & SDB_PARANOIA) 58165682Shibler swap_pager_clean_check(mlist, npages, B_WRITE); 58249289Shibler #endif 58365682Shibler if (swap_pager_free.tqh_first == NULL) { 58465682Shibler #ifdef DEBUG 58565682Shibler if (swpagerdebug & SDB_FAIL) 58665682Shibler printf("%s: no available io headers\n", 58765682Shibler "swap_pager_io"); 58865682Shibler #endif 58965682Shibler return(VM_PAGER_AGAIN); 59065682Shibler } 59149289Shibler } 59245749Smckusick 59345749Smckusick /* 59464860Shibler * Allocate a swap block if necessary. 59545749Smckusick */ 59664860Shibler if (swb->swb_block == 0) { 59745749Smckusick swb->swb_block = rmalloc(swapmap, swp->sw_bsize); 59845749Smckusick if (swb->swb_block == 0) { 59945749Smckusick #ifdef DEBUG 60045749Smckusick if (swpagerdebug & SDB_FAIL) 60145749Smckusick printf("swpg_io: rmalloc of %x failed\n", 60245749Smckusick swp->sw_bsize); 60345749Smckusick #endif 60465682Shibler /* 60565682Shibler * XXX this is technically a resource shortage that 60665682Shibler * should return AGAIN, but the situation isn't likely 60765682Shibler * to be remedied just by delaying a little while and 60865682Shibler * trying again (the pageout daemon's current response 60965682Shibler * to AGAIN) so we just return FAIL. 61065682Shibler */ 61145749Smckusick return(VM_PAGER_FAIL); 61245749Smckusick } 61345749Smckusick #ifdef DEBUG 61445749Smckusick if (swpagerdebug & (SDB_FULL|SDB_ALLOCBLK)) 61545749Smckusick printf("swpg_io: %x alloc blk %x at ix %x\n", 61645749Smckusick swp->sw_blocks, swb->swb_block, ix); 61745749Smckusick #endif 61845749Smckusick } 61945749Smckusick 62045749Smckusick /* 62145749Smckusick * Allocate a kernel virtual address and initialize so that PTE 62245749Smckusick * is available for lower level IO drivers. 62345749Smckusick */ 62465682Shibler kva = vm_pager_map_pages(mlist, npages, !(flags & B_ASYNC)); 62565682Shibler if (kva == NULL) { 62665682Shibler #ifdef DEBUG 62765682Shibler if (swpagerdebug & SDB_FAIL) 62865682Shibler printf("%s: no KVA space to map pages\n", 62965682Shibler "swap_pager_io"); 63065682Shibler #endif 63165682Shibler return(VM_PAGER_AGAIN); 63265682Shibler } 63345749Smckusick 63445749Smckusick /* 63565682Shibler * Get a swap buffer header and initialize it. 63645749Smckusick */ 63745749Smckusick s = splbio(); 63856393Smckusick while (bswlist.b_actf == NULL) { 63945749Smckusick #ifdef DEBUG 64045749Smckusick if (swpagerdebug & SDB_ANOM) 64149289Shibler printf("swap_pager_io: wait on swbuf for %x (%d)\n", 64245749Smckusick m, flags); 64345749Smckusick #endif 64445749Smckusick bswlist.b_flags |= B_WANTED; 64565682Shibler tsleep((caddr_t)&bswlist, PSWP+1, "swpgiobuf", 0); 64645749Smckusick } 64756393Smckusick bp = bswlist.b_actf; 64856393Smckusick bswlist.b_actf = bp->b_actf; 64945749Smckusick splx(s); 65045749Smckusick bp->b_flags = B_BUSY | (flags & B_READ); 65148386Skarels bp->b_proc = &proc0; /* XXX (but without B_PHYS set this is ok) */ 65264546Sbostic bp->b_data = (caddr_t)kva; 65345749Smckusick bp->b_blkno = swb->swb_block + btodb(off); 65445749Smckusick VHOLD(swapdev_vp); 65545749Smckusick bp->b_vp = swapdev_vp; 65646985Smckusick if (swapdev_vp->v_type == VBLK) 65746985Smckusick bp->b_dev = swapdev_vp->v_rdev; 65865682Shibler bp->b_bcount = npages * PAGE_SIZE; 65965682Shibler 66065682Shibler /* 66165682Shibler * For writes we set up additional buffer fields, record a pageout 66265682Shibler * in progress and mark that these swap blocks are now allocated. 66365682Shibler */ 66453213Smckusick if ((bp->b_flags & B_READ) == 0) { 66553213Smckusick bp->b_dirtyoff = 0; 66665682Shibler bp->b_dirtyend = npages * PAGE_SIZE; 66745749Smckusick swapdev_vp->v_numoutput++; 66865682Shibler s = splbio(); 66965682Shibler swp->sw_poip++; 67065682Shibler splx(s); 67165682Shibler mask = (~(~0 << npages)) << atop(off); 67265682Shibler #ifdef DEBUG 67365682Shibler swap_pager_poip++; 67465682Shibler if (swpagerdebug & SDB_WRITE) 67565682Shibler printf("swpg_io: write: bp=%x swp=%x poip=%d\n", 67665682Shibler bp, swp, swp->sw_poip); 67765682Shibler if ((swpagerdebug & SDB_ALLOCBLK) && 67865682Shibler (swb->swb_mask & mask) != mask) 67965682Shibler printf("swpg_io: %x write %d pages at %x+%x\n", 68065682Shibler swp->sw_blocks, npages, swb->swb_block, 68165682Shibler atop(off)); 68265682Shibler if (swpagerdebug & SDB_CLUSTER) 68365682Shibler printf("swpg_io: off=%x, npg=%x, mask=%x, bmask=%x\n", 68465682Shibler off, npages, mask, swb->swb_mask); 68565682Shibler #endif 68665682Shibler swb->swb_mask |= mask; 68753213Smckusick } 68845749Smckusick /* 68965682Shibler * If this is an async write we set up still more buffer fields 69045749Smckusick * and place a "cleaning" entry on the inuse queue. 69145749Smckusick */ 69245749Smckusick if ((flags & (B_READ|B_ASYNC)) == B_ASYNC) { 69345749Smckusick #ifdef DEBUG 69465231Smckusick if (swap_pager_free.tqh_first == NULL) 69545749Smckusick panic("swpg_io: lost spc"); 69645749Smckusick #endif 69765231Smckusick spc = swap_pager_free.tqh_first; 69865231Smckusick TAILQ_REMOVE(&swap_pager_free, spc, spc_list); 69945749Smckusick #ifdef DEBUG 70045749Smckusick if (spc->spc_flags != SPC_FREE) 70145749Smckusick panic("swpg_io: bad free spc"); 70245749Smckusick #endif 70345749Smckusick spc->spc_flags = SPC_BUSY; 70445749Smckusick spc->spc_bp = bp; 70545749Smckusick spc->spc_swp = swp; 70645749Smckusick spc->spc_kva = kva; 70765682Shibler /* 708*65723Shibler * Record the first page. This allows swap_pager_clean 70965682Shibler * to efficiently handle the common case of a single page. 71065682Shibler * For clusters, it allows us to locate the object easily 71165682Shibler * and we then reconstruct the rest of the mlist from spc_kva. 71265682Shibler */ 71345749Smckusick spc->spc_m = m; 71465682Shibler spc->spc_npages = npages; 71545749Smckusick bp->b_flags |= B_CALL; 71645749Smckusick bp->b_iodone = swap_pager_iodone; 71745749Smckusick s = splbio(); 71865231Smckusick TAILQ_INSERT_TAIL(&swap_pager_inuse, spc, spc_list); 71945749Smckusick splx(s); 72045749Smckusick } 72165682Shibler 72265682Shibler /* 72365682Shibler * Finally, start the IO operation. 72465682Shibler * If it is async we are all done, otherwise we must wait for 72565682Shibler * completion and cleanup afterwards. 72665682Shibler */ 72745749Smckusick #ifdef DEBUG 72845749Smckusick if (swpagerdebug & SDB_IO) 72945749Smckusick printf("swpg_io: IO start: bp %x, db %x, va %x, pa %x\n", 73045749Smckusick bp, swb->swb_block+btodb(off), kva, VM_PAGE_TO_PHYS(m)); 73145749Smckusick #endif 73245749Smckusick VOP_STRATEGY(bp); 73345749Smckusick if ((flags & (B_READ|B_ASYNC)) == B_ASYNC) { 73445749Smckusick #ifdef DEBUG 73545749Smckusick if (swpagerdebug & SDB_IO) 73645749Smckusick printf("swpg_io: IO started: bp %x\n", bp); 73745749Smckusick #endif 73845749Smckusick return(VM_PAGER_PEND); 73945749Smckusick } 74045749Smckusick s = splbio(); 74145749Smckusick #ifdef DEBUG 74245749Smckusick if (flags & B_READ) 74345749Smckusick swap_pager_piip++; 74445749Smckusick else 74545749Smckusick swap_pager_poip++; 74645749Smckusick #endif 74765682Shibler while ((bp->b_flags & B_DONE) == 0) 74865682Shibler (void) tsleep(bp, PVM, "swpgio", 0); 74965682Shibler if ((flags & B_READ) == 0) 75065682Shibler --swp->sw_poip; 75145749Smckusick #ifdef DEBUG 75245749Smckusick if (flags & B_READ) 75345749Smckusick --swap_pager_piip; 75445749Smckusick else 75545749Smckusick --swap_pager_poip; 75645749Smckusick #endif 75756320Shibler rv = (bp->b_flags & B_ERROR) ? VM_PAGER_ERROR : VM_PAGER_OK; 75845749Smckusick bp->b_flags &= ~(B_BUSY|B_WANTED|B_PHYS|B_PAGET|B_UAREA|B_DIRTY); 75956393Smckusick bp->b_actf = bswlist.b_actf; 76056393Smckusick bswlist.b_actf = bp; 76145749Smckusick if (bp->b_vp) 76245749Smckusick brelvp(bp); 76345749Smckusick if (bswlist.b_flags & B_WANTED) { 76445749Smckusick bswlist.b_flags &= ~B_WANTED; 76565682Shibler wakeup(&bswlist); 76645749Smckusick } 76745749Smckusick if ((flags & B_READ) == 0 && rv == VM_PAGER_OK) { 76856382Smckusick m->flags |= PG_CLEAN; 76945749Smckusick pmap_clear_modify(VM_PAGE_TO_PHYS(m)); 77045749Smckusick } 77145749Smckusick splx(s); 77245749Smckusick #ifdef DEBUG 77345749Smckusick if (swpagerdebug & SDB_IO) 77445749Smckusick printf("swpg_io: IO done: bp %x, rv %d\n", bp, rv); 77556320Shibler if ((swpagerdebug & SDB_FAIL) && rv == VM_PAGER_ERROR) 77645749Smckusick printf("swpg_io: IO error\n"); 77745749Smckusick #endif 77865682Shibler vm_pager_unmap_pages(kva, npages); 77945749Smckusick return(rv); 78045749Smckusick } 78145749Smckusick 78265682Shibler static void 78365682Shibler swap_pager_clean(rw) 78445749Smckusick int rw; 78545749Smckusick { 78665682Shibler register swp_clean_t spc; 78765682Shibler register int s, i; 78865682Shibler vm_object_t object; 78965682Shibler vm_page_t m; 79045749Smckusick 79145749Smckusick #ifdef DEBUG 79245749Smckusick /* save panic time state */ 79345749Smckusick if ((swpagerdebug & SDB_ANOMPANIC) && panicstr) 79465682Shibler return; 79545749Smckusick if (swpagerdebug & SDB_FOLLOW) 79665682Shibler printf("swpg_clean(%x)\n", rw); 79745749Smckusick #endif 79865682Shibler 79945749Smckusick for (;;) { 80045749Smckusick /* 80145749Smckusick * Look up and removal from inuse list must be done 80245749Smckusick * at splbio() to avoid conflicts with swap_pager_iodone. 80345749Smckusick */ 80445749Smckusick s = splbio(); 80565231Smckusick for (spc = swap_pager_inuse.tqh_first; 80665231Smckusick spc != NULL; 80765231Smckusick spc = spc->spc_list.tqe_next) { 80865682Shibler /* 80965682Shibler * If the operation is done, remove it from the 81065682Shibler * list and process it. 81165682Shibler * 81265682Shibler * XXX if we can't get the object lock we also 81365682Shibler * leave it on the list and try again later. 81465682Shibler * Is there something better we could do? 81565682Shibler */ 81645749Smckusick if ((spc->spc_flags & SPC_DONE) && 81765682Shibler vm_object_lock_try(spc->spc_m->object)) { 81865231Smckusick TAILQ_REMOVE(&swap_pager_inuse, spc, spc_list); 81945749Smckusick break; 82045749Smckusick } 82145749Smckusick } 82245749Smckusick 82345749Smckusick /* 82445749Smckusick * No operations done, thats all we can do for now. 82545749Smckusick */ 82665682Shibler if (spc == NULL) { 82765682Shibler splx(s); 82845749Smckusick break; 82965682Shibler } 83045749Smckusick splx(s); 83145749Smckusick 83245749Smckusick /* 83365682Shibler * Found a completed operation so finish it off. 83465682Shibler * Note: no longer at splbio since entry is off the list. 83545749Smckusick */ 83665682Shibler m = spc->spc_m; 83765682Shibler object = m->object; 83865682Shibler 83965682Shibler /* 84065682Shibler * Process each page in the cluster. 84165682Shibler * The first page is explicitly kept in the cleaning 84265682Shibler * entry, others must be reconstructed from the KVA. 84365682Shibler */ 84465682Shibler for (i = 0; i < spc->spc_npages; i++) { 84565682Shibler if (i) 84665682Shibler m = vm_pager_atop(spc->spc_kva + ptoa(i)); 84765682Shibler /* 84865682Shibler * If no error mark as clean and inform the pmap 84965682Shibler * system. If there was an error, mark as dirty 85065682Shibler * so we will try again. 85165682Shibler * 85265682Shibler * XXX could get stuck doing this, should give up 85365682Shibler * after awhile. 85465682Shibler */ 85565682Shibler if (spc->spc_flags & SPC_ERROR) { 85665682Shibler printf("%s: clean of page %x failed\n", 85765682Shibler "swap_pager_clean", 85865682Shibler VM_PAGE_TO_PHYS(m)); 85965682Shibler m->flags |= PG_LAUNDRY; 86065682Shibler } else { 86165682Shibler m->flags |= PG_CLEAN; 86265682Shibler pmap_clear_modify(VM_PAGE_TO_PHYS(m)); 86365682Shibler } 86465682Shibler m->flags &= ~PG_BUSY; 86565682Shibler PAGE_WAKEUP(m); 86645749Smckusick } 86765682Shibler 86865682Shibler /* 86965682Shibler * Done with the object, decrement the paging count 87065682Shibler * and unlock it. 87165682Shibler */ 87265682Shibler if (--object->paging_in_progress == 0) 87365682Shibler wakeup(object); 87465682Shibler vm_object_unlock(object); 87565682Shibler 87665682Shibler /* 87765682Shibler * Free up KVM used and put the entry back on the list. 87865682Shibler */ 87965682Shibler vm_pager_unmap_pages(spc->spc_kva, spc->spc_npages); 88045749Smckusick spc->spc_flags = SPC_FREE; 88165231Smckusick TAILQ_INSERT_TAIL(&swap_pager_free, spc, spc_list); 88245749Smckusick #ifdef DEBUG 88345749Smckusick if (swpagerdebug & SDB_WRITE) 88445749Smckusick printf("swpg_clean: free spc %x\n", spc); 88545749Smckusick #endif 88645749Smckusick } 88765682Shibler } 88845749Smckusick 88945749Smckusick #ifdef DEBUG 89065682Shibler static void 89165682Shibler swap_pager_clean_check(mlist, npages, rw) 89265682Shibler vm_page_t *mlist; 89365682Shibler int npages; 89465682Shibler int rw; 89565682Shibler { 89645749Smckusick register swp_clean_t spc; 89765682Shibler boolean_t bad; 89865682Shibler int i, j, s; 89965682Shibler vm_page_t m; 90045749Smckusick 90165682Shibler if (panicstr) 90265682Shibler return; 90345749Smckusick 90465682Shibler bad = FALSE; 90565682Shibler s = splbio(); 90665682Shibler for (spc = swap_pager_inuse.tqh_first; 90765682Shibler spc != NULL; 90865682Shibler spc = spc->spc_list.tqe_next) { 90965682Shibler for (j = 0; j < spc->spc_npages; j++) { 91065682Shibler m = vm_pager_atop(spc->spc_kva + ptoa(j)); 91165682Shibler for (i = 0; i < npages; i++) 91265682Shibler if (m == mlist[i]) { 91365682Shibler if (swpagerdebug & SDB_ANOM) 91465682Shibler printf( 91565682Shibler "swpg_clean_check: %s: page %x on list, flags %x\n", 91665682Shibler rw == B_WRITE ? "write" : "read", mlist[i], spc->spc_flags); 91765682Shibler bad = TRUE; 91865682Shibler } 91965682Shibler } 92045749Smckusick } 92165682Shibler splx(s); 92265682Shibler if (bad) 92365682Shibler panic("swpg_clean_check"); 92465682Shibler } 92549289Shibler #endif 92645749Smckusick 92753341Sbostic static void 92845749Smckusick swap_pager_iodone(bp) 92945749Smckusick register struct buf *bp; 93045749Smckusick { 93145749Smckusick register swp_clean_t spc; 93245749Smckusick daddr_t blk; 93345749Smckusick int s; 93445749Smckusick 93545749Smckusick #ifdef DEBUG 93645749Smckusick /* save panic time state */ 93745749Smckusick if ((swpagerdebug & SDB_ANOMPANIC) && panicstr) 93845749Smckusick return; 93945749Smckusick if (swpagerdebug & SDB_FOLLOW) 94045749Smckusick printf("swpg_iodone(%x)\n", bp); 94145749Smckusick #endif 94245749Smckusick s = splbio(); 94365231Smckusick for (spc = swap_pager_inuse.tqh_first; 94465231Smckusick spc != NULL; 94565231Smckusick spc = spc->spc_list.tqe_next) 94645749Smckusick if (spc->spc_bp == bp) 94745749Smckusick break; 94845749Smckusick #ifdef DEBUG 94965231Smckusick if (spc == NULL) 95049289Shibler panic("swap_pager_iodone: bp not found"); 95145749Smckusick #endif 95245749Smckusick 95345749Smckusick spc->spc_flags &= ~SPC_BUSY; 95445749Smckusick spc->spc_flags |= SPC_DONE; 95545749Smckusick if (bp->b_flags & B_ERROR) 95645749Smckusick spc->spc_flags |= SPC_ERROR; 95745749Smckusick spc->spc_bp = NULL; 95845749Smckusick blk = bp->b_blkno; 95945749Smckusick 96045749Smckusick #ifdef DEBUG 96145749Smckusick --swap_pager_poip; 96245749Smckusick if (swpagerdebug & SDB_WRITE) 96345749Smckusick printf("swpg_iodone: bp=%x swp=%x flags=%x spc=%x poip=%x\n", 96445749Smckusick bp, spc->spc_swp, spc->spc_swp->sw_flags, 96545749Smckusick spc, spc->spc_swp->sw_poip); 96645749Smckusick #endif 96745749Smckusick 96845749Smckusick spc->spc_swp->sw_poip--; 96945749Smckusick if (spc->spc_swp->sw_flags & SW_WANTED) { 97045749Smckusick spc->spc_swp->sw_flags &= ~SW_WANTED; 97165682Shibler wakeup(spc->spc_swp); 97245749Smckusick } 97345749Smckusick 97445749Smckusick bp->b_flags &= ~(B_BUSY|B_WANTED|B_PHYS|B_PAGET|B_UAREA|B_DIRTY); 97556393Smckusick bp->b_actf = bswlist.b_actf; 97656393Smckusick bswlist.b_actf = bp; 97745749Smckusick if (bp->b_vp) 97845749Smckusick brelvp(bp); 97945749Smckusick if (bswlist.b_flags & B_WANTED) { 98045749Smckusick bswlist.b_flags &= ~B_WANTED; 98165682Shibler wakeup(&bswlist); 98245749Smckusick } 98365682Shibler wakeup(&vm_pages_needed); 98445749Smckusick splx(s); 98545749Smckusick } 986