1 /* $NetBSD: uvm_pager.c,v 1.54 2001/11/10 07:37:00 lukem Exp $ */ 2 3 /* 4 * 5 * Copyright (c) 1997 Charles D. Cranor and Washington University. 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. All advertising materials mentioning features or use of this software 17 * must display the following acknowledgement: 18 * This product includes software developed by Charles D. Cranor and 19 * Washington University. 20 * 4. The name of the author may not be used to endorse or promote products 21 * derived from this software without specific prior written permission. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 24 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 25 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 26 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 27 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 28 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 29 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 30 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 32 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 33 * 34 * from: Id: uvm_pager.c,v 1.1.2.23 1998/02/02 20:38:06 chuck Exp 35 */ 36 37 /* 38 * uvm_pager.c: generic functions used to assist the pagers. 39 */ 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: uvm_pager.c,v 1.54 2001/11/10 07:37:00 lukem Exp $"); 43 44 #include "opt_uvmhist.h" 45 46 #include <sys/param.h> 47 #include <sys/systm.h> 48 #include <sys/proc.h> 49 #include <sys/malloc.h> 50 #include <sys/pool.h> 51 #include <sys/vnode.h> 52 53 #define UVM_PAGER 54 #include <uvm/uvm.h> 55 56 struct pool *uvm_aiobuf_pool; 57 58 /* 59 * list of uvm pagers in the system 60 */ 61 62 extern struct uvm_pagerops uvm_deviceops; 63 extern struct uvm_pagerops uvm_vnodeops; 64 extern struct uvm_pagerops ubc_pager; 65 66 struct uvm_pagerops *uvmpagerops[] = { 67 &aobj_pager, 68 &uvm_deviceops, 69 &uvm_vnodeops, 70 &ubc_pager, 71 }; 72 73 /* 74 * the pager map: provides KVA for I/O 75 */ 76 77 struct vm_map *pager_map; /* XXX */ 78 struct simplelock pager_map_wanted_lock; 79 boolean_t pager_map_wanted; /* locked by pager map */ 80 static vaddr_t emergva; 81 static boolean_t emerginuse; 82 83 /* 84 * uvm_pager_init: init pagers (at boot time) 85 */ 86 87 void 88 uvm_pager_init() 89 { 90 int lcv; 91 vaddr_t sva, eva; 92 93 /* 94 * init pager map 95 */ 96 97 sva = 0; 98 pager_map = uvm_km_suballoc(kernel_map, &sva, &eva, PAGER_MAP_SIZE, 0, 99 FALSE, NULL); 100 simple_lock_init(&pager_map_wanted_lock); 101 pager_map_wanted = FALSE; 102 emergva = uvm_km_valloc(kernel_map, MAXBSIZE); 103 emerginuse = FALSE; 104 105 /* 106 * init ASYNC I/O queue 107 */ 108 109 TAILQ_INIT(&uvm.aio_done); 110 111 /* 112 * call pager init functions 113 */ 114 for (lcv = 0 ; lcv < sizeof(uvmpagerops)/sizeof(struct uvm_pagerops *); 115 lcv++) { 116 if (uvmpagerops[lcv]->pgo_init) 117 uvmpagerops[lcv]->pgo_init(); 118 } 119 } 120 121 /* 122 * uvm_pagermapin: map pages into KVA (pager_map) for I/O that needs mappings 123 * 124 * we basically just map in a blank map entry to reserve the space in the 125 * map and then use pmap_enter() to put the mappings in by hand. 126 */ 127 128 vaddr_t 129 uvm_pagermapin(pps, npages, flags) 130 struct vm_page **pps; 131 int npages; 132 int flags; 133 { 134 vsize_t size; 135 vaddr_t kva; 136 vaddr_t cva; 137 struct vm_page *pp; 138 vm_prot_t prot; 139 UVMHIST_FUNC("uvm_pagermapin"); UVMHIST_CALLED(maphist); 140 141 UVMHIST_LOG(maphist,"(pps=0x%x, npages=%d)", pps, npages,0,0); 142 143 /* 144 * compute protection. outgoing I/O only needs read 145 * access to the page, whereas incoming needs read/write. 146 */ 147 148 prot = VM_PROT_READ; 149 if (flags & UVMPAGER_MAPIN_READ) 150 prot |= VM_PROT_WRITE; 151 152 ReStart: 153 size = npages << PAGE_SHIFT; 154 kva = 0; /* let system choose VA */ 155 156 if (uvm_map(pager_map, &kva, size, NULL, 157 UVM_UNKNOWN_OFFSET, 0, UVM_FLAG_NOMERGE) != 0) { 158 if (curproc == uvm.pagedaemon_proc) { 159 simple_lock(&pager_map_wanted_lock); 160 if (emerginuse) { 161 UVM_UNLOCK_AND_WAIT(&emergva, 162 &pager_map_wanted_lock, FALSE, 163 "emergva", 0); 164 goto ReStart; 165 } 166 emerginuse = TRUE; 167 simple_unlock(&pager_map_wanted_lock); 168 kva = emergva; 169 KASSERT(npages <= MAXBSIZE >> PAGE_SHIFT); 170 goto enter; 171 } 172 if ((flags & UVMPAGER_MAPIN_WAITOK) == 0) { 173 UVMHIST_LOG(maphist,"<- NOWAIT failed", 0,0,0,0); 174 return(0); 175 } 176 simple_lock(&pager_map_wanted_lock); 177 pager_map_wanted = TRUE; 178 UVMHIST_LOG(maphist, " SLEEPING on pager_map",0,0,0,0); 179 UVM_UNLOCK_AND_WAIT(pager_map, &pager_map_wanted_lock, FALSE, 180 "pager_map", 0); 181 goto ReStart; 182 } 183 184 enter: 185 /* got it */ 186 for (cva = kva ; size != 0 ; size -= PAGE_SIZE, cva += PAGE_SIZE) { 187 pp = *pps++; 188 KASSERT(pp); 189 KASSERT(pp->flags & PG_BUSY); 190 pmap_kenter_pa(cva, VM_PAGE_TO_PHYS(pp), prot); 191 } 192 pmap_update(vm_map_pmap(pager_map)); 193 194 UVMHIST_LOG(maphist, "<- done (KVA=0x%x)", kva,0,0,0); 195 return(kva); 196 } 197 198 /* 199 * uvm_pagermapout: remove pager_map mapping 200 * 201 * we remove our mappings by hand and then remove the mapping (waking 202 * up anyone wanting space). 203 */ 204 205 void 206 uvm_pagermapout(kva, npages) 207 vaddr_t kva; 208 int npages; 209 { 210 vsize_t size = npages << PAGE_SHIFT; 211 struct vm_map_entry *entries; 212 UVMHIST_FUNC("uvm_pagermapout"); UVMHIST_CALLED(maphist); 213 214 UVMHIST_LOG(maphist, " (kva=0x%x, npages=%d)", kva, npages,0,0); 215 216 /* 217 * duplicate uvm_unmap, but add in pager_map_wanted handling. 218 */ 219 220 pmap_kremove(kva, npages << PAGE_SHIFT); 221 if (kva == emergva) { 222 simple_lock(&pager_map_wanted_lock); 223 emerginuse = FALSE; 224 wakeup(&emergva); 225 simple_unlock(&pager_map_wanted_lock); 226 return; 227 } 228 229 vm_map_lock(pager_map); 230 uvm_unmap_remove(pager_map, kva, kva + size, &entries); 231 simple_lock(&pager_map_wanted_lock); 232 if (pager_map_wanted) { 233 pager_map_wanted = FALSE; 234 wakeup(pager_map); 235 } 236 simple_unlock(&pager_map_wanted_lock); 237 vm_map_unlock(pager_map); 238 if (entries) 239 uvm_unmap_detach(entries, 0); 240 pmap_update(pmap_kernel()); 241 UVMHIST_LOG(maphist,"<- done",0,0,0,0); 242 } 243 244 /* 245 * interrupt-context iodone handler for nested i/o bufs. 246 * 247 * => must be at splbio(). 248 */ 249 250 void 251 uvm_aio_biodone1(bp) 252 struct buf *bp; 253 { 254 struct buf *mbp = bp->b_private; 255 256 KASSERT(mbp != bp); 257 if (bp->b_flags & B_ERROR) { 258 mbp->b_flags |= B_ERROR; 259 mbp->b_error = bp->b_error; 260 } 261 mbp->b_resid -= bp->b_bcount; 262 pool_put(&bufpool, bp); 263 if (mbp->b_resid == 0) { 264 biodone(mbp); 265 } 266 } 267 268 /* 269 * interrupt-context iodone handler for single-buf i/os 270 * or the top-level buf of a nested-buf i/o. 271 * 272 * => must be at splbio(). 273 */ 274 275 void 276 uvm_aio_biodone(bp) 277 struct buf *bp; 278 { 279 /* reset b_iodone for when this is a single-buf i/o. */ 280 bp->b_iodone = uvm_aio_aiodone; 281 282 simple_lock(&uvm.aiodoned_lock); /* locks uvm.aio_done */ 283 TAILQ_INSERT_TAIL(&uvm.aio_done, bp, b_freelist); 284 wakeup(&uvm.aiodoned); 285 simple_unlock(&uvm.aiodoned_lock); 286 } 287 288 /* 289 * uvm_aio_aiodone: do iodone processing for async i/os. 290 * this should be called in thread context, not interrupt context. 291 */ 292 293 void 294 uvm_aio_aiodone(bp) 295 struct buf *bp; 296 { 297 int npages = bp->b_bufsize >> PAGE_SHIFT; 298 struct vm_page *pg, *pgs[npages]; 299 struct uvm_object *uobj; 300 struct simplelock *slock; 301 int s, i, error, swslot; 302 boolean_t write, swap, pageout; 303 UVMHIST_FUNC("uvm_aio_aiodone"); UVMHIST_CALLED(ubchist); 304 UVMHIST_LOG(ubchist, "bp %p", bp, 0,0,0); 305 306 error = (bp->b_flags & B_ERROR) ? (bp->b_error ? bp->b_error : EIO) : 0; 307 write = (bp->b_flags & B_READ) == 0; 308 /* XXXUBC B_NOCACHE is for swap pager, should be done differently */ 309 if (write && !(bp->b_flags & B_NOCACHE) && bioops.io_pageiodone) { 310 (*bioops.io_pageiodone)(bp); 311 } 312 313 uobj = NULL; 314 for (i = 0; i < npages; i++) { 315 pgs[i] = uvm_pageratop((vaddr_t)bp->b_data + (i << PAGE_SHIFT)); 316 UVMHIST_LOG(ubchist, "pgs[%d] = %p", i, pgs[i],0,0); 317 } 318 uvm_pagermapout((vaddr_t)bp->b_data, npages); 319 320 swslot = 0; 321 slock = NULL; 322 swap = (pgs[0]->pqflags & PQ_SWAPBACKED) != 0; 323 pageout = (pgs[0]->flags & PG_PAGEOUT) != 0; 324 if (!swap) { 325 uobj = pgs[0]->uobject; 326 slock = &uobj->vmobjlock; 327 simple_lock(slock); 328 uvm_lock_pageq(); 329 } else if (error) { 330 pg = pgs[0]; 331 if (pg->pqflags & PQ_ANON) { 332 swslot = pg->uanon->an_swslot; 333 } else { 334 swslot = uao_find_swslot(pg->uobject, pg->offset); 335 } 336 KASSERT(swslot); 337 } 338 for (i = 0; i < npages; i++) { 339 pg = pgs[i]; 340 KASSERT(swap || pg->uobject == uobj); 341 KASSERT(pageout ^ ((pg->flags & PG_PAGEOUT) == 0)); 342 UVMHIST_LOG(ubchist, "pg %p", pg, 0,0,0); 343 344 /* 345 * for swap i/os, lock each page's object (or anon) 346 * individually since each page may need a different lock. 347 */ 348 349 if (swap) { 350 if (pg->pqflags & PQ_ANON) { 351 slock = &pg->uanon->an_lock; 352 } else { 353 slock = &pg->uobject->vmobjlock; 354 } 355 simple_lock(slock); 356 uvm_lock_pageq(); 357 } 358 359 /* 360 * process errors. for reads, just mark the page to be freed. 361 * for writes, if the error was ENOMEM, we assume this was 362 * a transient failure so we mark the page dirty so that 363 * we'll try to write it again later. for all other write 364 * errors, we assume the error is permanent, thus the data 365 * in the page is lost. bummer. 366 */ 367 368 if (error) { 369 if (!write) { 370 pg->flags |= PG_RELEASED; 371 continue; 372 } else if (error == ENOMEM) { 373 if (pg->flags & PG_PAGEOUT) { 374 pg->flags &= ~PG_PAGEOUT; 375 uvmexp.paging--; 376 } 377 pg->flags &= ~PG_CLEAN; 378 uvm_pageactivate(pg); 379 } 380 } 381 382 /* 383 * if the page is PG_FAKE, this must have been a read to 384 * initialize the page. clear PG_FAKE and activate the page. 385 * we must also clear the pmap "modified" flag since it may 386 * still be set from the page's previous identity. 387 */ 388 389 if (pg->flags & PG_FAKE) { 390 KASSERT(!write); 391 pg->flags &= ~PG_FAKE; 392 uvm_pageactivate(pg); 393 pmap_clear_modify(pg); 394 } 395 396 /* 397 * do accounting for pagedaemon i/o and arrange to free 398 * the pages instead of just unbusying them. 399 */ 400 401 if (pg->flags & PG_PAGEOUT) { 402 pg->flags &= ~PG_PAGEOUT; 403 uvmexp.paging--; 404 pg->flags |= PG_RELEASED; 405 } 406 407 /* 408 * for swap pages, unlock everything for this page now. 409 */ 410 411 if (swap) { 412 uvm_page_unbusy(&pg, 1); 413 uvm_unlock_pageq(); 414 simple_unlock(slock); 415 } 416 } 417 if (!swap) { 418 uvm_page_unbusy(pgs, npages); 419 uvm_unlock_pageq(); 420 simple_unlock(slock); 421 } else { 422 KASSERT(write); 423 KASSERT(pageout); 424 425 /* these pages are now only in swap. */ 426 simple_lock(&uvm.swap_data_lock); 427 KASSERT(uvmexp.swpgonly + npages <= uvmexp.swpginuse); 428 uvmexp.swpgonly += npages; 429 simple_unlock(&uvm.swap_data_lock); 430 if (error) { 431 uvm_swap_markbad(swslot, npages); 432 } 433 } 434 s = splbio(); 435 if (write && (bp->b_flags & B_AGE) != 0) { 436 vwakeup(bp); 437 } 438 pool_put(&bufpool, bp); 439 splx(s); 440 } 441