1 /* $NetBSD: uvm_swap.c,v 1.94 2005/06/27 02:29:32 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 1995, 1996, 1997 Matthew R. Green 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of the author may not be used to endorse or promote products 16 * derived from this software without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 * from: NetBSD: vm_swap.c,v 1.52 1997/12/02 13:47:37 pk Exp 31 * from: Id: uvm_swap.c,v 1.1.2.42 1998/02/02 20:38:06 chuck Exp 32 */ 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.94 2005/06/27 02:29:32 thorpej Exp $"); 36 37 #include "fs_nfs.h" 38 #include "opt_uvmhist.h" 39 #include "opt_compat_netbsd.h" 40 #include "opt_ddb.h" 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/buf.h> 45 #include <sys/bufq.h> 46 #include <sys/conf.h> 47 #include <sys/proc.h> 48 #include <sys/namei.h> 49 #include <sys/disklabel.h> 50 #include <sys/errno.h> 51 #include <sys/kernel.h> 52 #include <sys/malloc.h> 53 #include <sys/vnode.h> 54 #include <sys/file.h> 55 #include <sys/extent.h> 56 #include <sys/blist.h> 57 #include <sys/mount.h> 58 #include <sys/pool.h> 59 #include <sys/sa.h> 60 #include <sys/syscallargs.h> 61 #include <sys/swap.h> 62 63 #include <uvm/uvm.h> 64 65 #include <miscfs/specfs/specdev.h> 66 67 /* 68 * uvm_swap.c: manage configuration and i/o to swap space. 69 */ 70 71 /* 72 * swap space is managed in the following way: 73 * 74 * each swap partition or file is described by a "swapdev" structure. 75 * each "swapdev" structure contains a "swapent" structure which contains 76 * information that is passed up to the user (via system calls). 77 * 78 * each swap partition is assigned a "priority" (int) which controls 79 * swap parition usage. 80 * 81 * the system maintains a global data structure describing all swap 82 * partitions/files. there is a sorted LIST of "swappri" structures 83 * which describe "swapdev"'s at that priority. this LIST is headed 84 * by the "swap_priority" global var. each "swappri" contains a 85 * CIRCLEQ of "swapdev" structures at that priority. 86 * 87 * locking: 88 * - swap_syscall_lock (sleep lock): this lock serializes the swapctl 89 * system call and prevents the swap priority list from changing 90 * while we are in the middle of a system call (e.g. SWAP_STATS). 91 * - uvm.swap_data_lock (simple_lock): this lock protects all swap data 92 * structures including the priority list, the swapdev structures, 93 * and the swapmap extent. 94 * 95 * each swap device has the following info: 96 * - swap device in use (could be disabled, preventing future use) 97 * - swap enabled (allows new allocations on swap) 98 * - map info in /dev/drum 99 * - vnode pointer 100 * for swap files only: 101 * - block size 102 * - max byte count in buffer 103 * - buffer 104 * 105 * userland controls and configures swap with the swapctl(2) system call. 106 * the sys_swapctl performs the following operations: 107 * [1] SWAP_NSWAP: returns the number of swap devices currently configured 108 * [2] SWAP_STATS: given a pointer to an array of swapent structures 109 * (passed in via "arg") of a size passed in via "misc" ... we load 110 * the current swap config into the array. The actual work is done 111 * in the uvm_swap_stats(9) function. 112 * [3] SWAP_ON: given a pathname in arg (could be device or file) and a 113 * priority in "misc", start swapping on it. 114 * [4] SWAP_OFF: as SWAP_ON, but stops swapping to a device 115 * [5] SWAP_CTL: changes the priority of a swap device (new priority in 116 * "misc") 117 */ 118 119 /* 120 * swapdev: describes a single swap partition/file 121 * 122 * note the following should be true: 123 * swd_inuse <= swd_nblks [number of blocks in use is <= total blocks] 124 * swd_nblks <= swd_mapsize [because mapsize includes miniroot+disklabel] 125 */ 126 struct swapdev { 127 struct oswapent swd_ose; 128 #define swd_dev swd_ose.ose_dev /* device id */ 129 #define swd_flags swd_ose.ose_flags /* flags:inuse/enable/fake */ 130 #define swd_priority swd_ose.ose_priority /* our priority */ 131 /* also: swd_ose.ose_nblks, swd_ose.ose_inuse */ 132 char *swd_path; /* saved pathname of device */ 133 int swd_pathlen; /* length of pathname */ 134 int swd_npages; /* #pages we can use */ 135 int swd_npginuse; /* #pages in use */ 136 int swd_npgbad; /* #pages bad */ 137 int swd_drumoffset; /* page0 offset in drum */ 138 int swd_drumsize; /* #pages in drum */ 139 blist_t swd_blist; /* blist for this swapdev */ 140 struct vnode *swd_vp; /* backing vnode */ 141 CIRCLEQ_ENTRY(swapdev) swd_next; /* priority circleq */ 142 143 int swd_bsize; /* blocksize (bytes) */ 144 int swd_maxactive; /* max active i/o reqs */ 145 struct bufq_state swd_tab; /* buffer list */ 146 int swd_active; /* number of active buffers */ 147 }; 148 149 /* 150 * swap device priority entry; the list is kept sorted on `spi_priority'. 151 */ 152 struct swappri { 153 int spi_priority; /* priority */ 154 CIRCLEQ_HEAD(spi_swapdev, swapdev) spi_swapdev; 155 /* circleq of swapdevs at this priority */ 156 LIST_ENTRY(swappri) spi_swappri; /* global list of pri's */ 157 }; 158 159 /* 160 * The following two structures are used to keep track of data transfers 161 * on swap devices associated with regular files. 162 * NOTE: this code is more or less a copy of vnd.c; we use the same 163 * structure names here to ease porting.. 164 */ 165 struct vndxfer { 166 struct buf *vx_bp; /* Pointer to parent buffer */ 167 struct swapdev *vx_sdp; 168 int vx_error; 169 int vx_pending; /* # of pending aux buffers */ 170 int vx_flags; 171 #define VX_BUSY 1 172 #define VX_DEAD 2 173 }; 174 175 struct vndbuf { 176 struct buf vb_buf; 177 struct vndxfer *vb_xfer; 178 }; 179 180 181 /* 182 * We keep a of pool vndbuf's and vndxfer structures. 183 */ 184 POOL_INIT(vndxfer_pool, sizeof(struct vndxfer), 0, 0, 0, "swp vnx", NULL); 185 POOL_INIT(vndbuf_pool, sizeof(struct vndbuf), 0, 0, 0, "swp vnd", NULL); 186 187 #define getvndxfer(vnx) do { \ 188 int sp = splbio(); \ 189 vnx = pool_get(&vndxfer_pool, PR_WAITOK); \ 190 splx(sp); \ 191 } while (/*CONSTCOND*/ 0) 192 193 #define putvndxfer(vnx) { \ 194 pool_put(&vndxfer_pool, (void *)(vnx)); \ 195 } 196 197 #define getvndbuf(vbp) do { \ 198 int sp = splbio(); \ 199 vbp = pool_get(&vndbuf_pool, PR_WAITOK); \ 200 splx(sp); \ 201 } while (/*CONSTCOND*/ 0) 202 203 #define putvndbuf(vbp) { \ 204 pool_put(&vndbuf_pool, (void *)(vbp)); \ 205 } 206 207 /* 208 * local variables 209 */ 210 static struct extent *swapmap; /* controls the mapping of /dev/drum */ 211 212 MALLOC_DEFINE(M_VMSWAP, "VM swap", "VM swap structures"); 213 214 /* list of all active swap devices [by priority] */ 215 LIST_HEAD(swap_priority, swappri); 216 static struct swap_priority swap_priority; 217 218 /* locks */ 219 struct lock swap_syscall_lock; 220 221 /* 222 * prototypes 223 */ 224 static struct swapdev *swapdrum_getsdp(int); 225 226 static struct swapdev *swaplist_find(struct vnode *, int); 227 static void swaplist_insert(struct swapdev *, 228 struct swappri *, int); 229 static void swaplist_trim(void); 230 231 static int swap_on(struct proc *, struct swapdev *); 232 static int swap_off(struct proc *, struct swapdev *); 233 234 static void sw_reg_strategy(struct swapdev *, struct buf *, int); 235 static void sw_reg_iodone(struct buf *); 236 static void sw_reg_start(struct swapdev *); 237 238 static int uvm_swap_io(struct vm_page **, int, int, int); 239 240 /* 241 * uvm_swap_init: init the swap system data structures and locks 242 * 243 * => called at boot time from init_main.c after the filesystems 244 * are brought up (which happens after uvm_init()) 245 */ 246 void 247 uvm_swap_init(void) 248 { 249 UVMHIST_FUNC("uvm_swap_init"); 250 251 UVMHIST_CALLED(pdhist); 252 /* 253 * first, init the swap list, its counter, and its lock. 254 * then get a handle on the vnode for /dev/drum by using 255 * the its dev_t number ("swapdev", from MD conf.c). 256 */ 257 258 LIST_INIT(&swap_priority); 259 uvmexp.nswapdev = 0; 260 lockinit(&swap_syscall_lock, PVM, "swapsys", 0, 0); 261 simple_lock_init(&uvm.swap_data_lock); 262 263 if (bdevvp(swapdev, &swapdev_vp)) 264 panic("uvm_swap_init: can't get vnode for swap device"); 265 266 /* 267 * create swap block resource map to map /dev/drum. the range 268 * from 1 to INT_MAX allows 2 gigablocks of swap space. note 269 * that block 0 is reserved (used to indicate an allocation 270 * failure, or no allocation). 271 */ 272 swapmap = extent_create("swapmap", 1, INT_MAX, 273 M_VMSWAP, 0, 0, EX_NOWAIT); 274 if (swapmap == 0) 275 panic("uvm_swap_init: extent_create failed"); 276 277 /* 278 * done! 279 */ 280 UVMHIST_LOG(pdhist, "<- done", 0, 0, 0, 0); 281 } 282 283 /* 284 * swaplist functions: functions that operate on the list of swap 285 * devices on the system. 286 */ 287 288 /* 289 * swaplist_insert: insert swap device "sdp" into the global list 290 * 291 * => caller must hold both swap_syscall_lock and uvm.swap_data_lock 292 * => caller must provide a newly malloc'd swappri structure (we will 293 * FREE it if we don't need it... this it to prevent malloc blocking 294 * here while adding swap) 295 */ 296 static void 297 swaplist_insert(struct swapdev *sdp, struct swappri *newspp, int priority) 298 { 299 struct swappri *spp, *pspp; 300 UVMHIST_FUNC("swaplist_insert"); UVMHIST_CALLED(pdhist); 301 302 /* 303 * find entry at or after which to insert the new device. 304 */ 305 pspp = NULL; 306 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 307 if (priority <= spp->spi_priority) 308 break; 309 pspp = spp; 310 } 311 312 /* 313 * new priority? 314 */ 315 if (spp == NULL || spp->spi_priority != priority) { 316 spp = newspp; /* use newspp! */ 317 UVMHIST_LOG(pdhist, "created new swappri = %d", 318 priority, 0, 0, 0); 319 320 spp->spi_priority = priority; 321 CIRCLEQ_INIT(&spp->spi_swapdev); 322 323 if (pspp) 324 LIST_INSERT_AFTER(pspp, spp, spi_swappri); 325 else 326 LIST_INSERT_HEAD(&swap_priority, spp, spi_swappri); 327 } else { 328 /* we don't need a new priority structure, free it */ 329 FREE(newspp, M_VMSWAP); 330 } 331 332 /* 333 * priority found (or created). now insert on the priority's 334 * circleq list and bump the total number of swapdevs. 335 */ 336 sdp->swd_priority = priority; 337 CIRCLEQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next); 338 uvmexp.nswapdev++; 339 } 340 341 /* 342 * swaplist_find: find and optionally remove a swap device from the 343 * global list. 344 * 345 * => caller must hold both swap_syscall_lock and uvm.swap_data_lock 346 * => we return the swapdev we found (and removed) 347 */ 348 static struct swapdev * 349 swaplist_find(struct vnode *vp, boolean_t remove) 350 { 351 struct swapdev *sdp; 352 struct swappri *spp; 353 354 /* 355 * search the lists for the requested vp 356 */ 357 358 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 359 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 360 if (sdp->swd_vp == vp) { 361 if (remove) { 362 CIRCLEQ_REMOVE(&spp->spi_swapdev, 363 sdp, swd_next); 364 uvmexp.nswapdev--; 365 } 366 return(sdp); 367 } 368 } 369 } 370 return (NULL); 371 } 372 373 374 /* 375 * swaplist_trim: scan priority list for empty priority entries and kill 376 * them. 377 * 378 * => caller must hold both swap_syscall_lock and uvm.swap_data_lock 379 */ 380 static void 381 swaplist_trim(void) 382 { 383 struct swappri *spp, *nextspp; 384 385 for (spp = LIST_FIRST(&swap_priority); spp != NULL; spp = nextspp) { 386 nextspp = LIST_NEXT(spp, spi_swappri); 387 if (CIRCLEQ_FIRST(&spp->spi_swapdev) != 388 (void *)&spp->spi_swapdev) 389 continue; 390 LIST_REMOVE(spp, spi_swappri); 391 free(spp, M_VMSWAP); 392 } 393 } 394 395 /* 396 * swapdrum_getsdp: given a page offset in /dev/drum, convert it back 397 * to the "swapdev" that maps that section of the drum. 398 * 399 * => each swapdev takes one big contig chunk of the drum 400 * => caller must hold uvm.swap_data_lock 401 */ 402 static struct swapdev * 403 swapdrum_getsdp(int pgno) 404 { 405 struct swapdev *sdp; 406 struct swappri *spp; 407 408 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 409 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 410 if (sdp->swd_flags & SWF_FAKE) 411 continue; 412 if (pgno >= sdp->swd_drumoffset && 413 pgno < (sdp->swd_drumoffset + sdp->swd_drumsize)) { 414 return sdp; 415 } 416 } 417 } 418 return NULL; 419 } 420 421 422 /* 423 * sys_swapctl: main entry point for swapctl(2) system call 424 * [with two helper functions: swap_on and swap_off] 425 */ 426 int 427 sys_swapctl(struct lwp *l, void *v, register_t *retval) 428 { 429 struct sys_swapctl_args /* { 430 syscallarg(int) cmd; 431 syscallarg(void *) arg; 432 syscallarg(int) misc; 433 } */ *uap = (struct sys_swapctl_args *)v; 434 struct proc *p = l->l_proc; 435 struct vnode *vp; 436 struct nameidata nd; 437 struct swappri *spp; 438 struct swapdev *sdp; 439 struct swapent *sep; 440 char userpath[PATH_MAX + 1]; 441 size_t len; 442 int error, misc; 443 int priority; 444 UVMHIST_FUNC("sys_swapctl"); UVMHIST_CALLED(pdhist); 445 446 misc = SCARG(uap, misc); 447 448 /* 449 * ensure serialized syscall access by grabbing the swap_syscall_lock 450 */ 451 lockmgr(&swap_syscall_lock, LK_EXCLUSIVE, NULL); 452 453 /* 454 * we handle the non-priv NSWAP and STATS request first. 455 * 456 * SWAP_NSWAP: return number of config'd swap devices 457 * [can also be obtained with uvmexp sysctl] 458 */ 459 if (SCARG(uap, cmd) == SWAP_NSWAP) { 460 UVMHIST_LOG(pdhist, "<- done SWAP_NSWAP=%d", uvmexp.nswapdev, 461 0, 0, 0); 462 *retval = uvmexp.nswapdev; 463 error = 0; 464 goto out; 465 } 466 467 /* 468 * SWAP_STATS: get stats on current # of configured swap devs 469 * 470 * note that the swap_priority list can't change as long 471 * as we are holding the swap_syscall_lock. we don't want 472 * to grab the uvm.swap_data_lock because we may fault&sleep during 473 * copyout() and we don't want to be holding that lock then! 474 */ 475 if (SCARG(uap, cmd) == SWAP_STATS 476 #if defined(COMPAT_13) 477 || SCARG(uap, cmd) == SWAP_OSTATS 478 #endif 479 ) { 480 if ((size_t)misc > (size_t)uvmexp.nswapdev) 481 misc = uvmexp.nswapdev; 482 #if defined(COMPAT_13) 483 if (SCARG(uap, cmd) == SWAP_OSTATS) 484 len = sizeof(struct oswapent) * misc; 485 else 486 #endif 487 len = sizeof(struct swapent) * misc; 488 sep = (struct swapent *)malloc(len, M_TEMP, M_WAITOK); 489 490 uvm_swap_stats(SCARG(uap, cmd), sep, misc, retval); 491 error = copyout(sep, SCARG(uap, arg), len); 492 493 free(sep, M_TEMP); 494 UVMHIST_LOG(pdhist, "<- done SWAP_STATS", 0, 0, 0, 0); 495 goto out; 496 } 497 if (SCARG(uap, cmd) == SWAP_GETDUMPDEV) { 498 dev_t *devp = (dev_t *)SCARG(uap, arg); 499 500 error = copyout(&dumpdev, devp, sizeof(dumpdev)); 501 goto out; 502 } 503 504 /* 505 * all other requests require superuser privs. verify. 506 */ 507 if ((error = suser(p->p_ucred, &p->p_acflag))) 508 goto out; 509 510 /* 511 * at this point we expect a path name in arg. we will 512 * use namei() to gain a vnode reference (vref), and lock 513 * the vnode (VOP_LOCK). 514 * 515 * XXX: a NULL arg means use the root vnode pointer (e.g. for 516 * miniroot) 517 */ 518 if (SCARG(uap, arg) == NULL) { 519 vp = rootvp; /* miniroot */ 520 if (vget(vp, LK_EXCLUSIVE)) { 521 error = EBUSY; 522 goto out; 523 } 524 if (SCARG(uap, cmd) == SWAP_ON && 525 copystr("miniroot", userpath, sizeof userpath, &len)) 526 panic("swapctl: miniroot copy failed"); 527 } else { 528 int space; 529 char *where; 530 531 if (SCARG(uap, cmd) == SWAP_ON) { 532 if ((error = copyinstr(SCARG(uap, arg), userpath, 533 sizeof userpath, &len))) 534 goto out; 535 space = UIO_SYSSPACE; 536 where = userpath; 537 } else { 538 space = UIO_USERSPACE; 539 where = (char *)SCARG(uap, arg); 540 } 541 NDINIT(&nd, LOOKUP, FOLLOW|LOCKLEAF, space, where, p); 542 if ((error = namei(&nd))) 543 goto out; 544 vp = nd.ni_vp; 545 } 546 /* note: "vp" is referenced and locked */ 547 548 error = 0; /* assume no error */ 549 switch(SCARG(uap, cmd)) { 550 551 case SWAP_DUMPDEV: 552 if (vp->v_type != VBLK) { 553 error = ENOTBLK; 554 break; 555 } 556 dumpdev = vp->v_rdev; 557 cpu_dumpconf(); 558 break; 559 560 case SWAP_CTL: 561 /* 562 * get new priority, remove old entry (if any) and then 563 * reinsert it in the correct place. finally, prune out 564 * any empty priority structures. 565 */ 566 priority = SCARG(uap, misc); 567 spp = malloc(sizeof *spp, M_VMSWAP, M_WAITOK); 568 simple_lock(&uvm.swap_data_lock); 569 if ((sdp = swaplist_find(vp, 1)) == NULL) { 570 error = ENOENT; 571 } else { 572 swaplist_insert(sdp, spp, priority); 573 swaplist_trim(); 574 } 575 simple_unlock(&uvm.swap_data_lock); 576 if (error) 577 free(spp, M_VMSWAP); 578 break; 579 580 case SWAP_ON: 581 582 /* 583 * check for duplicates. if none found, then insert a 584 * dummy entry on the list to prevent someone else from 585 * trying to enable this device while we are working on 586 * it. 587 */ 588 589 priority = SCARG(uap, misc); 590 sdp = malloc(sizeof *sdp, M_VMSWAP, M_WAITOK); 591 spp = malloc(sizeof *spp, M_VMSWAP, M_WAITOK); 592 memset(sdp, 0, sizeof(*sdp)); 593 sdp->swd_flags = SWF_FAKE; 594 sdp->swd_vp = vp; 595 sdp->swd_dev = (vp->v_type == VBLK) ? vp->v_rdev : NODEV; 596 bufq_alloc(&sdp->swd_tab, BUFQ_DISKSORT|BUFQ_SORT_RAWBLOCK); 597 simple_lock(&uvm.swap_data_lock); 598 if (swaplist_find(vp, 0) != NULL) { 599 error = EBUSY; 600 simple_unlock(&uvm.swap_data_lock); 601 bufq_free(&sdp->swd_tab); 602 free(sdp, M_VMSWAP); 603 free(spp, M_VMSWAP); 604 break; 605 } 606 swaplist_insert(sdp, spp, priority); 607 simple_unlock(&uvm.swap_data_lock); 608 609 sdp->swd_pathlen = len; 610 sdp->swd_path = malloc(sdp->swd_pathlen, M_VMSWAP, M_WAITOK); 611 if (copystr(userpath, sdp->swd_path, sdp->swd_pathlen, 0) != 0) 612 panic("swapctl: copystr"); 613 614 /* 615 * we've now got a FAKE placeholder in the swap list. 616 * now attempt to enable swap on it. if we fail, undo 617 * what we've done and kill the fake entry we just inserted. 618 * if swap_on is a success, it will clear the SWF_FAKE flag 619 */ 620 621 if ((error = swap_on(p, sdp)) != 0) { 622 simple_lock(&uvm.swap_data_lock); 623 (void) swaplist_find(vp, 1); /* kill fake entry */ 624 swaplist_trim(); 625 simple_unlock(&uvm.swap_data_lock); 626 bufq_free(&sdp->swd_tab); 627 free(sdp->swd_path, M_VMSWAP); 628 free(sdp, M_VMSWAP); 629 break; 630 } 631 break; 632 633 case SWAP_OFF: 634 simple_lock(&uvm.swap_data_lock); 635 if ((sdp = swaplist_find(vp, 0)) == NULL) { 636 simple_unlock(&uvm.swap_data_lock); 637 error = ENXIO; 638 break; 639 } 640 641 /* 642 * If a device isn't in use or enabled, we 643 * can't stop swapping from it (again). 644 */ 645 if ((sdp->swd_flags & (SWF_INUSE|SWF_ENABLE)) == 0) { 646 simple_unlock(&uvm.swap_data_lock); 647 error = EBUSY; 648 break; 649 } 650 651 /* 652 * do the real work. 653 */ 654 error = swap_off(p, sdp); 655 break; 656 657 default: 658 error = EINVAL; 659 } 660 661 /* 662 * done! release the ref gained by namei() and unlock. 663 */ 664 vput(vp); 665 666 out: 667 lockmgr(&swap_syscall_lock, LK_RELEASE, NULL); 668 669 UVMHIST_LOG(pdhist, "<- done! error=%d", error, 0, 0, 0); 670 return (error); 671 } 672 673 /* 674 * swap_stats: implements swapctl(SWAP_STATS). The function is kept 675 * away from sys_swapctl() in order to allow COMPAT_* swapctl() 676 * emulation to use it directly without going through sys_swapctl(). 677 * The problem with using sys_swapctl() there is that it involves 678 * copying the swapent array to the stackgap, and this array's size 679 * is not known at build time. Hence it would not be possible to 680 * ensure it would fit in the stackgap in any case. 681 */ 682 void 683 uvm_swap_stats(int cmd, struct swapent *sep, int sec, register_t *retval) 684 { 685 struct swappri *spp; 686 struct swapdev *sdp; 687 int count = 0; 688 689 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 690 for (sdp = CIRCLEQ_FIRST(&spp->spi_swapdev); 691 sdp != (void *)&spp->spi_swapdev && sec-- > 0; 692 sdp = CIRCLEQ_NEXT(sdp, swd_next)) { 693 /* 694 * backwards compatibility for system call. 695 * note that we use 'struct oswapent' as an 696 * overlay into both 'struct swapdev' and 697 * the userland 'struct swapent', as we 698 * want to retain backwards compatibility 699 * with NetBSD 1.3. 700 */ 701 sdp->swd_ose.ose_inuse = 702 btodb((u_int64_t)sdp->swd_npginuse << 703 PAGE_SHIFT); 704 (void)memcpy(sep, &sdp->swd_ose, 705 sizeof(struct oswapent)); 706 707 /* now copy out the path if necessary */ 708 #if defined(COMPAT_13) 709 if (cmd == SWAP_STATS) 710 #endif 711 (void)memcpy(&sep->se_path, sdp->swd_path, 712 sdp->swd_pathlen); 713 714 count++; 715 #if defined(COMPAT_13) 716 if (cmd == SWAP_OSTATS) 717 sep = (struct swapent *) 718 ((struct oswapent *)sep + 1); 719 else 720 #endif 721 sep++; 722 } 723 } 724 725 *retval = count; 726 return; 727 } 728 729 /* 730 * swap_on: attempt to enable a swapdev for swapping. note that the 731 * swapdev is already on the global list, but disabled (marked 732 * SWF_FAKE). 733 * 734 * => we avoid the start of the disk (to protect disk labels) 735 * => we also avoid the miniroot, if we are swapping to root. 736 * => caller should leave uvm.swap_data_lock unlocked, we may lock it 737 * if needed. 738 */ 739 static int 740 swap_on(struct proc *p, struct swapdev *sdp) 741 { 742 struct vnode *vp; 743 int error, npages, nblocks, size; 744 long addr; 745 u_long result; 746 struct vattr va; 747 #ifdef NFS 748 extern int (**nfsv2_vnodeop_p)(void *); 749 #endif /* NFS */ 750 const struct bdevsw *bdev; 751 dev_t dev; 752 UVMHIST_FUNC("swap_on"); UVMHIST_CALLED(pdhist); 753 754 /* 755 * we want to enable swapping on sdp. the swd_vp contains 756 * the vnode we want (locked and ref'd), and the swd_dev 757 * contains the dev_t of the file, if it a block device. 758 */ 759 760 vp = sdp->swd_vp; 761 dev = sdp->swd_dev; 762 763 /* 764 * open the swap file (mostly useful for block device files to 765 * let device driver know what is up). 766 * 767 * we skip the open/close for root on swap because the root 768 * has already been opened when root was mounted (mountroot). 769 */ 770 if (vp != rootvp) { 771 if ((error = VOP_OPEN(vp, FREAD|FWRITE, p->p_ucred, p))) 772 return (error); 773 } 774 775 /* XXX this only works for block devices */ 776 UVMHIST_LOG(pdhist, " dev=%d, major(dev)=%d", dev, major(dev), 0,0); 777 778 /* 779 * we now need to determine the size of the swap area. for 780 * block specials we can call the d_psize function. 781 * for normal files, we must stat [get attrs]. 782 * 783 * we put the result in nblks. 784 * for normal files, we also want the filesystem block size 785 * (which we get with statfs). 786 */ 787 switch (vp->v_type) { 788 case VBLK: 789 bdev = bdevsw_lookup(dev); 790 if (bdev == NULL || bdev->d_psize == NULL || 791 (nblocks = (*bdev->d_psize)(dev)) == -1) { 792 error = ENXIO; 793 goto bad; 794 } 795 break; 796 797 case VREG: 798 if ((error = VOP_GETATTR(vp, &va, p->p_ucred, p))) 799 goto bad; 800 nblocks = (int)btodb(va.va_size); 801 if ((error = 802 VFS_STATVFS(vp->v_mount, &vp->v_mount->mnt_stat, p)) != 0) 803 goto bad; 804 805 sdp->swd_bsize = vp->v_mount->mnt_stat.f_iosize; 806 /* 807 * limit the max # of outstanding I/O requests we issue 808 * at any one time. take it easy on NFS servers. 809 */ 810 #ifdef NFS 811 if (vp->v_op == nfsv2_vnodeop_p) 812 sdp->swd_maxactive = 2; /* XXX */ 813 else 814 #endif /* NFS */ 815 sdp->swd_maxactive = 8; /* XXX */ 816 break; 817 818 default: 819 error = ENXIO; 820 goto bad; 821 } 822 823 /* 824 * save nblocks in a safe place and convert to pages. 825 */ 826 827 sdp->swd_ose.ose_nblks = nblocks; 828 npages = dbtob((u_int64_t)nblocks) >> PAGE_SHIFT; 829 830 /* 831 * for block special files, we want to make sure that leave 832 * the disklabel and bootblocks alone, so we arrange to skip 833 * over them (arbitrarily choosing to skip PAGE_SIZE bytes). 834 * note that because of this the "size" can be less than the 835 * actual number of blocks on the device. 836 */ 837 if (vp->v_type == VBLK) { 838 /* we use pages 1 to (size - 1) [inclusive] */ 839 size = npages - 1; 840 addr = 1; 841 } else { 842 /* we use pages 0 to (size - 1) [inclusive] */ 843 size = npages; 844 addr = 0; 845 } 846 847 /* 848 * make sure we have enough blocks for a reasonable sized swap 849 * area. we want at least one page. 850 */ 851 852 if (size < 1) { 853 UVMHIST_LOG(pdhist, " size <= 1!!", 0, 0, 0, 0); 854 error = EINVAL; 855 goto bad; 856 } 857 858 UVMHIST_LOG(pdhist, " dev=%x: size=%d addr=%ld\n", dev, size, addr, 0); 859 860 /* 861 * now we need to allocate an extent to manage this swap device 862 */ 863 864 sdp->swd_blist = blist_create(npages); 865 /* mark all expect the `saved' region free. */ 866 blist_free(sdp->swd_blist, addr, size); 867 868 /* 869 * if the vnode we are swapping to is the root vnode 870 * (i.e. we are swapping to the miniroot) then we want 871 * to make sure we don't overwrite it. do a statfs to 872 * find its size and skip over it. 873 */ 874 if (vp == rootvp) { 875 struct mount *mp; 876 struct statvfs *sp; 877 int rootblocks, rootpages; 878 879 mp = rootvnode->v_mount; 880 sp = &mp->mnt_stat; 881 rootblocks = sp->f_blocks * btodb(sp->f_frsize); 882 /* 883 * XXX: sp->f_blocks isn't the total number of 884 * blocks in the filesystem, it's the number of 885 * data blocks. so, our rootblocks almost 886 * definitely underestimates the total size 887 * of the filesystem - how badly depends on the 888 * details of the filesystem type. there isn't 889 * an obvious way to deal with this cleanly 890 * and perfectly, so for now we just pad our 891 * rootblocks estimate with an extra 5 percent. 892 */ 893 rootblocks += (rootblocks >> 5) + 894 (rootblocks >> 6) + 895 (rootblocks >> 7); 896 rootpages = round_page(dbtob(rootblocks)) >> PAGE_SHIFT; 897 if (rootpages > size) 898 panic("swap_on: miniroot larger than swap?"); 899 900 if (rootpages != blist_fill(sdp->swd_blist, addr, rootpages)) { 901 panic("swap_on: unable to preserve miniroot"); 902 } 903 904 size -= rootpages; 905 printf("Preserved %d pages of miniroot ", rootpages); 906 printf("leaving %d pages of swap\n", size); 907 } 908 909 /* 910 * add a ref to vp to reflect usage as a swap device. 911 */ 912 vref(vp); 913 914 /* 915 * now add the new swapdev to the drum and enable. 916 */ 917 if (extent_alloc(swapmap, npages, EX_NOALIGN, EX_NOBOUNDARY, 918 EX_WAITOK, &result)) 919 panic("swapdrum_add"); 920 921 sdp->swd_drumoffset = (int)result; 922 sdp->swd_drumsize = npages; 923 sdp->swd_npages = size; 924 simple_lock(&uvm.swap_data_lock); 925 sdp->swd_flags &= ~SWF_FAKE; /* going live */ 926 sdp->swd_flags |= (SWF_INUSE|SWF_ENABLE); 927 uvmexp.swpages += size; 928 uvmexp.swpgavail += size; 929 simple_unlock(&uvm.swap_data_lock); 930 return (0); 931 932 /* 933 * failure: clean up and return error. 934 */ 935 936 bad: 937 if (sdp->swd_blist) { 938 blist_destroy(sdp->swd_blist); 939 } 940 if (vp != rootvp) { 941 (void)VOP_CLOSE(vp, FREAD|FWRITE, p->p_ucred, p); 942 } 943 return (error); 944 } 945 946 /* 947 * swap_off: stop swapping on swapdev 948 * 949 * => swap data should be locked, we will unlock. 950 */ 951 static int 952 swap_off(struct proc *p, struct swapdev *sdp) 953 { 954 int npages = sdp->swd_npages; 955 int error = 0; 956 957 UVMHIST_FUNC("swap_off"); UVMHIST_CALLED(pdhist); 958 UVMHIST_LOG(pdhist, " dev=%x, npages=%d", sdp->swd_dev,npages,0,0); 959 960 /* disable the swap area being removed */ 961 sdp->swd_flags &= ~SWF_ENABLE; 962 uvmexp.swpgavail -= npages; 963 simple_unlock(&uvm.swap_data_lock); 964 965 /* 966 * the idea is to find all the pages that are paged out to this 967 * device, and page them all in. in uvm, swap-backed pageable 968 * memory can take two forms: aobjs and anons. call the 969 * swapoff hook for each subsystem to bring in pages. 970 */ 971 972 if (uao_swap_off(sdp->swd_drumoffset, 973 sdp->swd_drumoffset + sdp->swd_drumsize) || 974 amap_swap_off(sdp->swd_drumoffset, 975 sdp->swd_drumoffset + sdp->swd_drumsize)) { 976 error = ENOMEM; 977 } else if (sdp->swd_npginuse > sdp->swd_npgbad) { 978 error = EBUSY; 979 } 980 981 if (error) { 982 simple_lock(&uvm.swap_data_lock); 983 sdp->swd_flags |= SWF_ENABLE; 984 uvmexp.swpgavail += npages; 985 simple_unlock(&uvm.swap_data_lock); 986 987 return error; 988 } 989 990 /* 991 * done with the vnode. 992 * drop our ref on the vnode before calling VOP_CLOSE() 993 * so that spec_close() can tell if this is the last close. 994 */ 995 vrele(sdp->swd_vp); 996 if (sdp->swd_vp != rootvp) { 997 (void) VOP_CLOSE(sdp->swd_vp, FREAD|FWRITE, p->p_ucred, p); 998 } 999 1000 simple_lock(&uvm.swap_data_lock); 1001 uvmexp.swpages -= npages; 1002 uvmexp.swpginuse -= sdp->swd_npgbad; 1003 1004 if (swaplist_find(sdp->swd_vp, 1) == NULL) 1005 panic("swap_off: swapdev not in list"); 1006 swaplist_trim(); 1007 simple_unlock(&uvm.swap_data_lock); 1008 1009 /* 1010 * free all resources! 1011 */ 1012 extent_free(swapmap, sdp->swd_drumoffset, sdp->swd_drumsize, 1013 EX_WAITOK); 1014 blist_destroy(sdp->swd_blist); 1015 bufq_free(&sdp->swd_tab); 1016 free(sdp, M_VMSWAP); 1017 return (0); 1018 } 1019 1020 /* 1021 * /dev/drum interface and i/o functions 1022 */ 1023 1024 /* 1025 * swstrategy: perform I/O on the drum 1026 * 1027 * => we must map the i/o request from the drum to the correct swapdev. 1028 */ 1029 static void 1030 swstrategy(struct buf *bp) 1031 { 1032 struct swapdev *sdp; 1033 struct vnode *vp; 1034 int s, pageno, bn; 1035 UVMHIST_FUNC("swstrategy"); UVMHIST_CALLED(pdhist); 1036 1037 /* 1038 * convert block number to swapdev. note that swapdev can't 1039 * be yanked out from under us because we are holding resources 1040 * in it (i.e. the blocks we are doing I/O on). 1041 */ 1042 pageno = dbtob((int64_t)bp->b_blkno) >> PAGE_SHIFT; 1043 simple_lock(&uvm.swap_data_lock); 1044 sdp = swapdrum_getsdp(pageno); 1045 simple_unlock(&uvm.swap_data_lock); 1046 if (sdp == NULL) { 1047 bp->b_error = EINVAL; 1048 bp->b_flags |= B_ERROR; 1049 biodone(bp); 1050 UVMHIST_LOG(pdhist, " failed to get swap device", 0, 0, 0, 0); 1051 return; 1052 } 1053 1054 /* 1055 * convert drum page number to block number on this swapdev. 1056 */ 1057 1058 pageno -= sdp->swd_drumoffset; /* page # on swapdev */ 1059 bn = btodb((u_int64_t)pageno << PAGE_SHIFT); /* convert to diskblock */ 1060 1061 UVMHIST_LOG(pdhist, " %s: mapoff=%x bn=%x bcount=%ld", 1062 ((bp->b_flags & B_READ) == 0) ? "write" : "read", 1063 sdp->swd_drumoffset, bn, bp->b_bcount); 1064 1065 /* 1066 * for block devices we finish up here. 1067 * for regular files we have to do more work which we delegate 1068 * to sw_reg_strategy(). 1069 */ 1070 1071 switch (sdp->swd_vp->v_type) { 1072 default: 1073 panic("swstrategy: vnode type 0x%x", sdp->swd_vp->v_type); 1074 1075 case VBLK: 1076 1077 /* 1078 * must convert "bp" from an I/O on /dev/drum to an I/O 1079 * on the swapdev (sdp). 1080 */ 1081 s = splbio(); 1082 bp->b_blkno = bn; /* swapdev block number */ 1083 vp = sdp->swd_vp; /* swapdev vnode pointer */ 1084 bp->b_dev = sdp->swd_dev; /* swapdev dev_t */ 1085 1086 /* 1087 * if we are doing a write, we have to redirect the i/o on 1088 * drum's v_numoutput counter to the swapdevs. 1089 */ 1090 if ((bp->b_flags & B_READ) == 0) { 1091 vwakeup(bp); /* kills one 'v_numoutput' on drum */ 1092 V_INCR_NUMOUTPUT(vp); /* put it on swapdev */ 1093 } 1094 1095 /* 1096 * finally plug in swapdev vnode and start I/O 1097 */ 1098 bp->b_vp = vp; 1099 splx(s); 1100 VOP_STRATEGY(vp, bp); 1101 return; 1102 1103 case VREG: 1104 /* 1105 * delegate to sw_reg_strategy function. 1106 */ 1107 sw_reg_strategy(sdp, bp, bn); 1108 return; 1109 } 1110 /* NOTREACHED */ 1111 } 1112 1113 /* 1114 * swread: the read function for the drum (just a call to physio) 1115 */ 1116 /*ARGSUSED*/ 1117 static int 1118 swread(dev_t dev, struct uio *uio, int ioflag) 1119 { 1120 UVMHIST_FUNC("swread"); UVMHIST_CALLED(pdhist); 1121 1122 UVMHIST_LOG(pdhist, " dev=%x offset=%qx", dev, uio->uio_offset, 0, 0); 1123 return (physio(swstrategy, NULL, dev, B_READ, minphys, uio)); 1124 } 1125 1126 /* 1127 * swwrite: the write function for the drum (just a call to physio) 1128 */ 1129 /*ARGSUSED*/ 1130 static int 1131 swwrite(dev_t dev, struct uio *uio, int ioflag) 1132 { 1133 UVMHIST_FUNC("swwrite"); UVMHIST_CALLED(pdhist); 1134 1135 UVMHIST_LOG(pdhist, " dev=%x offset=%qx", dev, uio->uio_offset, 0, 0); 1136 return (physio(swstrategy, NULL, dev, B_WRITE, minphys, uio)); 1137 } 1138 1139 const struct bdevsw swap_bdevsw = { 1140 noopen, noclose, swstrategy, noioctl, nodump, nosize, 1141 }; 1142 1143 const struct cdevsw swap_cdevsw = { 1144 nullopen, nullclose, swread, swwrite, noioctl, 1145 nostop, notty, nopoll, nommap, nokqfilter 1146 }; 1147 1148 /* 1149 * sw_reg_strategy: handle swap i/o to regular files 1150 */ 1151 static void 1152 sw_reg_strategy(struct swapdev *sdp, struct buf *bp, int bn) 1153 { 1154 struct vnode *vp; 1155 struct vndxfer *vnx; 1156 daddr_t nbn; 1157 caddr_t addr; 1158 off_t byteoff; 1159 int s, off, nra, error, sz, resid; 1160 UVMHIST_FUNC("sw_reg_strategy"); UVMHIST_CALLED(pdhist); 1161 1162 /* 1163 * allocate a vndxfer head for this transfer and point it to 1164 * our buffer. 1165 */ 1166 getvndxfer(vnx); 1167 vnx->vx_flags = VX_BUSY; 1168 vnx->vx_error = 0; 1169 vnx->vx_pending = 0; 1170 vnx->vx_bp = bp; 1171 vnx->vx_sdp = sdp; 1172 1173 /* 1174 * setup for main loop where we read filesystem blocks into 1175 * our buffer. 1176 */ 1177 error = 0; 1178 bp->b_resid = bp->b_bcount; /* nothing transfered yet! */ 1179 addr = bp->b_data; /* current position in buffer */ 1180 byteoff = dbtob((u_int64_t)bn); 1181 1182 for (resid = bp->b_resid; resid; resid -= sz) { 1183 struct vndbuf *nbp; 1184 1185 /* 1186 * translate byteoffset into block number. return values: 1187 * vp = vnode of underlying device 1188 * nbn = new block number (on underlying vnode dev) 1189 * nra = num blocks we can read-ahead (excludes requested 1190 * block) 1191 */ 1192 nra = 0; 1193 error = VOP_BMAP(sdp->swd_vp, byteoff / sdp->swd_bsize, 1194 &vp, &nbn, &nra); 1195 1196 if (error == 0 && nbn == (daddr_t)-1) { 1197 /* 1198 * this used to just set error, but that doesn't 1199 * do the right thing. Instead, it causes random 1200 * memory errors. The panic() should remain until 1201 * this condition doesn't destabilize the system. 1202 */ 1203 #if 1 1204 panic("sw_reg_strategy: swap to sparse file"); 1205 #else 1206 error = EIO; /* failure */ 1207 #endif 1208 } 1209 1210 /* 1211 * punt if there was an error or a hole in the file. 1212 * we must wait for any i/o ops we have already started 1213 * to finish before returning. 1214 * 1215 * XXX we could deal with holes here but it would be 1216 * a hassle (in the write case). 1217 */ 1218 if (error) { 1219 s = splbio(); 1220 vnx->vx_error = error; /* pass error up */ 1221 goto out; 1222 } 1223 1224 /* 1225 * compute the size ("sz") of this transfer (in bytes). 1226 */ 1227 off = byteoff % sdp->swd_bsize; 1228 sz = (1 + nra) * sdp->swd_bsize - off; 1229 if (sz > resid) 1230 sz = resid; 1231 1232 UVMHIST_LOG(pdhist, "sw_reg_strategy: " 1233 "vp %p/%p offset 0x%x/0x%x", 1234 sdp->swd_vp, vp, byteoff, nbn); 1235 1236 /* 1237 * now get a buf structure. note that the vb_buf is 1238 * at the front of the nbp structure so that you can 1239 * cast pointers between the two structure easily. 1240 */ 1241 getvndbuf(nbp); 1242 BUF_INIT(&nbp->vb_buf); 1243 nbp->vb_buf.b_flags = bp->b_flags | B_CALL; 1244 nbp->vb_buf.b_bcount = sz; 1245 nbp->vb_buf.b_bufsize = sz; 1246 nbp->vb_buf.b_error = 0; 1247 nbp->vb_buf.b_data = addr; 1248 nbp->vb_buf.b_lblkno = 0; 1249 nbp->vb_buf.b_blkno = nbn + btodb(off); 1250 nbp->vb_buf.b_rawblkno = nbp->vb_buf.b_blkno; 1251 nbp->vb_buf.b_iodone = sw_reg_iodone; 1252 nbp->vb_buf.b_vp = vp; 1253 if (vp->v_type == VBLK) { 1254 nbp->vb_buf.b_dev = vp->v_rdev; 1255 } 1256 1257 nbp->vb_xfer = vnx; /* patch it back in to vnx */ 1258 1259 /* 1260 * Just sort by block number 1261 */ 1262 s = splbio(); 1263 if (vnx->vx_error != 0) { 1264 putvndbuf(nbp); 1265 goto out; 1266 } 1267 vnx->vx_pending++; 1268 1269 /* sort it in and start I/O if we are not over our limit */ 1270 BUFQ_PUT(&sdp->swd_tab, &nbp->vb_buf); 1271 sw_reg_start(sdp); 1272 splx(s); 1273 1274 /* 1275 * advance to the next I/O 1276 */ 1277 byteoff += sz; 1278 addr += sz; 1279 } 1280 1281 s = splbio(); 1282 1283 out: /* Arrive here at splbio */ 1284 vnx->vx_flags &= ~VX_BUSY; 1285 if (vnx->vx_pending == 0) { 1286 if (vnx->vx_error != 0) { 1287 bp->b_error = vnx->vx_error; 1288 bp->b_flags |= B_ERROR; 1289 } 1290 putvndxfer(vnx); 1291 biodone(bp); 1292 } 1293 splx(s); 1294 } 1295 1296 /* 1297 * sw_reg_start: start an I/O request on the requested swapdev 1298 * 1299 * => reqs are sorted by b_rawblkno (above) 1300 */ 1301 static void 1302 sw_reg_start(struct swapdev *sdp) 1303 { 1304 struct buf *bp; 1305 UVMHIST_FUNC("sw_reg_start"); UVMHIST_CALLED(pdhist); 1306 1307 /* recursion control */ 1308 if ((sdp->swd_flags & SWF_BUSY) != 0) 1309 return; 1310 1311 sdp->swd_flags |= SWF_BUSY; 1312 1313 while (sdp->swd_active < sdp->swd_maxactive) { 1314 bp = BUFQ_GET(&sdp->swd_tab); 1315 if (bp == NULL) 1316 break; 1317 sdp->swd_active++; 1318 1319 UVMHIST_LOG(pdhist, 1320 "sw_reg_start: bp %p vp %p blkno %p cnt %lx", 1321 bp, bp->b_vp, bp->b_blkno, bp->b_bcount); 1322 if ((bp->b_flags & B_READ) == 0) 1323 V_INCR_NUMOUTPUT(bp->b_vp); 1324 1325 VOP_STRATEGY(bp->b_vp, bp); 1326 } 1327 sdp->swd_flags &= ~SWF_BUSY; 1328 } 1329 1330 /* 1331 * sw_reg_iodone: one of our i/o's has completed and needs post-i/o cleanup 1332 * 1333 * => note that we can recover the vndbuf struct by casting the buf ptr 1334 */ 1335 static void 1336 sw_reg_iodone(struct buf *bp) 1337 { 1338 struct vndbuf *vbp = (struct vndbuf *) bp; 1339 struct vndxfer *vnx = vbp->vb_xfer; 1340 struct buf *pbp = vnx->vx_bp; /* parent buffer */ 1341 struct swapdev *sdp = vnx->vx_sdp; 1342 int s, resid, error; 1343 UVMHIST_FUNC("sw_reg_iodone"); UVMHIST_CALLED(pdhist); 1344 1345 UVMHIST_LOG(pdhist, " vbp=%p vp=%p blkno=%x addr=%p", 1346 vbp, vbp->vb_buf.b_vp, vbp->vb_buf.b_blkno, vbp->vb_buf.b_data); 1347 UVMHIST_LOG(pdhist, " cnt=%lx resid=%lx", 1348 vbp->vb_buf.b_bcount, vbp->vb_buf.b_resid, 0, 0); 1349 1350 /* 1351 * protect vbp at splbio and update. 1352 */ 1353 1354 s = splbio(); 1355 resid = vbp->vb_buf.b_bcount - vbp->vb_buf.b_resid; 1356 pbp->b_resid -= resid; 1357 vnx->vx_pending--; 1358 1359 if (vbp->vb_buf.b_flags & B_ERROR) { 1360 /* pass error upward */ 1361 error = vbp->vb_buf.b_error ? vbp->vb_buf.b_error : EIO; 1362 UVMHIST_LOG(pdhist, " got error=%d !", error, 0, 0, 0); 1363 vnx->vx_error = error; 1364 } 1365 1366 /* 1367 * kill vbp structure 1368 */ 1369 putvndbuf(vbp); 1370 1371 /* 1372 * wrap up this transaction if it has run to completion or, in 1373 * case of an error, when all auxiliary buffers have returned. 1374 */ 1375 if (vnx->vx_error != 0) { 1376 /* pass error upward */ 1377 pbp->b_flags |= B_ERROR; 1378 pbp->b_error = vnx->vx_error; 1379 if ((vnx->vx_flags & VX_BUSY) == 0 && vnx->vx_pending == 0) { 1380 putvndxfer(vnx); 1381 biodone(pbp); 1382 } 1383 } else if (pbp->b_resid == 0) { 1384 KASSERT(vnx->vx_pending == 0); 1385 if ((vnx->vx_flags & VX_BUSY) == 0) { 1386 UVMHIST_LOG(pdhist, " iodone error=%d !", 1387 pbp, vnx->vx_error, 0, 0); 1388 putvndxfer(vnx); 1389 biodone(pbp); 1390 } 1391 } 1392 1393 /* 1394 * done! start next swapdev I/O if one is pending 1395 */ 1396 sdp->swd_active--; 1397 sw_reg_start(sdp); 1398 splx(s); 1399 } 1400 1401 1402 /* 1403 * uvm_swap_alloc: allocate space on swap 1404 * 1405 * => allocation is done "round robin" down the priority list, as we 1406 * allocate in a priority we "rotate" the circle queue. 1407 * => space can be freed with uvm_swap_free 1408 * => we return the page slot number in /dev/drum (0 == invalid slot) 1409 * => we lock uvm.swap_data_lock 1410 * => XXXMRG: "LESSOK" INTERFACE NEEDED TO EXTENT SYSTEM 1411 */ 1412 int 1413 uvm_swap_alloc(int *nslots /* IN/OUT */, boolean_t lessok) 1414 { 1415 struct swapdev *sdp; 1416 struct swappri *spp; 1417 UVMHIST_FUNC("uvm_swap_alloc"); UVMHIST_CALLED(pdhist); 1418 1419 /* 1420 * no swap devices configured yet? definite failure. 1421 */ 1422 if (uvmexp.nswapdev < 1) 1423 return 0; 1424 1425 /* 1426 * lock data lock, convert slots into blocks, and enter loop 1427 */ 1428 simple_lock(&uvm.swap_data_lock); 1429 1430 ReTry: /* XXXMRG */ 1431 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 1432 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 1433 uint64_t result; 1434 1435 /* if it's not enabled, then we can't swap from it */ 1436 if ((sdp->swd_flags & SWF_ENABLE) == 0) 1437 continue; 1438 if (sdp->swd_npginuse + *nslots > sdp->swd_npages) 1439 continue; 1440 result = blist_alloc(sdp->swd_blist, *nslots); 1441 if (result == BLIST_NONE) { 1442 continue; 1443 } 1444 KASSERT(result < sdp->swd_drumsize); 1445 1446 /* 1447 * successful allocation! now rotate the circleq. 1448 */ 1449 CIRCLEQ_REMOVE(&spp->spi_swapdev, sdp, swd_next); 1450 CIRCLEQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next); 1451 sdp->swd_npginuse += *nslots; 1452 uvmexp.swpginuse += *nslots; 1453 simple_unlock(&uvm.swap_data_lock); 1454 /* done! return drum slot number */ 1455 UVMHIST_LOG(pdhist, 1456 "success! returning %d slots starting at %d", 1457 *nslots, result + sdp->swd_drumoffset, 0, 0); 1458 return (result + sdp->swd_drumoffset); 1459 } 1460 } 1461 1462 /* XXXMRG: BEGIN HACK */ 1463 if (*nslots > 1 && lessok) { 1464 *nslots = 1; 1465 /* XXXMRG: ugh! blist should support this for us */ 1466 goto ReTry; 1467 } 1468 /* XXXMRG: END HACK */ 1469 1470 simple_unlock(&uvm.swap_data_lock); 1471 return 0; 1472 } 1473 1474 boolean_t 1475 uvm_swapisfull(void) 1476 { 1477 boolean_t rv; 1478 1479 simple_lock(&uvm.swap_data_lock); 1480 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1481 rv = (uvmexp.swpgonly >= uvmexp.swpgavail); 1482 simple_unlock(&uvm.swap_data_lock); 1483 1484 return (rv); 1485 } 1486 1487 /* 1488 * uvm_swap_markbad: keep track of swap ranges where we've had i/o errors 1489 * 1490 * => we lock uvm.swap_data_lock 1491 */ 1492 void 1493 uvm_swap_markbad(int startslot, int nslots) 1494 { 1495 struct swapdev *sdp; 1496 UVMHIST_FUNC("uvm_swap_markbad"); UVMHIST_CALLED(pdhist); 1497 1498 simple_lock(&uvm.swap_data_lock); 1499 sdp = swapdrum_getsdp(startslot); 1500 KASSERT(sdp != NULL); 1501 1502 /* 1503 * we just keep track of how many pages have been marked bad 1504 * in this device, to make everything add up in swap_off(). 1505 * we assume here that the range of slots will all be within 1506 * one swap device. 1507 */ 1508 1509 KASSERT(uvmexp.swpgonly >= nslots); 1510 uvmexp.swpgonly -= nslots; 1511 sdp->swd_npgbad += nslots; 1512 UVMHIST_LOG(pdhist, "now %d bad", sdp->swd_npgbad, 0,0,0); 1513 simple_unlock(&uvm.swap_data_lock); 1514 } 1515 1516 /* 1517 * uvm_swap_free: free swap slots 1518 * 1519 * => this can be all or part of an allocation made by uvm_swap_alloc 1520 * => we lock uvm.swap_data_lock 1521 */ 1522 void 1523 uvm_swap_free(int startslot, int nslots) 1524 { 1525 struct swapdev *sdp; 1526 UVMHIST_FUNC("uvm_swap_free"); UVMHIST_CALLED(pdhist); 1527 1528 UVMHIST_LOG(pdhist, "freeing %d slots starting at %d", nslots, 1529 startslot, 0, 0); 1530 1531 /* 1532 * ignore attempts to free the "bad" slot. 1533 */ 1534 1535 if (startslot == SWSLOT_BAD) { 1536 return; 1537 } 1538 1539 /* 1540 * convert drum slot offset back to sdp, free the blocks 1541 * in the extent, and return. must hold pri lock to do 1542 * lookup and access the extent. 1543 */ 1544 1545 simple_lock(&uvm.swap_data_lock); 1546 sdp = swapdrum_getsdp(startslot); 1547 KASSERT(uvmexp.nswapdev >= 1); 1548 KASSERT(sdp != NULL); 1549 KASSERT(sdp->swd_npginuse >= nslots); 1550 blist_free(sdp->swd_blist, startslot - sdp->swd_drumoffset, nslots); 1551 sdp->swd_npginuse -= nslots; 1552 uvmexp.swpginuse -= nslots; 1553 simple_unlock(&uvm.swap_data_lock); 1554 } 1555 1556 /* 1557 * uvm_swap_put: put any number of pages into a contig place on swap 1558 * 1559 * => can be sync or async 1560 */ 1561 1562 int 1563 uvm_swap_put(int swslot, struct vm_page **ppsp, int npages, int flags) 1564 { 1565 int error; 1566 1567 error = uvm_swap_io(ppsp, swslot, npages, B_WRITE | 1568 ((flags & PGO_SYNCIO) ? 0 : B_ASYNC)); 1569 return error; 1570 } 1571 1572 /* 1573 * uvm_swap_get: get a single page from swap 1574 * 1575 * => usually a sync op (from fault) 1576 */ 1577 1578 int 1579 uvm_swap_get(struct vm_page *page, int swslot, int flags) 1580 { 1581 int error; 1582 1583 uvmexp.nswget++; 1584 KASSERT(flags & PGO_SYNCIO); 1585 if (swslot == SWSLOT_BAD) { 1586 return EIO; 1587 } 1588 1589 error = uvm_swap_io(&page, swslot, 1, B_READ | 1590 ((flags & PGO_SYNCIO) ? 0 : B_ASYNC)); 1591 if (error == 0) { 1592 1593 /* 1594 * this page is no longer only in swap. 1595 */ 1596 1597 simple_lock(&uvm.swap_data_lock); 1598 KASSERT(uvmexp.swpgonly > 0); 1599 uvmexp.swpgonly--; 1600 simple_unlock(&uvm.swap_data_lock); 1601 } 1602 return error; 1603 } 1604 1605 /* 1606 * uvm_swap_io: do an i/o operation to swap 1607 */ 1608 1609 static int 1610 uvm_swap_io(struct vm_page **pps, int startslot, int npages, int flags) 1611 { 1612 daddr_t startblk; 1613 struct buf *bp; 1614 vaddr_t kva; 1615 int error, s, mapinflags; 1616 boolean_t write, async; 1617 UVMHIST_FUNC("uvm_swap_io"); UVMHIST_CALLED(pdhist); 1618 1619 UVMHIST_LOG(pdhist, "<- called, startslot=%d, npages=%d, flags=%d", 1620 startslot, npages, flags, 0); 1621 1622 write = (flags & B_READ) == 0; 1623 async = (flags & B_ASYNC) != 0; 1624 1625 /* 1626 * convert starting drum slot to block number 1627 */ 1628 1629 startblk = btodb((u_int64_t)startslot << PAGE_SHIFT); 1630 1631 /* 1632 * first, map the pages into the kernel. 1633 */ 1634 1635 mapinflags = !write ? 1636 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_READ : 1637 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_WRITE; 1638 kva = uvm_pagermapin(pps, npages, mapinflags); 1639 1640 /* 1641 * now allocate a buf for the i/o. 1642 */ 1643 1644 s = splbio(); 1645 bp = pool_get(&bufpool, PR_WAITOK); 1646 splx(s); 1647 1648 /* 1649 * fill in the bp/sbp. we currently route our i/o through 1650 * /dev/drum's vnode [swapdev_vp]. 1651 */ 1652 1653 BUF_INIT(bp); 1654 bp->b_flags = B_BUSY | B_NOCACHE | (flags & (B_READ|B_ASYNC)); 1655 bp->b_proc = &proc0; /* XXX */ 1656 bp->b_vnbufs.le_next = NOLIST; 1657 bp->b_data = (caddr_t)kva; 1658 bp->b_blkno = startblk; 1659 bp->b_vp = swapdev_vp; 1660 bp->b_bufsize = bp->b_bcount = npages << PAGE_SHIFT; 1661 1662 /* 1663 * bump v_numoutput (counter of number of active outputs). 1664 */ 1665 1666 if (write) { 1667 s = splbio(); 1668 V_INCR_NUMOUTPUT(swapdev_vp); 1669 splx(s); 1670 } 1671 1672 /* 1673 * for async ops we must set up the iodone handler. 1674 */ 1675 1676 if (async) { 1677 bp->b_flags |= B_CALL; 1678 bp->b_iodone = uvm_aio_biodone; 1679 UVMHIST_LOG(pdhist, "doing async!", 0, 0, 0, 0); 1680 if (curproc == uvm.pagedaemon_proc) 1681 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); 1682 else 1683 BIO_SETPRIO(bp, BPRIO_TIMELIMITED); 1684 } else { 1685 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); 1686 } 1687 UVMHIST_LOG(pdhist, 1688 "about to start io: data = %p blkno = 0x%x, bcount = %ld", 1689 bp->b_data, bp->b_blkno, bp->b_bcount, 0); 1690 1691 /* 1692 * now we start the I/O, and if async, return. 1693 */ 1694 1695 VOP_STRATEGY(swapdev_vp, bp); 1696 if (async) 1697 return 0; 1698 1699 /* 1700 * must be sync i/o. wait for it to finish 1701 */ 1702 1703 error = biowait(bp); 1704 1705 /* 1706 * kill the pager mapping 1707 */ 1708 1709 uvm_pagermapout(kva, npages); 1710 1711 /* 1712 * now dispose of the buf and we're done. 1713 */ 1714 1715 s = splbio(); 1716 if (write) 1717 vwakeup(bp); 1718 pool_put(&bufpool, bp); 1719 splx(s); 1720 UVMHIST_LOG(pdhist, "<- done (sync) error=%d", error, 0, 0, 0); 1721 return (error); 1722 } 1723