1 /* $NetBSD: uvm_swap.c,v 1.108 2006/10/12 04:35:40 thorpej Exp $ */ 2 3 /* 4 * Copyright (c) 1995, 1996, 1997 Matthew R. Green 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of the author may not be used to endorse or promote products 16 * derived from this software without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 * from: NetBSD: vm_swap.c,v 1.52 1997/12/02 13:47:37 pk Exp 31 * from: Id: uvm_swap.c,v 1.1.2.42 1998/02/02 20:38:06 chuck Exp 32 */ 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.108 2006/10/12 04:35:40 thorpej Exp $"); 36 37 #include "fs_nfs.h" 38 #include "opt_uvmhist.h" 39 #include "opt_compat_netbsd.h" 40 #include "opt_ddb.h" 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/buf.h> 45 #include <sys/bufq.h> 46 #include <sys/conf.h> 47 #include <sys/proc.h> 48 #include <sys/namei.h> 49 #include <sys/disklabel.h> 50 #include <sys/errno.h> 51 #include <sys/kernel.h> 52 #include <sys/malloc.h> 53 #include <sys/vnode.h> 54 #include <sys/file.h> 55 #include <sys/extent.h> 56 #include <sys/blist.h> 57 #include <sys/mount.h> 58 #include <sys/pool.h> 59 #include <sys/sa.h> 60 #include <sys/syscallargs.h> 61 #include <sys/swap.h> 62 #include <sys/kauth.h> 63 64 #include <uvm/uvm.h> 65 66 #include <miscfs/specfs/specdev.h> 67 68 /* 69 * uvm_swap.c: manage configuration and i/o to swap space. 70 */ 71 72 /* 73 * swap space is managed in the following way: 74 * 75 * each swap partition or file is described by a "swapdev" structure. 76 * each "swapdev" structure contains a "swapent" structure which contains 77 * information that is passed up to the user (via system calls). 78 * 79 * each swap partition is assigned a "priority" (int) which controls 80 * swap parition usage. 81 * 82 * the system maintains a global data structure describing all swap 83 * partitions/files. there is a sorted LIST of "swappri" structures 84 * which describe "swapdev"'s at that priority. this LIST is headed 85 * by the "swap_priority" global var. each "swappri" contains a 86 * CIRCLEQ of "swapdev" structures at that priority. 87 * 88 * locking: 89 * - swap_syscall_lock (sleep lock): this lock serializes the swapctl 90 * system call and prevents the swap priority list from changing 91 * while we are in the middle of a system call (e.g. SWAP_STATS). 92 * - uvm.swap_data_lock (simple_lock): this lock protects all swap data 93 * structures including the priority list, the swapdev structures, 94 * and the swapmap extent. 95 * 96 * each swap device has the following info: 97 * - swap device in use (could be disabled, preventing future use) 98 * - swap enabled (allows new allocations on swap) 99 * - map info in /dev/drum 100 * - vnode pointer 101 * for swap files only: 102 * - block size 103 * - max byte count in buffer 104 * - buffer 105 * 106 * userland controls and configures swap with the swapctl(2) system call. 107 * the sys_swapctl performs the following operations: 108 * [1] SWAP_NSWAP: returns the number of swap devices currently configured 109 * [2] SWAP_STATS: given a pointer to an array of swapent structures 110 * (passed in via "arg") of a size passed in via "misc" ... we load 111 * the current swap config into the array. The actual work is done 112 * in the uvm_swap_stats(9) function. 113 * [3] SWAP_ON: given a pathname in arg (could be device or file) and a 114 * priority in "misc", start swapping on it. 115 * [4] SWAP_OFF: as SWAP_ON, but stops swapping to a device 116 * [5] SWAP_CTL: changes the priority of a swap device (new priority in 117 * "misc") 118 */ 119 120 /* 121 * swapdev: describes a single swap partition/file 122 * 123 * note the following should be true: 124 * swd_inuse <= swd_nblks [number of blocks in use is <= total blocks] 125 * swd_nblks <= swd_mapsize [because mapsize includes miniroot+disklabel] 126 */ 127 struct swapdev { 128 struct oswapent swd_ose; 129 #define swd_dev swd_ose.ose_dev /* device id */ 130 #define swd_flags swd_ose.ose_flags /* flags:inuse/enable/fake */ 131 #define swd_priority swd_ose.ose_priority /* our priority */ 132 /* also: swd_ose.ose_nblks, swd_ose.ose_inuse */ 133 char *swd_path; /* saved pathname of device */ 134 int swd_pathlen; /* length of pathname */ 135 int swd_npages; /* #pages we can use */ 136 int swd_npginuse; /* #pages in use */ 137 int swd_npgbad; /* #pages bad */ 138 int swd_drumoffset; /* page0 offset in drum */ 139 int swd_drumsize; /* #pages in drum */ 140 blist_t swd_blist; /* blist for this swapdev */ 141 struct vnode *swd_vp; /* backing vnode */ 142 CIRCLEQ_ENTRY(swapdev) swd_next; /* priority circleq */ 143 144 int swd_bsize; /* blocksize (bytes) */ 145 int swd_maxactive; /* max active i/o reqs */ 146 struct bufq_state *swd_tab; /* buffer list */ 147 int swd_active; /* number of active buffers */ 148 }; 149 150 /* 151 * swap device priority entry; the list is kept sorted on `spi_priority'. 152 */ 153 struct swappri { 154 int spi_priority; /* priority */ 155 CIRCLEQ_HEAD(spi_swapdev, swapdev) spi_swapdev; 156 /* circleq of swapdevs at this priority */ 157 LIST_ENTRY(swappri) spi_swappri; /* global list of pri's */ 158 }; 159 160 /* 161 * The following two structures are used to keep track of data transfers 162 * on swap devices associated with regular files. 163 * NOTE: this code is more or less a copy of vnd.c; we use the same 164 * structure names here to ease porting.. 165 */ 166 struct vndxfer { 167 struct buf *vx_bp; /* Pointer to parent buffer */ 168 struct swapdev *vx_sdp; 169 int vx_error; 170 int vx_pending; /* # of pending aux buffers */ 171 int vx_flags; 172 #define VX_BUSY 1 173 #define VX_DEAD 2 174 }; 175 176 struct vndbuf { 177 struct buf vb_buf; 178 struct vndxfer *vb_xfer; 179 }; 180 181 182 /* 183 * We keep a of pool vndbuf's and vndxfer structures. 184 */ 185 POOL_INIT(vndxfer_pool, sizeof(struct vndxfer), 0, 0, 0, "swp vnx", NULL); 186 POOL_INIT(vndbuf_pool, sizeof(struct vndbuf), 0, 0, 0, "swp vnd", NULL); 187 188 #define getvndxfer(vnx) do { \ 189 int sp = splbio(); \ 190 vnx = pool_get(&vndxfer_pool, PR_WAITOK); \ 191 splx(sp); \ 192 } while (/*CONSTCOND*/ 0) 193 194 #define putvndxfer(vnx) { \ 195 pool_put(&vndxfer_pool, (void *)(vnx)); \ 196 } 197 198 #define getvndbuf(vbp) do { \ 199 int sp = splbio(); \ 200 vbp = pool_get(&vndbuf_pool, PR_WAITOK); \ 201 splx(sp); \ 202 } while (/*CONSTCOND*/ 0) 203 204 #define putvndbuf(vbp) { \ 205 pool_put(&vndbuf_pool, (void *)(vbp)); \ 206 } 207 208 /* 209 * local variables 210 */ 211 static struct extent *swapmap; /* controls the mapping of /dev/drum */ 212 213 MALLOC_DEFINE(M_VMSWAP, "VM swap", "VM swap structures"); 214 215 /* list of all active swap devices [by priority] */ 216 LIST_HEAD(swap_priority, swappri); 217 static struct swap_priority swap_priority; 218 219 /* locks */ 220 static struct lock swap_syscall_lock; 221 222 /* 223 * prototypes 224 */ 225 static struct swapdev *swapdrum_getsdp(int); 226 227 static struct swapdev *swaplist_find(struct vnode *, int); 228 static void swaplist_insert(struct swapdev *, 229 struct swappri *, int); 230 static void swaplist_trim(void); 231 232 static int swap_on(struct lwp *, struct swapdev *); 233 static int swap_off(struct lwp *, struct swapdev *); 234 235 static void uvm_swap_stats_locked(int, struct swapent *, int, register_t *); 236 237 static void sw_reg_strategy(struct swapdev *, struct buf *, int); 238 static void sw_reg_iodone(struct buf *); 239 static void sw_reg_start(struct swapdev *); 240 241 static int uvm_swap_io(struct vm_page **, int, int, int); 242 243 /* 244 * uvm_swap_init: init the swap system data structures and locks 245 * 246 * => called at boot time from init_main.c after the filesystems 247 * are brought up (which happens after uvm_init()) 248 */ 249 void 250 uvm_swap_init(void) 251 { 252 UVMHIST_FUNC("uvm_swap_init"); 253 254 UVMHIST_CALLED(pdhist); 255 /* 256 * first, init the swap list, its counter, and its lock. 257 * then get a handle on the vnode for /dev/drum by using 258 * the its dev_t number ("swapdev", from MD conf.c). 259 */ 260 261 LIST_INIT(&swap_priority); 262 uvmexp.nswapdev = 0; 263 lockinit(&swap_syscall_lock, PVM, "swapsys", 0, 0); 264 simple_lock_init(&uvm.swap_data_lock); 265 266 if (bdevvp(swapdev, &swapdev_vp)) 267 panic("uvm_swap_init: can't get vnode for swap device"); 268 269 /* 270 * create swap block resource map to map /dev/drum. the range 271 * from 1 to INT_MAX allows 2 gigablocks of swap space. note 272 * that block 0 is reserved (used to indicate an allocation 273 * failure, or no allocation). 274 */ 275 swapmap = extent_create("swapmap", 1, INT_MAX, 276 M_VMSWAP, 0, 0, EX_NOWAIT); 277 if (swapmap == 0) 278 panic("uvm_swap_init: extent_create failed"); 279 280 /* 281 * done! 282 */ 283 UVMHIST_LOG(pdhist, "<- done", 0, 0, 0, 0); 284 } 285 286 /* 287 * swaplist functions: functions that operate on the list of swap 288 * devices on the system. 289 */ 290 291 /* 292 * swaplist_insert: insert swap device "sdp" into the global list 293 * 294 * => caller must hold both swap_syscall_lock and uvm.swap_data_lock 295 * => caller must provide a newly malloc'd swappri structure (we will 296 * FREE it if we don't need it... this it to prevent malloc blocking 297 * here while adding swap) 298 */ 299 static void 300 swaplist_insert(struct swapdev *sdp, struct swappri *newspp, int priority) 301 { 302 struct swappri *spp, *pspp; 303 UVMHIST_FUNC("swaplist_insert"); UVMHIST_CALLED(pdhist); 304 305 /* 306 * find entry at or after which to insert the new device. 307 */ 308 pspp = NULL; 309 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 310 if (priority <= spp->spi_priority) 311 break; 312 pspp = spp; 313 } 314 315 /* 316 * new priority? 317 */ 318 if (spp == NULL || spp->spi_priority != priority) { 319 spp = newspp; /* use newspp! */ 320 UVMHIST_LOG(pdhist, "created new swappri = %d", 321 priority, 0, 0, 0); 322 323 spp->spi_priority = priority; 324 CIRCLEQ_INIT(&spp->spi_swapdev); 325 326 if (pspp) 327 LIST_INSERT_AFTER(pspp, spp, spi_swappri); 328 else 329 LIST_INSERT_HEAD(&swap_priority, spp, spi_swappri); 330 } else { 331 /* we don't need a new priority structure, free it */ 332 FREE(newspp, M_VMSWAP); 333 } 334 335 /* 336 * priority found (or created). now insert on the priority's 337 * circleq list and bump the total number of swapdevs. 338 */ 339 sdp->swd_priority = priority; 340 CIRCLEQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next); 341 uvmexp.nswapdev++; 342 } 343 344 /* 345 * swaplist_find: find and optionally remove a swap device from the 346 * global list. 347 * 348 * => caller must hold both swap_syscall_lock and uvm.swap_data_lock 349 * => we return the swapdev we found (and removed) 350 */ 351 static struct swapdev * 352 swaplist_find(struct vnode *vp, boolean_t remove) 353 { 354 struct swapdev *sdp; 355 struct swappri *spp; 356 357 /* 358 * search the lists for the requested vp 359 */ 360 361 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 362 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 363 if (sdp->swd_vp == vp) { 364 if (remove) { 365 CIRCLEQ_REMOVE(&spp->spi_swapdev, 366 sdp, swd_next); 367 uvmexp.nswapdev--; 368 } 369 return(sdp); 370 } 371 } 372 } 373 return (NULL); 374 } 375 376 377 /* 378 * swaplist_trim: scan priority list for empty priority entries and kill 379 * them. 380 * 381 * => caller must hold both swap_syscall_lock and uvm.swap_data_lock 382 */ 383 static void 384 swaplist_trim(void) 385 { 386 struct swappri *spp, *nextspp; 387 388 for (spp = LIST_FIRST(&swap_priority); spp != NULL; spp = nextspp) { 389 nextspp = LIST_NEXT(spp, spi_swappri); 390 if (CIRCLEQ_FIRST(&spp->spi_swapdev) != 391 (void *)&spp->spi_swapdev) 392 continue; 393 LIST_REMOVE(spp, spi_swappri); 394 free(spp, M_VMSWAP); 395 } 396 } 397 398 /* 399 * swapdrum_getsdp: given a page offset in /dev/drum, convert it back 400 * to the "swapdev" that maps that section of the drum. 401 * 402 * => each swapdev takes one big contig chunk of the drum 403 * => caller must hold uvm.swap_data_lock 404 */ 405 static struct swapdev * 406 swapdrum_getsdp(int pgno) 407 { 408 struct swapdev *sdp; 409 struct swappri *spp; 410 411 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 412 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 413 if (sdp->swd_flags & SWF_FAKE) 414 continue; 415 if (pgno >= sdp->swd_drumoffset && 416 pgno < (sdp->swd_drumoffset + sdp->swd_drumsize)) { 417 return sdp; 418 } 419 } 420 } 421 return NULL; 422 } 423 424 425 /* 426 * sys_swapctl: main entry point for swapctl(2) system call 427 * [with two helper functions: swap_on and swap_off] 428 */ 429 int 430 sys_swapctl(struct lwp *l, void *v, register_t *retval) 431 { 432 struct sys_swapctl_args /* { 433 syscallarg(int) cmd; 434 syscallarg(void *) arg; 435 syscallarg(int) misc; 436 } */ *uap = (struct sys_swapctl_args *)v; 437 struct vnode *vp; 438 struct nameidata nd; 439 struct swappri *spp; 440 struct swapdev *sdp; 441 struct swapent *sep; 442 #define SWAP_PATH_MAX (PATH_MAX + 1) 443 char *userpath; 444 size_t len; 445 int error, misc; 446 int priority; 447 UVMHIST_FUNC("sys_swapctl"); UVMHIST_CALLED(pdhist); 448 449 misc = SCARG(uap, misc); 450 451 /* 452 * ensure serialized syscall access by grabbing the swap_syscall_lock 453 */ 454 lockmgr(&swap_syscall_lock, LK_EXCLUSIVE, NULL); 455 456 userpath = malloc(SWAP_PATH_MAX, M_TEMP, M_WAITOK); 457 /* 458 * we handle the non-priv NSWAP and STATS request first. 459 * 460 * SWAP_NSWAP: return number of config'd swap devices 461 * [can also be obtained with uvmexp sysctl] 462 */ 463 if (SCARG(uap, cmd) == SWAP_NSWAP) { 464 UVMHIST_LOG(pdhist, "<- done SWAP_NSWAP=%d", uvmexp.nswapdev, 465 0, 0, 0); 466 *retval = uvmexp.nswapdev; 467 error = 0; 468 goto out; 469 } 470 471 /* 472 * SWAP_STATS: get stats on current # of configured swap devs 473 * 474 * note that the swap_priority list can't change as long 475 * as we are holding the swap_syscall_lock. we don't want 476 * to grab the uvm.swap_data_lock because we may fault&sleep during 477 * copyout() and we don't want to be holding that lock then! 478 */ 479 if (SCARG(uap, cmd) == SWAP_STATS 480 #if defined(COMPAT_13) 481 || SCARG(uap, cmd) == SWAP_OSTATS 482 #endif 483 ) { 484 if ((size_t)misc > (size_t)uvmexp.nswapdev) 485 misc = uvmexp.nswapdev; 486 #if defined(COMPAT_13) 487 if (SCARG(uap, cmd) == SWAP_OSTATS) 488 len = sizeof(struct oswapent) * misc; 489 else 490 #endif 491 len = sizeof(struct swapent) * misc; 492 sep = (struct swapent *)malloc(len, M_TEMP, M_WAITOK); 493 494 uvm_swap_stats_locked(SCARG(uap, cmd), sep, misc, retval); 495 error = copyout(sep, SCARG(uap, arg), len); 496 497 free(sep, M_TEMP); 498 UVMHIST_LOG(pdhist, "<- done SWAP_STATS", 0, 0, 0, 0); 499 goto out; 500 } 501 if (SCARG(uap, cmd) == SWAP_GETDUMPDEV) { 502 dev_t *devp = (dev_t *)SCARG(uap, arg); 503 504 error = copyout(&dumpdev, devp, sizeof(dumpdev)); 505 goto out; 506 } 507 508 /* 509 * all other requests require superuser privs. verify. 510 */ 511 if ((error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_SWAPCTL, 512 0, NULL, NULL, NULL))) 513 goto out; 514 515 if (SCARG(uap, cmd) == SWAP_DUMPOFF) { 516 /* drop the current dump device */ 517 dumpdev = NODEV; 518 cpu_dumpconf(); 519 goto out; 520 } 521 522 /* 523 * at this point we expect a path name in arg. we will 524 * use namei() to gain a vnode reference (vref), and lock 525 * the vnode (VOP_LOCK). 526 * 527 * XXX: a NULL arg means use the root vnode pointer (e.g. for 528 * miniroot) 529 */ 530 if (SCARG(uap, arg) == NULL) { 531 vp = rootvp; /* miniroot */ 532 if (vget(vp, LK_EXCLUSIVE)) { 533 error = EBUSY; 534 goto out; 535 } 536 if (SCARG(uap, cmd) == SWAP_ON && 537 copystr("miniroot", userpath, SWAP_PATH_MAX, &len)) 538 panic("swapctl: miniroot copy failed"); 539 } else { 540 int space; 541 char *where; 542 543 if (SCARG(uap, cmd) == SWAP_ON) { 544 if ((error = copyinstr(SCARG(uap, arg), userpath, 545 SWAP_PATH_MAX, &len))) 546 goto out; 547 space = UIO_SYSSPACE; 548 where = userpath; 549 } else { 550 space = UIO_USERSPACE; 551 where = (char *)SCARG(uap, arg); 552 } 553 NDINIT(&nd, LOOKUP, FOLLOW|LOCKLEAF, space, where, l); 554 if ((error = namei(&nd))) 555 goto out; 556 vp = nd.ni_vp; 557 } 558 /* note: "vp" is referenced and locked */ 559 560 error = 0; /* assume no error */ 561 switch(SCARG(uap, cmd)) { 562 563 case SWAP_DUMPDEV: 564 if (vp->v_type != VBLK) { 565 error = ENOTBLK; 566 break; 567 } 568 dumpdev = vp->v_rdev; 569 cpu_dumpconf(); 570 break; 571 572 case SWAP_CTL: 573 /* 574 * get new priority, remove old entry (if any) and then 575 * reinsert it in the correct place. finally, prune out 576 * any empty priority structures. 577 */ 578 priority = SCARG(uap, misc); 579 spp = malloc(sizeof *spp, M_VMSWAP, M_WAITOK); 580 simple_lock(&uvm.swap_data_lock); 581 if ((sdp = swaplist_find(vp, 1)) == NULL) { 582 error = ENOENT; 583 } else { 584 swaplist_insert(sdp, spp, priority); 585 swaplist_trim(); 586 } 587 simple_unlock(&uvm.swap_data_lock); 588 if (error) 589 free(spp, M_VMSWAP); 590 break; 591 592 case SWAP_ON: 593 594 /* 595 * check for duplicates. if none found, then insert a 596 * dummy entry on the list to prevent someone else from 597 * trying to enable this device while we are working on 598 * it. 599 */ 600 601 priority = SCARG(uap, misc); 602 sdp = malloc(sizeof *sdp, M_VMSWAP, M_WAITOK); 603 spp = malloc(sizeof *spp, M_VMSWAP, M_WAITOK); 604 memset(sdp, 0, sizeof(*sdp)); 605 sdp->swd_flags = SWF_FAKE; 606 sdp->swd_vp = vp; 607 sdp->swd_dev = (vp->v_type == VBLK) ? vp->v_rdev : NODEV; 608 bufq_alloc(&sdp->swd_tab, "disksort", BUFQ_SORT_RAWBLOCK); 609 simple_lock(&uvm.swap_data_lock); 610 if (swaplist_find(vp, 0) != NULL) { 611 error = EBUSY; 612 simple_unlock(&uvm.swap_data_lock); 613 bufq_free(sdp->swd_tab); 614 free(sdp, M_VMSWAP); 615 free(spp, M_VMSWAP); 616 break; 617 } 618 swaplist_insert(sdp, spp, priority); 619 simple_unlock(&uvm.swap_data_lock); 620 621 sdp->swd_pathlen = len; 622 sdp->swd_path = malloc(sdp->swd_pathlen, M_VMSWAP, M_WAITOK); 623 if (copystr(userpath, sdp->swd_path, sdp->swd_pathlen, 0) != 0) 624 panic("swapctl: copystr"); 625 626 /* 627 * we've now got a FAKE placeholder in the swap list. 628 * now attempt to enable swap on it. if we fail, undo 629 * what we've done and kill the fake entry we just inserted. 630 * if swap_on is a success, it will clear the SWF_FAKE flag 631 */ 632 633 if ((error = swap_on(l, sdp)) != 0) { 634 simple_lock(&uvm.swap_data_lock); 635 (void) swaplist_find(vp, 1); /* kill fake entry */ 636 swaplist_trim(); 637 simple_unlock(&uvm.swap_data_lock); 638 bufq_free(sdp->swd_tab); 639 free(sdp->swd_path, M_VMSWAP); 640 free(sdp, M_VMSWAP); 641 break; 642 } 643 break; 644 645 case SWAP_OFF: 646 simple_lock(&uvm.swap_data_lock); 647 if ((sdp = swaplist_find(vp, 0)) == NULL) { 648 simple_unlock(&uvm.swap_data_lock); 649 error = ENXIO; 650 break; 651 } 652 653 /* 654 * If a device isn't in use or enabled, we 655 * can't stop swapping from it (again). 656 */ 657 if ((sdp->swd_flags & (SWF_INUSE|SWF_ENABLE)) == 0) { 658 simple_unlock(&uvm.swap_data_lock); 659 error = EBUSY; 660 break; 661 } 662 663 /* 664 * do the real work. 665 */ 666 error = swap_off(l, sdp); 667 break; 668 669 default: 670 error = EINVAL; 671 } 672 673 /* 674 * done! release the ref gained by namei() and unlock. 675 */ 676 vput(vp); 677 678 out: 679 free(userpath, M_TEMP); 680 lockmgr(&swap_syscall_lock, LK_RELEASE, NULL); 681 682 UVMHIST_LOG(pdhist, "<- done! error=%d", error, 0, 0, 0); 683 return (error); 684 } 685 686 /* 687 * swap_stats: implements swapctl(SWAP_STATS). The function is kept 688 * away from sys_swapctl() in order to allow COMPAT_* swapctl() 689 * emulation to use it directly without going through sys_swapctl(). 690 * The problem with using sys_swapctl() there is that it involves 691 * copying the swapent array to the stackgap, and this array's size 692 * is not known at build time. Hence it would not be possible to 693 * ensure it would fit in the stackgap in any case. 694 */ 695 void 696 uvm_swap_stats(int cmd, struct swapent *sep, int sec, register_t *retval) 697 { 698 699 lockmgr(&swap_syscall_lock, LK_EXCLUSIVE, NULL); 700 uvm_swap_stats_locked(cmd, sep, sec, retval); 701 lockmgr(&swap_syscall_lock, LK_RELEASE, NULL); 702 } 703 704 static void 705 uvm_swap_stats_locked(int cmd, struct swapent *sep, int sec, register_t *retval) 706 { 707 struct swappri *spp; 708 struct swapdev *sdp; 709 int count = 0; 710 711 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 712 for (sdp = CIRCLEQ_FIRST(&spp->spi_swapdev); 713 sdp != (void *)&spp->spi_swapdev && sec-- > 0; 714 sdp = CIRCLEQ_NEXT(sdp, swd_next)) { 715 /* 716 * backwards compatibility for system call. 717 * note that we use 'struct oswapent' as an 718 * overlay into both 'struct swapdev' and 719 * the userland 'struct swapent', as we 720 * want to retain backwards compatibility 721 * with NetBSD 1.3. 722 */ 723 sdp->swd_ose.ose_inuse = 724 btodb((uint64_t)sdp->swd_npginuse << 725 PAGE_SHIFT); 726 (void)memcpy(sep, &sdp->swd_ose, 727 sizeof(struct oswapent)); 728 729 /* now copy out the path if necessary */ 730 #if !defined(COMPAT_13) 731 (void) cmd; 732 #endif 733 #if defined(COMPAT_13) 734 if (cmd == SWAP_STATS) 735 #endif 736 (void)memcpy(&sep->se_path, sdp->swd_path, 737 sdp->swd_pathlen); 738 739 count++; 740 #if defined(COMPAT_13) 741 if (cmd == SWAP_OSTATS) 742 sep = (struct swapent *) 743 ((struct oswapent *)sep + 1); 744 else 745 #endif 746 sep++; 747 } 748 } 749 750 *retval = count; 751 return; 752 } 753 754 /* 755 * swap_on: attempt to enable a swapdev for swapping. note that the 756 * swapdev is already on the global list, but disabled (marked 757 * SWF_FAKE). 758 * 759 * => we avoid the start of the disk (to protect disk labels) 760 * => we also avoid the miniroot, if we are swapping to root. 761 * => caller should leave uvm.swap_data_lock unlocked, we may lock it 762 * if needed. 763 */ 764 static int 765 swap_on(struct lwp *l, struct swapdev *sdp) 766 { 767 struct vnode *vp; 768 int error, npages, nblocks, size; 769 long addr; 770 u_long result; 771 struct vattr va; 772 #ifdef NFS 773 extern int (**nfsv2_vnodeop_p)(void *); 774 #endif /* NFS */ 775 const struct bdevsw *bdev; 776 dev_t dev; 777 UVMHIST_FUNC("swap_on"); UVMHIST_CALLED(pdhist); 778 779 /* 780 * we want to enable swapping on sdp. the swd_vp contains 781 * the vnode we want (locked and ref'd), and the swd_dev 782 * contains the dev_t of the file, if it a block device. 783 */ 784 785 vp = sdp->swd_vp; 786 dev = sdp->swd_dev; 787 788 /* 789 * open the swap file (mostly useful for block device files to 790 * let device driver know what is up). 791 * 792 * we skip the open/close for root on swap because the root 793 * has already been opened when root was mounted (mountroot). 794 */ 795 if (vp != rootvp) { 796 if ((error = VOP_OPEN(vp, FREAD|FWRITE, l->l_cred, l))) 797 return (error); 798 } 799 800 /* XXX this only works for block devices */ 801 UVMHIST_LOG(pdhist, " dev=%d, major(dev)=%d", dev, major(dev), 0,0); 802 803 /* 804 * we now need to determine the size of the swap area. for 805 * block specials we can call the d_psize function. 806 * for normal files, we must stat [get attrs]. 807 * 808 * we put the result in nblks. 809 * for normal files, we also want the filesystem block size 810 * (which we get with statfs). 811 */ 812 switch (vp->v_type) { 813 case VBLK: 814 bdev = bdevsw_lookup(dev); 815 if (bdev == NULL || bdev->d_psize == NULL || 816 (nblocks = (*bdev->d_psize)(dev)) == -1) { 817 error = ENXIO; 818 goto bad; 819 } 820 break; 821 822 case VREG: 823 if ((error = VOP_GETATTR(vp, &va, l->l_cred, l))) 824 goto bad; 825 nblocks = (int)btodb(va.va_size); 826 if ((error = 827 VFS_STATVFS(vp->v_mount, &vp->v_mount->mnt_stat, l)) != 0) 828 goto bad; 829 830 sdp->swd_bsize = vp->v_mount->mnt_stat.f_iosize; 831 /* 832 * limit the max # of outstanding I/O requests we issue 833 * at any one time. take it easy on NFS servers. 834 */ 835 #ifdef NFS 836 if (vp->v_op == nfsv2_vnodeop_p) 837 sdp->swd_maxactive = 2; /* XXX */ 838 else 839 #endif /* NFS */ 840 sdp->swd_maxactive = 8; /* XXX */ 841 break; 842 843 default: 844 error = ENXIO; 845 goto bad; 846 } 847 848 /* 849 * save nblocks in a safe place and convert to pages. 850 */ 851 852 sdp->swd_ose.ose_nblks = nblocks; 853 npages = dbtob((uint64_t)nblocks) >> PAGE_SHIFT; 854 855 /* 856 * for block special files, we want to make sure that leave 857 * the disklabel and bootblocks alone, so we arrange to skip 858 * over them (arbitrarily choosing to skip PAGE_SIZE bytes). 859 * note that because of this the "size" can be less than the 860 * actual number of blocks on the device. 861 */ 862 if (vp->v_type == VBLK) { 863 /* we use pages 1 to (size - 1) [inclusive] */ 864 size = npages - 1; 865 addr = 1; 866 } else { 867 /* we use pages 0 to (size - 1) [inclusive] */ 868 size = npages; 869 addr = 0; 870 } 871 872 /* 873 * make sure we have enough blocks for a reasonable sized swap 874 * area. we want at least one page. 875 */ 876 877 if (size < 1) { 878 UVMHIST_LOG(pdhist, " size <= 1!!", 0, 0, 0, 0); 879 error = EINVAL; 880 goto bad; 881 } 882 883 UVMHIST_LOG(pdhist, " dev=%x: size=%d addr=%ld\n", dev, size, addr, 0); 884 885 /* 886 * now we need to allocate an extent to manage this swap device 887 */ 888 889 sdp->swd_blist = blist_create(npages); 890 /* mark all expect the `saved' region free. */ 891 blist_free(sdp->swd_blist, addr, size); 892 893 /* 894 * if the vnode we are swapping to is the root vnode 895 * (i.e. we are swapping to the miniroot) then we want 896 * to make sure we don't overwrite it. do a statfs to 897 * find its size and skip over it. 898 */ 899 if (vp == rootvp) { 900 struct mount *mp; 901 struct statvfs *sp; 902 int rootblocks, rootpages; 903 904 mp = rootvnode->v_mount; 905 sp = &mp->mnt_stat; 906 rootblocks = sp->f_blocks * btodb(sp->f_frsize); 907 /* 908 * XXX: sp->f_blocks isn't the total number of 909 * blocks in the filesystem, it's the number of 910 * data blocks. so, our rootblocks almost 911 * definitely underestimates the total size 912 * of the filesystem - how badly depends on the 913 * details of the filesystem type. there isn't 914 * an obvious way to deal with this cleanly 915 * and perfectly, so for now we just pad our 916 * rootblocks estimate with an extra 5 percent. 917 */ 918 rootblocks += (rootblocks >> 5) + 919 (rootblocks >> 6) + 920 (rootblocks >> 7); 921 rootpages = round_page(dbtob(rootblocks)) >> PAGE_SHIFT; 922 if (rootpages > size) 923 panic("swap_on: miniroot larger than swap?"); 924 925 if (rootpages != blist_fill(sdp->swd_blist, addr, rootpages)) { 926 panic("swap_on: unable to preserve miniroot"); 927 } 928 929 size -= rootpages; 930 printf("Preserved %d pages of miniroot ", rootpages); 931 printf("leaving %d pages of swap\n", size); 932 } 933 934 /* 935 * add a ref to vp to reflect usage as a swap device. 936 */ 937 vref(vp); 938 939 /* 940 * now add the new swapdev to the drum and enable. 941 */ 942 if (extent_alloc(swapmap, npages, EX_NOALIGN, EX_NOBOUNDARY, 943 EX_WAITOK, &result)) 944 panic("swapdrum_add"); 945 946 sdp->swd_drumoffset = (int)result; 947 sdp->swd_drumsize = npages; 948 sdp->swd_npages = size; 949 simple_lock(&uvm.swap_data_lock); 950 sdp->swd_flags &= ~SWF_FAKE; /* going live */ 951 sdp->swd_flags |= (SWF_INUSE|SWF_ENABLE); 952 uvmexp.swpages += size; 953 uvmexp.swpgavail += size; 954 simple_unlock(&uvm.swap_data_lock); 955 return (0); 956 957 /* 958 * failure: clean up and return error. 959 */ 960 961 bad: 962 if (sdp->swd_blist) { 963 blist_destroy(sdp->swd_blist); 964 } 965 if (vp != rootvp) { 966 (void)VOP_CLOSE(vp, FREAD|FWRITE, l->l_cred, l); 967 } 968 return (error); 969 } 970 971 /* 972 * swap_off: stop swapping on swapdev 973 * 974 * => swap data should be locked, we will unlock. 975 */ 976 static int 977 swap_off(struct lwp *l, struct swapdev *sdp) 978 { 979 int npages = sdp->swd_npages; 980 int error = 0; 981 982 UVMHIST_FUNC("swap_off"); UVMHIST_CALLED(pdhist); 983 UVMHIST_LOG(pdhist, " dev=%x, npages=%d", sdp->swd_dev,npages,0,0); 984 985 /* disable the swap area being removed */ 986 sdp->swd_flags &= ~SWF_ENABLE; 987 uvmexp.swpgavail -= npages; 988 simple_unlock(&uvm.swap_data_lock); 989 990 /* 991 * the idea is to find all the pages that are paged out to this 992 * device, and page them all in. in uvm, swap-backed pageable 993 * memory can take two forms: aobjs and anons. call the 994 * swapoff hook for each subsystem to bring in pages. 995 */ 996 997 if (uao_swap_off(sdp->swd_drumoffset, 998 sdp->swd_drumoffset + sdp->swd_drumsize) || 999 amap_swap_off(sdp->swd_drumoffset, 1000 sdp->swd_drumoffset + sdp->swd_drumsize)) { 1001 error = ENOMEM; 1002 } else if (sdp->swd_npginuse > sdp->swd_npgbad) { 1003 error = EBUSY; 1004 } 1005 1006 if (error) { 1007 simple_lock(&uvm.swap_data_lock); 1008 sdp->swd_flags |= SWF_ENABLE; 1009 uvmexp.swpgavail += npages; 1010 simple_unlock(&uvm.swap_data_lock); 1011 1012 return error; 1013 } 1014 1015 /* 1016 * done with the vnode. 1017 * drop our ref on the vnode before calling VOP_CLOSE() 1018 * so that spec_close() can tell if this is the last close. 1019 */ 1020 vrele(sdp->swd_vp); 1021 if (sdp->swd_vp != rootvp) { 1022 (void) VOP_CLOSE(sdp->swd_vp, FREAD|FWRITE, l->l_cred, l); 1023 } 1024 1025 simple_lock(&uvm.swap_data_lock); 1026 uvmexp.swpages -= npages; 1027 uvmexp.swpginuse -= sdp->swd_npgbad; 1028 1029 if (swaplist_find(sdp->swd_vp, 1) == NULL) 1030 panic("swap_off: swapdev not in list"); 1031 swaplist_trim(); 1032 simple_unlock(&uvm.swap_data_lock); 1033 1034 /* 1035 * free all resources! 1036 */ 1037 extent_free(swapmap, sdp->swd_drumoffset, sdp->swd_drumsize, 1038 EX_WAITOK); 1039 blist_destroy(sdp->swd_blist); 1040 bufq_free(sdp->swd_tab); 1041 free(sdp, M_VMSWAP); 1042 return (0); 1043 } 1044 1045 /* 1046 * /dev/drum interface and i/o functions 1047 */ 1048 1049 /* 1050 * swstrategy: perform I/O on the drum 1051 * 1052 * => we must map the i/o request from the drum to the correct swapdev. 1053 */ 1054 static void 1055 swstrategy(struct buf *bp) 1056 { 1057 struct swapdev *sdp; 1058 struct vnode *vp; 1059 int s, pageno, bn; 1060 UVMHIST_FUNC("swstrategy"); UVMHIST_CALLED(pdhist); 1061 1062 /* 1063 * convert block number to swapdev. note that swapdev can't 1064 * be yanked out from under us because we are holding resources 1065 * in it (i.e. the blocks we are doing I/O on). 1066 */ 1067 pageno = dbtob((int64_t)bp->b_blkno) >> PAGE_SHIFT; 1068 simple_lock(&uvm.swap_data_lock); 1069 sdp = swapdrum_getsdp(pageno); 1070 simple_unlock(&uvm.swap_data_lock); 1071 if (sdp == NULL) { 1072 bp->b_error = EINVAL; 1073 bp->b_flags |= B_ERROR; 1074 biodone(bp); 1075 UVMHIST_LOG(pdhist, " failed to get swap device", 0, 0, 0, 0); 1076 return; 1077 } 1078 1079 /* 1080 * convert drum page number to block number on this swapdev. 1081 */ 1082 1083 pageno -= sdp->swd_drumoffset; /* page # on swapdev */ 1084 bn = btodb((uint64_t)pageno << PAGE_SHIFT); /* convert to diskblock */ 1085 1086 UVMHIST_LOG(pdhist, " %s: mapoff=%x bn=%x bcount=%ld", 1087 ((bp->b_flags & B_READ) == 0) ? "write" : "read", 1088 sdp->swd_drumoffset, bn, bp->b_bcount); 1089 1090 /* 1091 * for block devices we finish up here. 1092 * for regular files we have to do more work which we delegate 1093 * to sw_reg_strategy(). 1094 */ 1095 1096 switch (sdp->swd_vp->v_type) { 1097 default: 1098 panic("swstrategy: vnode type 0x%x", sdp->swd_vp->v_type); 1099 1100 case VBLK: 1101 1102 /* 1103 * must convert "bp" from an I/O on /dev/drum to an I/O 1104 * on the swapdev (sdp). 1105 */ 1106 s = splbio(); 1107 bp->b_blkno = bn; /* swapdev block number */ 1108 vp = sdp->swd_vp; /* swapdev vnode pointer */ 1109 bp->b_dev = sdp->swd_dev; /* swapdev dev_t */ 1110 1111 /* 1112 * if we are doing a write, we have to redirect the i/o on 1113 * drum's v_numoutput counter to the swapdevs. 1114 */ 1115 if ((bp->b_flags & B_READ) == 0) { 1116 vwakeup(bp); /* kills one 'v_numoutput' on drum */ 1117 V_INCR_NUMOUTPUT(vp); /* put it on swapdev */ 1118 } 1119 1120 /* 1121 * finally plug in swapdev vnode and start I/O 1122 */ 1123 bp->b_vp = vp; 1124 splx(s); 1125 VOP_STRATEGY(vp, bp); 1126 return; 1127 1128 case VREG: 1129 /* 1130 * delegate to sw_reg_strategy function. 1131 */ 1132 sw_reg_strategy(sdp, bp, bn); 1133 return; 1134 } 1135 /* NOTREACHED */ 1136 } 1137 1138 /* 1139 * swread: the read function for the drum (just a call to physio) 1140 */ 1141 /*ARGSUSED*/ 1142 static int 1143 swread(dev_t dev, struct uio *uio, int ioflag __unused) 1144 { 1145 UVMHIST_FUNC("swread"); UVMHIST_CALLED(pdhist); 1146 1147 UVMHIST_LOG(pdhist, " dev=%x offset=%qx", dev, uio->uio_offset, 0, 0); 1148 return (physio(swstrategy, NULL, dev, B_READ, minphys, uio)); 1149 } 1150 1151 /* 1152 * swwrite: the write function for the drum (just a call to physio) 1153 */ 1154 /*ARGSUSED*/ 1155 static int 1156 swwrite(dev_t dev, struct uio *uio, int ioflag __unused) 1157 { 1158 UVMHIST_FUNC("swwrite"); UVMHIST_CALLED(pdhist); 1159 1160 UVMHIST_LOG(pdhist, " dev=%x offset=%qx", dev, uio->uio_offset, 0, 0); 1161 return (physio(swstrategy, NULL, dev, B_WRITE, minphys, uio)); 1162 } 1163 1164 const struct bdevsw swap_bdevsw = { 1165 noopen, noclose, swstrategy, noioctl, nodump, nosize, D_OTHER, 1166 }; 1167 1168 const struct cdevsw swap_cdevsw = { 1169 nullopen, nullclose, swread, swwrite, noioctl, 1170 nostop, notty, nopoll, nommap, nokqfilter, D_OTHER, 1171 }; 1172 1173 /* 1174 * sw_reg_strategy: handle swap i/o to regular files 1175 */ 1176 static void 1177 sw_reg_strategy(struct swapdev *sdp, struct buf *bp, int bn) 1178 { 1179 struct vnode *vp; 1180 struct vndxfer *vnx; 1181 daddr_t nbn; 1182 caddr_t addr; 1183 off_t byteoff; 1184 int s, off, nra, error, sz, resid; 1185 UVMHIST_FUNC("sw_reg_strategy"); UVMHIST_CALLED(pdhist); 1186 1187 /* 1188 * allocate a vndxfer head for this transfer and point it to 1189 * our buffer. 1190 */ 1191 getvndxfer(vnx); 1192 vnx->vx_flags = VX_BUSY; 1193 vnx->vx_error = 0; 1194 vnx->vx_pending = 0; 1195 vnx->vx_bp = bp; 1196 vnx->vx_sdp = sdp; 1197 1198 /* 1199 * setup for main loop where we read filesystem blocks into 1200 * our buffer. 1201 */ 1202 error = 0; 1203 bp->b_resid = bp->b_bcount; /* nothing transfered yet! */ 1204 addr = bp->b_data; /* current position in buffer */ 1205 byteoff = dbtob((uint64_t)bn); 1206 1207 for (resid = bp->b_resid; resid; resid -= sz) { 1208 struct vndbuf *nbp; 1209 1210 /* 1211 * translate byteoffset into block number. return values: 1212 * vp = vnode of underlying device 1213 * nbn = new block number (on underlying vnode dev) 1214 * nra = num blocks we can read-ahead (excludes requested 1215 * block) 1216 */ 1217 nra = 0; 1218 error = VOP_BMAP(sdp->swd_vp, byteoff / sdp->swd_bsize, 1219 &vp, &nbn, &nra); 1220 1221 if (error == 0 && nbn == (daddr_t)-1) { 1222 /* 1223 * this used to just set error, but that doesn't 1224 * do the right thing. Instead, it causes random 1225 * memory errors. The panic() should remain until 1226 * this condition doesn't destabilize the system. 1227 */ 1228 #if 1 1229 panic("sw_reg_strategy: swap to sparse file"); 1230 #else 1231 error = EIO; /* failure */ 1232 #endif 1233 } 1234 1235 /* 1236 * punt if there was an error or a hole in the file. 1237 * we must wait for any i/o ops we have already started 1238 * to finish before returning. 1239 * 1240 * XXX we could deal with holes here but it would be 1241 * a hassle (in the write case). 1242 */ 1243 if (error) { 1244 s = splbio(); 1245 vnx->vx_error = error; /* pass error up */ 1246 goto out; 1247 } 1248 1249 /* 1250 * compute the size ("sz") of this transfer (in bytes). 1251 */ 1252 off = byteoff % sdp->swd_bsize; 1253 sz = (1 + nra) * sdp->swd_bsize - off; 1254 if (sz > resid) 1255 sz = resid; 1256 1257 UVMHIST_LOG(pdhist, "sw_reg_strategy: " 1258 "vp %p/%p offset 0x%x/0x%x", 1259 sdp->swd_vp, vp, byteoff, nbn); 1260 1261 /* 1262 * now get a buf structure. note that the vb_buf is 1263 * at the front of the nbp structure so that you can 1264 * cast pointers between the two structure easily. 1265 */ 1266 getvndbuf(nbp); 1267 BUF_INIT(&nbp->vb_buf); 1268 nbp->vb_buf.b_flags = bp->b_flags | B_CALL; 1269 nbp->vb_buf.b_bcount = sz; 1270 nbp->vb_buf.b_bufsize = sz; 1271 nbp->vb_buf.b_error = 0; 1272 nbp->vb_buf.b_data = addr; 1273 nbp->vb_buf.b_lblkno = 0; 1274 nbp->vb_buf.b_blkno = nbn + btodb(off); 1275 nbp->vb_buf.b_rawblkno = nbp->vb_buf.b_blkno; 1276 nbp->vb_buf.b_iodone = sw_reg_iodone; 1277 nbp->vb_buf.b_vp = vp; 1278 if (vp->v_type == VBLK) { 1279 nbp->vb_buf.b_dev = vp->v_rdev; 1280 } 1281 1282 nbp->vb_xfer = vnx; /* patch it back in to vnx */ 1283 1284 /* 1285 * Just sort by block number 1286 */ 1287 s = splbio(); 1288 if (vnx->vx_error != 0) { 1289 putvndbuf(nbp); 1290 goto out; 1291 } 1292 vnx->vx_pending++; 1293 1294 /* sort it in and start I/O if we are not over our limit */ 1295 BUFQ_PUT(sdp->swd_tab, &nbp->vb_buf); 1296 sw_reg_start(sdp); 1297 splx(s); 1298 1299 /* 1300 * advance to the next I/O 1301 */ 1302 byteoff += sz; 1303 addr += sz; 1304 } 1305 1306 s = splbio(); 1307 1308 out: /* Arrive here at splbio */ 1309 vnx->vx_flags &= ~VX_BUSY; 1310 if (vnx->vx_pending == 0) { 1311 if (vnx->vx_error != 0) { 1312 bp->b_error = vnx->vx_error; 1313 bp->b_flags |= B_ERROR; 1314 } 1315 putvndxfer(vnx); 1316 biodone(bp); 1317 } 1318 splx(s); 1319 } 1320 1321 /* 1322 * sw_reg_start: start an I/O request on the requested swapdev 1323 * 1324 * => reqs are sorted by b_rawblkno (above) 1325 */ 1326 static void 1327 sw_reg_start(struct swapdev *sdp) 1328 { 1329 struct buf *bp; 1330 UVMHIST_FUNC("sw_reg_start"); UVMHIST_CALLED(pdhist); 1331 1332 /* recursion control */ 1333 if ((sdp->swd_flags & SWF_BUSY) != 0) 1334 return; 1335 1336 sdp->swd_flags |= SWF_BUSY; 1337 1338 while (sdp->swd_active < sdp->swd_maxactive) { 1339 bp = BUFQ_GET(sdp->swd_tab); 1340 if (bp == NULL) 1341 break; 1342 sdp->swd_active++; 1343 1344 UVMHIST_LOG(pdhist, 1345 "sw_reg_start: bp %p vp %p blkno %p cnt %lx", 1346 bp, bp->b_vp, bp->b_blkno, bp->b_bcount); 1347 if ((bp->b_flags & B_READ) == 0) 1348 V_INCR_NUMOUTPUT(bp->b_vp); 1349 1350 VOP_STRATEGY(bp->b_vp, bp); 1351 } 1352 sdp->swd_flags &= ~SWF_BUSY; 1353 } 1354 1355 /* 1356 * sw_reg_iodone: one of our i/o's has completed and needs post-i/o cleanup 1357 * 1358 * => note that we can recover the vndbuf struct by casting the buf ptr 1359 */ 1360 static void 1361 sw_reg_iodone(struct buf *bp) 1362 { 1363 struct vndbuf *vbp = (struct vndbuf *) bp; 1364 struct vndxfer *vnx = vbp->vb_xfer; 1365 struct buf *pbp = vnx->vx_bp; /* parent buffer */ 1366 struct swapdev *sdp = vnx->vx_sdp; 1367 int s, resid, error; 1368 UVMHIST_FUNC("sw_reg_iodone"); UVMHIST_CALLED(pdhist); 1369 1370 UVMHIST_LOG(pdhist, " vbp=%p vp=%p blkno=%x addr=%p", 1371 vbp, vbp->vb_buf.b_vp, vbp->vb_buf.b_blkno, vbp->vb_buf.b_data); 1372 UVMHIST_LOG(pdhist, " cnt=%lx resid=%lx", 1373 vbp->vb_buf.b_bcount, vbp->vb_buf.b_resid, 0, 0); 1374 1375 /* 1376 * protect vbp at splbio and update. 1377 */ 1378 1379 s = splbio(); 1380 resid = vbp->vb_buf.b_bcount - vbp->vb_buf.b_resid; 1381 pbp->b_resid -= resid; 1382 vnx->vx_pending--; 1383 1384 if (vbp->vb_buf.b_flags & B_ERROR) { 1385 /* pass error upward */ 1386 error = vbp->vb_buf.b_error ? vbp->vb_buf.b_error : EIO; 1387 UVMHIST_LOG(pdhist, " got error=%d !", error, 0, 0, 0); 1388 vnx->vx_error = error; 1389 } 1390 1391 /* 1392 * kill vbp structure 1393 */ 1394 putvndbuf(vbp); 1395 1396 /* 1397 * wrap up this transaction if it has run to completion or, in 1398 * case of an error, when all auxiliary buffers have returned. 1399 */ 1400 if (vnx->vx_error != 0) { 1401 /* pass error upward */ 1402 pbp->b_flags |= B_ERROR; 1403 pbp->b_error = vnx->vx_error; 1404 if ((vnx->vx_flags & VX_BUSY) == 0 && vnx->vx_pending == 0) { 1405 putvndxfer(vnx); 1406 biodone(pbp); 1407 } 1408 } else if (pbp->b_resid == 0) { 1409 KASSERT(vnx->vx_pending == 0); 1410 if ((vnx->vx_flags & VX_BUSY) == 0) { 1411 UVMHIST_LOG(pdhist, " iodone error=%d !", 1412 pbp, vnx->vx_error, 0, 0); 1413 putvndxfer(vnx); 1414 biodone(pbp); 1415 } 1416 } 1417 1418 /* 1419 * done! start next swapdev I/O if one is pending 1420 */ 1421 sdp->swd_active--; 1422 sw_reg_start(sdp); 1423 splx(s); 1424 } 1425 1426 1427 /* 1428 * uvm_swap_alloc: allocate space on swap 1429 * 1430 * => allocation is done "round robin" down the priority list, as we 1431 * allocate in a priority we "rotate" the circle queue. 1432 * => space can be freed with uvm_swap_free 1433 * => we return the page slot number in /dev/drum (0 == invalid slot) 1434 * => we lock uvm.swap_data_lock 1435 * => XXXMRG: "LESSOK" INTERFACE NEEDED TO EXTENT SYSTEM 1436 */ 1437 int 1438 uvm_swap_alloc(int *nslots /* IN/OUT */, boolean_t lessok) 1439 { 1440 struct swapdev *sdp; 1441 struct swappri *spp; 1442 UVMHIST_FUNC("uvm_swap_alloc"); UVMHIST_CALLED(pdhist); 1443 1444 /* 1445 * no swap devices configured yet? definite failure. 1446 */ 1447 if (uvmexp.nswapdev < 1) 1448 return 0; 1449 1450 /* 1451 * lock data lock, convert slots into blocks, and enter loop 1452 */ 1453 simple_lock(&uvm.swap_data_lock); 1454 1455 ReTry: /* XXXMRG */ 1456 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 1457 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 1458 uint64_t result; 1459 1460 /* if it's not enabled, then we can't swap from it */ 1461 if ((sdp->swd_flags & SWF_ENABLE) == 0) 1462 continue; 1463 if (sdp->swd_npginuse + *nslots > sdp->swd_npages) 1464 continue; 1465 result = blist_alloc(sdp->swd_blist, *nslots); 1466 if (result == BLIST_NONE) { 1467 continue; 1468 } 1469 KASSERT(result < sdp->swd_drumsize); 1470 1471 /* 1472 * successful allocation! now rotate the circleq. 1473 */ 1474 CIRCLEQ_REMOVE(&spp->spi_swapdev, sdp, swd_next); 1475 CIRCLEQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next); 1476 sdp->swd_npginuse += *nslots; 1477 uvmexp.swpginuse += *nslots; 1478 simple_unlock(&uvm.swap_data_lock); 1479 /* done! return drum slot number */ 1480 UVMHIST_LOG(pdhist, 1481 "success! returning %d slots starting at %d", 1482 *nslots, result + sdp->swd_drumoffset, 0, 0); 1483 return (result + sdp->swd_drumoffset); 1484 } 1485 } 1486 1487 /* XXXMRG: BEGIN HACK */ 1488 if (*nslots > 1 && lessok) { 1489 *nslots = 1; 1490 /* XXXMRG: ugh! blist should support this for us */ 1491 goto ReTry; 1492 } 1493 /* XXXMRG: END HACK */ 1494 1495 simple_unlock(&uvm.swap_data_lock); 1496 return 0; 1497 } 1498 1499 boolean_t 1500 uvm_swapisfull(void) 1501 { 1502 boolean_t rv; 1503 1504 simple_lock(&uvm.swap_data_lock); 1505 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1506 rv = (uvmexp.swpgonly >= uvmexp.swpgavail); 1507 simple_unlock(&uvm.swap_data_lock); 1508 1509 return (rv); 1510 } 1511 1512 /* 1513 * uvm_swap_markbad: keep track of swap ranges where we've had i/o errors 1514 * 1515 * => we lock uvm.swap_data_lock 1516 */ 1517 void 1518 uvm_swap_markbad(int startslot, int nslots) 1519 { 1520 struct swapdev *sdp; 1521 UVMHIST_FUNC("uvm_swap_markbad"); UVMHIST_CALLED(pdhist); 1522 1523 simple_lock(&uvm.swap_data_lock); 1524 sdp = swapdrum_getsdp(startslot); 1525 KASSERT(sdp != NULL); 1526 1527 /* 1528 * we just keep track of how many pages have been marked bad 1529 * in this device, to make everything add up in swap_off(). 1530 * we assume here that the range of slots will all be within 1531 * one swap device. 1532 */ 1533 1534 KASSERT(uvmexp.swpgonly >= nslots); 1535 uvmexp.swpgonly -= nslots; 1536 sdp->swd_npgbad += nslots; 1537 UVMHIST_LOG(pdhist, "now %d bad", sdp->swd_npgbad, 0,0,0); 1538 simple_unlock(&uvm.swap_data_lock); 1539 } 1540 1541 /* 1542 * uvm_swap_free: free swap slots 1543 * 1544 * => this can be all or part of an allocation made by uvm_swap_alloc 1545 * => we lock uvm.swap_data_lock 1546 */ 1547 void 1548 uvm_swap_free(int startslot, int nslots) 1549 { 1550 struct swapdev *sdp; 1551 UVMHIST_FUNC("uvm_swap_free"); UVMHIST_CALLED(pdhist); 1552 1553 UVMHIST_LOG(pdhist, "freeing %d slots starting at %d", nslots, 1554 startslot, 0, 0); 1555 1556 /* 1557 * ignore attempts to free the "bad" slot. 1558 */ 1559 1560 if (startslot == SWSLOT_BAD) { 1561 return; 1562 } 1563 1564 /* 1565 * convert drum slot offset back to sdp, free the blocks 1566 * in the extent, and return. must hold pri lock to do 1567 * lookup and access the extent. 1568 */ 1569 1570 simple_lock(&uvm.swap_data_lock); 1571 sdp = swapdrum_getsdp(startslot); 1572 KASSERT(uvmexp.nswapdev >= 1); 1573 KASSERT(sdp != NULL); 1574 KASSERT(sdp->swd_npginuse >= nslots); 1575 blist_free(sdp->swd_blist, startslot - sdp->swd_drumoffset, nslots); 1576 sdp->swd_npginuse -= nslots; 1577 uvmexp.swpginuse -= nslots; 1578 simple_unlock(&uvm.swap_data_lock); 1579 } 1580 1581 /* 1582 * uvm_swap_put: put any number of pages into a contig place on swap 1583 * 1584 * => can be sync or async 1585 */ 1586 1587 int 1588 uvm_swap_put(int swslot, struct vm_page **ppsp, int npages, int flags) 1589 { 1590 int error; 1591 1592 error = uvm_swap_io(ppsp, swslot, npages, B_WRITE | 1593 ((flags & PGO_SYNCIO) ? 0 : B_ASYNC)); 1594 return error; 1595 } 1596 1597 /* 1598 * uvm_swap_get: get a single page from swap 1599 * 1600 * => usually a sync op (from fault) 1601 */ 1602 1603 int 1604 uvm_swap_get(struct vm_page *page, int swslot, int flags) 1605 { 1606 int error; 1607 1608 uvmexp.nswget++; 1609 KASSERT(flags & PGO_SYNCIO); 1610 if (swslot == SWSLOT_BAD) { 1611 return EIO; 1612 } 1613 1614 error = uvm_swap_io(&page, swslot, 1, B_READ | 1615 ((flags & PGO_SYNCIO) ? 0 : B_ASYNC)); 1616 if (error == 0) { 1617 1618 /* 1619 * this page is no longer only in swap. 1620 */ 1621 1622 simple_lock(&uvm.swap_data_lock); 1623 KASSERT(uvmexp.swpgonly > 0); 1624 uvmexp.swpgonly--; 1625 simple_unlock(&uvm.swap_data_lock); 1626 } 1627 return error; 1628 } 1629 1630 /* 1631 * uvm_swap_io: do an i/o operation to swap 1632 */ 1633 1634 static int 1635 uvm_swap_io(struct vm_page **pps, int startslot, int npages, int flags) 1636 { 1637 daddr_t startblk; 1638 struct buf *bp; 1639 vaddr_t kva; 1640 int error, s, mapinflags; 1641 boolean_t write, async; 1642 UVMHIST_FUNC("uvm_swap_io"); UVMHIST_CALLED(pdhist); 1643 1644 UVMHIST_LOG(pdhist, "<- called, startslot=%d, npages=%d, flags=%d", 1645 startslot, npages, flags, 0); 1646 1647 write = (flags & B_READ) == 0; 1648 async = (flags & B_ASYNC) != 0; 1649 1650 /* 1651 * convert starting drum slot to block number 1652 */ 1653 1654 startblk = btodb((uint64_t)startslot << PAGE_SHIFT); 1655 1656 /* 1657 * first, map the pages into the kernel. 1658 */ 1659 1660 mapinflags = !write ? 1661 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_READ : 1662 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_WRITE; 1663 kva = uvm_pagermapin(pps, npages, mapinflags); 1664 1665 /* 1666 * now allocate a buf for the i/o. 1667 */ 1668 1669 bp = getiobuf(); 1670 1671 /* 1672 * fill in the bp/sbp. we currently route our i/o through 1673 * /dev/drum's vnode [swapdev_vp]. 1674 */ 1675 1676 bp->b_flags = B_BUSY | B_NOCACHE | (flags & (B_READ|B_ASYNC)); 1677 bp->b_proc = &proc0; /* XXX */ 1678 bp->b_vnbufs.le_next = NOLIST; 1679 bp->b_data = (caddr_t)kva; 1680 bp->b_blkno = startblk; 1681 bp->b_vp = swapdev_vp; 1682 bp->b_bufsize = bp->b_bcount = npages << PAGE_SHIFT; 1683 1684 /* 1685 * bump v_numoutput (counter of number of active outputs). 1686 */ 1687 1688 if (write) { 1689 s = splbio(); 1690 V_INCR_NUMOUTPUT(swapdev_vp); 1691 splx(s); 1692 } 1693 1694 /* 1695 * for async ops we must set up the iodone handler. 1696 */ 1697 1698 if (async) { 1699 bp->b_flags |= B_CALL; 1700 bp->b_iodone = uvm_aio_biodone; 1701 UVMHIST_LOG(pdhist, "doing async!", 0, 0, 0, 0); 1702 if (curproc == uvm.pagedaemon_proc) 1703 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); 1704 else 1705 BIO_SETPRIO(bp, BPRIO_TIMELIMITED); 1706 } else { 1707 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); 1708 } 1709 UVMHIST_LOG(pdhist, 1710 "about to start io: data = %p blkno = 0x%x, bcount = %ld", 1711 bp->b_data, bp->b_blkno, bp->b_bcount, 0); 1712 1713 /* 1714 * now we start the I/O, and if async, return. 1715 */ 1716 1717 VOP_STRATEGY(swapdev_vp, bp); 1718 if (async) 1719 return 0; 1720 1721 /* 1722 * must be sync i/o. wait for it to finish 1723 */ 1724 1725 error = biowait(bp); 1726 1727 /* 1728 * kill the pager mapping 1729 */ 1730 1731 uvm_pagermapout(kva, npages); 1732 1733 /* 1734 * now dispose of the buf and we're done. 1735 */ 1736 1737 s = splbio(); 1738 if (write) 1739 vwakeup(bp); 1740 putiobuf(bp); 1741 splx(s); 1742 UVMHIST_LOG(pdhist, "<- done (sync) error=%d", error, 0, 0, 0); 1743 return (error); 1744 } 1745