1 /* $NetBSD: uvm_swap.c,v 1.145 2009/03/01 01:13:14 mrg Exp $ */ 2 3 /* 4 * Copyright (c) 1995, 1996, 1997, 2009 Matthew R. Green 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * from: NetBSD: vm_swap.c,v 1.52 1997/12/02 13:47:37 pk Exp 29 * from: Id: uvm_swap.c,v 1.1.2.42 1998/02/02 20:38:06 chuck Exp 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.145 2009/03/01 01:13:14 mrg Exp $"); 34 35 #include "fs_nfs.h" 36 #include "opt_uvmhist.h" 37 #include "opt_compat_netbsd.h" 38 #include "opt_ddb.h" 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #include <sys/buf.h> 43 #include <sys/bufq.h> 44 #include <sys/conf.h> 45 #include <sys/proc.h> 46 #include <sys/namei.h> 47 #include <sys/disklabel.h> 48 #include <sys/errno.h> 49 #include <sys/kernel.h> 50 #include <sys/malloc.h> 51 #include <sys/vnode.h> 52 #include <sys/file.h> 53 #include <sys/vmem.h> 54 #include <sys/blist.h> 55 #include <sys/mount.h> 56 #include <sys/pool.h> 57 #include <sys/syscallargs.h> 58 #include <sys/swap.h> 59 #include <sys/kauth.h> 60 #include <sys/sysctl.h> 61 #include <sys/workqueue.h> 62 63 #include <uvm/uvm.h> 64 65 #include <miscfs/specfs/specdev.h> 66 67 /* 68 * uvm_swap.c: manage configuration and i/o to swap space. 69 */ 70 71 /* 72 * swap space is managed in the following way: 73 * 74 * each swap partition or file is described by a "swapdev" structure. 75 * each "swapdev" structure contains a "swapent" structure which contains 76 * information that is passed up to the user (via system calls). 77 * 78 * each swap partition is assigned a "priority" (int) which controls 79 * swap parition usage. 80 * 81 * the system maintains a global data structure describing all swap 82 * partitions/files. there is a sorted LIST of "swappri" structures 83 * which describe "swapdev"'s at that priority. this LIST is headed 84 * by the "swap_priority" global var. each "swappri" contains a 85 * CIRCLEQ of "swapdev" structures at that priority. 86 * 87 * locking: 88 * - swap_syscall_lock (krwlock_t): this lock serializes the swapctl 89 * system call and prevents the swap priority list from changing 90 * while we are in the middle of a system call (e.g. SWAP_STATS). 91 * - uvm_swap_data_lock (kmutex_t): this lock protects all swap data 92 * structures including the priority list, the swapdev structures, 93 * and the swapmap arena. 94 * 95 * each swap device has the following info: 96 * - swap device in use (could be disabled, preventing future use) 97 * - swap enabled (allows new allocations on swap) 98 * - map info in /dev/drum 99 * - vnode pointer 100 * for swap files only: 101 * - block size 102 * - max byte count in buffer 103 * - buffer 104 * 105 * userland controls and configures swap with the swapctl(2) system call. 106 * the sys_swapctl performs the following operations: 107 * [1] SWAP_NSWAP: returns the number of swap devices currently configured 108 * [2] SWAP_STATS: given a pointer to an array of swapent structures 109 * (passed in via "arg") of a size passed in via "misc" ... we load 110 * the current swap config into the array. The actual work is done 111 * in the uvm_swap_stats(9) function. 112 * [3] SWAP_ON: given a pathname in arg (could be device or file) and a 113 * priority in "misc", start swapping on it. 114 * [4] SWAP_OFF: as SWAP_ON, but stops swapping to a device 115 * [5] SWAP_CTL: changes the priority of a swap device (new priority in 116 * "misc") 117 */ 118 119 /* 120 * swapdev: describes a single swap partition/file 121 * 122 * note the following should be true: 123 * swd_inuse <= swd_nblks [number of blocks in use is <= total blocks] 124 * swd_nblks <= swd_mapsize [because mapsize includes miniroot+disklabel] 125 */ 126 struct swapdev { 127 dev_t swd_dev; /* device id */ 128 int swd_flags; /* flags:inuse/enable/fake */ 129 int swd_priority; /* our priority */ 130 int swd_nblks; /* blocks in this device */ 131 char *swd_path; /* saved pathname of device */ 132 int swd_pathlen; /* length of pathname */ 133 int swd_npages; /* #pages we can use */ 134 int swd_npginuse; /* #pages in use */ 135 int swd_npgbad; /* #pages bad */ 136 int swd_drumoffset; /* page0 offset in drum */ 137 int swd_drumsize; /* #pages in drum */ 138 blist_t swd_blist; /* blist for this swapdev */ 139 struct vnode *swd_vp; /* backing vnode */ 140 CIRCLEQ_ENTRY(swapdev) swd_next; /* priority circleq */ 141 142 int swd_bsize; /* blocksize (bytes) */ 143 int swd_maxactive; /* max active i/o reqs */ 144 struct bufq_state *swd_tab; /* buffer list */ 145 int swd_active; /* number of active buffers */ 146 }; 147 148 /* 149 * swap device priority entry; the list is kept sorted on `spi_priority'. 150 */ 151 struct swappri { 152 int spi_priority; /* priority */ 153 CIRCLEQ_HEAD(spi_swapdev, swapdev) spi_swapdev; 154 /* circleq of swapdevs at this priority */ 155 LIST_ENTRY(swappri) spi_swappri; /* global list of pri's */ 156 }; 157 158 /* 159 * The following two structures are used to keep track of data transfers 160 * on swap devices associated with regular files. 161 * NOTE: this code is more or less a copy of vnd.c; we use the same 162 * structure names here to ease porting.. 163 */ 164 struct vndxfer { 165 struct buf *vx_bp; /* Pointer to parent buffer */ 166 struct swapdev *vx_sdp; 167 int vx_error; 168 int vx_pending; /* # of pending aux buffers */ 169 int vx_flags; 170 #define VX_BUSY 1 171 #define VX_DEAD 2 172 }; 173 174 struct vndbuf { 175 struct buf vb_buf; 176 struct vndxfer *vb_xfer; 177 }; 178 179 /* 180 * NetBSD 1.3 swapctl(SWAP_STATS, ...) swapent structure; uses 32 bit 181 * dev_t and has no se_path[] member. 182 */ 183 struct swapent13 { 184 int32_t se13_dev; /* device id */ 185 int se13_flags; /* flags */ 186 int se13_nblks; /* total blocks */ 187 int se13_inuse; /* blocks in use */ 188 int se13_priority; /* priority of this device */ 189 }; 190 191 /* 192 * NetBSD 5.0 swapctl(SWAP_STATS, ...) swapent structure; uses 32 bit 193 * dev_t. 194 */ 195 struct swapent50 { 196 int32_t se50_dev; /* device id */ 197 int se50_flags; /* flags */ 198 int se50_nblks; /* total blocks */ 199 int se50_inuse; /* blocks in use */ 200 int se50_priority; /* priority of this device */ 201 char se50_path[PATH_MAX+1]; /* path name */ 202 }; 203 204 /* 205 * We keep a of pool vndbuf's and vndxfer structures. 206 */ 207 POOL_INIT(vndxfer_pool, sizeof(struct vndxfer), 0, 0, 0, "swp vnx", NULL, 208 IPL_BIO); 209 POOL_INIT(vndbuf_pool, sizeof(struct vndbuf), 0, 0, 0, "swp vnd", NULL, 210 IPL_BIO); 211 212 /* 213 * local variables 214 */ 215 MALLOC_DEFINE(M_VMSWAP, "VM swap", "VM swap structures"); 216 static vmem_t *swapmap; /* controls the mapping of /dev/drum */ 217 218 /* list of all active swap devices [by priority] */ 219 LIST_HEAD(swap_priority, swappri); 220 static struct swap_priority swap_priority; 221 222 /* locks */ 223 static krwlock_t swap_syscall_lock; 224 225 /* workqueue and use counter for swap to regular files */ 226 static int sw_reg_count = 0; 227 static struct workqueue *sw_reg_workqueue; 228 229 /* tuneables */ 230 u_int uvm_swapisfull_factor = 99; 231 232 /* 233 * prototypes 234 */ 235 static struct swapdev *swapdrum_getsdp(int); 236 237 static struct swapdev *swaplist_find(struct vnode *, bool); 238 static void swaplist_insert(struct swapdev *, 239 struct swappri *, int); 240 static void swaplist_trim(void); 241 242 static int swap_on(struct lwp *, struct swapdev *); 243 static int swap_off(struct lwp *, struct swapdev *); 244 245 static void uvm_swap_stats_locked(int, struct swapent *, int, register_t *); 246 247 static void sw_reg_strategy(struct swapdev *, struct buf *, int); 248 static void sw_reg_biodone(struct buf *); 249 static void sw_reg_iodone(struct work *wk, void *dummy); 250 static void sw_reg_start(struct swapdev *); 251 252 static int uvm_swap_io(struct vm_page **, int, int, int); 253 254 /* 255 * uvm_swap_init: init the swap system data structures and locks 256 * 257 * => called at boot time from init_main.c after the filesystems 258 * are brought up (which happens after uvm_init()) 259 */ 260 void 261 uvm_swap_init(void) 262 { 263 UVMHIST_FUNC("uvm_swap_init"); 264 265 UVMHIST_CALLED(pdhist); 266 /* 267 * first, init the swap list, its counter, and its lock. 268 * then get a handle on the vnode for /dev/drum by using 269 * the its dev_t number ("swapdev", from MD conf.c). 270 */ 271 272 LIST_INIT(&swap_priority); 273 uvmexp.nswapdev = 0; 274 rw_init(&swap_syscall_lock); 275 cv_init(&uvm.scheduler_cv, "schedule"); 276 mutex_init(&uvm_swap_data_lock, MUTEX_DEFAULT, IPL_NONE); 277 278 /* XXXSMP should be at IPL_VM, but for audio interrupt handlers. */ 279 mutex_init(&uvm_scheduler_mutex, MUTEX_SPIN, IPL_SCHED); 280 281 if (bdevvp(swapdev, &swapdev_vp)) 282 panic("%s: can't get vnode for swap device", __func__); 283 if (vn_lock(swapdev_vp, LK_EXCLUSIVE | LK_RETRY)) 284 panic("%s: can't lock swap device", __func__); 285 if (VOP_OPEN(swapdev_vp, FREAD | FWRITE, NOCRED)) 286 panic("%s: can't open swap device", __func__); 287 VOP_UNLOCK(swapdev_vp, 0); 288 289 /* 290 * create swap block resource map to map /dev/drum. the range 291 * from 1 to INT_MAX allows 2 gigablocks of swap space. note 292 * that block 0 is reserved (used to indicate an allocation 293 * failure, or no allocation). 294 */ 295 swapmap = vmem_create("swapmap", 1, INT_MAX - 1, 1, NULL, NULL, NULL, 0, 296 VM_NOSLEEP, IPL_NONE); 297 if (swapmap == 0) 298 panic("%s: vmem_create failed", __func__); 299 300 /* 301 * done! 302 */ 303 uvm.swap_running = true; 304 #ifdef __SWAP_BROKEN 305 uvm.swapout_enabled = 0; 306 #else 307 uvm.swapout_enabled = 1; 308 #endif 309 UVMHIST_LOG(pdhist, "<- done", 0, 0, 0, 0); 310 311 sysctl_createv(NULL, 0, NULL, NULL, 312 CTLFLAG_READWRITE, 313 CTLTYPE_INT, "swapout", 314 SYSCTL_DESCR("Set 0 to disable swapout of kernel stacks"), 315 NULL, 0, &uvm.swapout_enabled, 0, CTL_VM, CTL_CREATE, CTL_EOL); 316 } 317 318 /* 319 * swaplist functions: functions that operate on the list of swap 320 * devices on the system. 321 */ 322 323 /* 324 * swaplist_insert: insert swap device "sdp" into the global list 325 * 326 * => caller must hold both swap_syscall_lock and uvm_swap_data_lock 327 * => caller must provide a newly malloc'd swappri structure (we will 328 * FREE it if we don't need it... this it to prevent malloc blocking 329 * here while adding swap) 330 */ 331 static void 332 swaplist_insert(struct swapdev *sdp, struct swappri *newspp, int priority) 333 { 334 struct swappri *spp, *pspp; 335 UVMHIST_FUNC("swaplist_insert"); UVMHIST_CALLED(pdhist); 336 337 /* 338 * find entry at or after which to insert the new device. 339 */ 340 pspp = NULL; 341 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 342 if (priority <= spp->spi_priority) 343 break; 344 pspp = spp; 345 } 346 347 /* 348 * new priority? 349 */ 350 if (spp == NULL || spp->spi_priority != priority) { 351 spp = newspp; /* use newspp! */ 352 UVMHIST_LOG(pdhist, "created new swappri = %d", 353 priority, 0, 0, 0); 354 355 spp->spi_priority = priority; 356 CIRCLEQ_INIT(&spp->spi_swapdev); 357 358 if (pspp) 359 LIST_INSERT_AFTER(pspp, spp, spi_swappri); 360 else 361 LIST_INSERT_HEAD(&swap_priority, spp, spi_swappri); 362 } else { 363 /* we don't need a new priority structure, free it */ 364 free(newspp, M_VMSWAP); 365 } 366 367 /* 368 * priority found (or created). now insert on the priority's 369 * circleq list and bump the total number of swapdevs. 370 */ 371 sdp->swd_priority = priority; 372 CIRCLEQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next); 373 uvmexp.nswapdev++; 374 } 375 376 /* 377 * swaplist_find: find and optionally remove a swap device from the 378 * global list. 379 * 380 * => caller must hold both swap_syscall_lock and uvm_swap_data_lock 381 * => we return the swapdev we found (and removed) 382 */ 383 static struct swapdev * 384 swaplist_find(struct vnode *vp, bool remove) 385 { 386 struct swapdev *sdp; 387 struct swappri *spp; 388 389 /* 390 * search the lists for the requested vp 391 */ 392 393 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 394 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 395 if (sdp->swd_vp == vp) { 396 if (remove) { 397 CIRCLEQ_REMOVE(&spp->spi_swapdev, 398 sdp, swd_next); 399 uvmexp.nswapdev--; 400 } 401 return(sdp); 402 } 403 } 404 } 405 return (NULL); 406 } 407 408 /* 409 * swaplist_trim: scan priority list for empty priority entries and kill 410 * them. 411 * 412 * => caller must hold both swap_syscall_lock and uvm_swap_data_lock 413 */ 414 static void 415 swaplist_trim(void) 416 { 417 struct swappri *spp, *nextspp; 418 419 for (spp = LIST_FIRST(&swap_priority); spp != NULL; spp = nextspp) { 420 nextspp = LIST_NEXT(spp, spi_swappri); 421 if (CIRCLEQ_FIRST(&spp->spi_swapdev) != 422 (void *)&spp->spi_swapdev) 423 continue; 424 LIST_REMOVE(spp, spi_swappri); 425 free(spp, M_VMSWAP); 426 } 427 } 428 429 /* 430 * swapdrum_getsdp: given a page offset in /dev/drum, convert it back 431 * to the "swapdev" that maps that section of the drum. 432 * 433 * => each swapdev takes one big contig chunk of the drum 434 * => caller must hold uvm_swap_data_lock 435 */ 436 static struct swapdev * 437 swapdrum_getsdp(int pgno) 438 { 439 struct swapdev *sdp; 440 struct swappri *spp; 441 442 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 443 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 444 if (sdp->swd_flags & SWF_FAKE) 445 continue; 446 if (pgno >= sdp->swd_drumoffset && 447 pgno < (sdp->swd_drumoffset + sdp->swd_drumsize)) { 448 return sdp; 449 } 450 } 451 } 452 return NULL; 453 } 454 455 456 /* 457 * sys_swapctl: main entry point for swapctl(2) system call 458 * [with two helper functions: swap_on and swap_off] 459 */ 460 int 461 sys_swapctl(struct lwp *l, const struct sys_swapctl_args *uap, register_t *retval) 462 { 463 /* { 464 syscallarg(int) cmd; 465 syscallarg(void *) arg; 466 syscallarg(int) misc; 467 } */ 468 struct vnode *vp; 469 struct nameidata nd; 470 struct swappri *spp; 471 struct swapdev *sdp; 472 struct swapent *sep; 473 #define SWAP_PATH_MAX (PATH_MAX + 1) 474 char *userpath; 475 size_t len; 476 int error, misc; 477 int priority; 478 UVMHIST_FUNC("sys_swapctl"); UVMHIST_CALLED(pdhist); 479 480 misc = SCARG(uap, misc); 481 482 /* 483 * ensure serialized syscall access by grabbing the swap_syscall_lock 484 */ 485 rw_enter(&swap_syscall_lock, RW_WRITER); 486 487 userpath = malloc(SWAP_PATH_MAX, M_TEMP, M_WAITOK); 488 /* 489 * we handle the non-priv NSWAP and STATS request first. 490 * 491 * SWAP_NSWAP: return number of config'd swap devices 492 * [can also be obtained with uvmexp sysctl] 493 */ 494 if (SCARG(uap, cmd) == SWAP_NSWAP) { 495 UVMHIST_LOG(pdhist, "<- done SWAP_NSWAP=%d", uvmexp.nswapdev, 496 0, 0, 0); 497 *retval = uvmexp.nswapdev; 498 error = 0; 499 goto out; 500 } 501 502 /* 503 * SWAP_STATS: get stats on current # of configured swap devs 504 * 505 * note that the swap_priority list can't change as long 506 * as we are holding the swap_syscall_lock. we don't want 507 * to grab the uvm_swap_data_lock because we may fault&sleep during 508 * copyout() and we don't want to be holding that lock then! 509 */ 510 if (SCARG(uap, cmd) == SWAP_STATS 511 #if defined(COMPAT_50) 512 || SCARG(uap, cmd) == SWAP_STATS50 513 #endif 514 #if defined(COMPAT_13) 515 || SCARG(uap, cmd) == SWAP_STATS13 516 #endif 517 ) { 518 if ((size_t)misc > (size_t)uvmexp.nswapdev) 519 misc = uvmexp.nswapdev; 520 #if defined(COMPAT_13) 521 if (SCARG(uap, cmd) == SWAP_STATS13) 522 len = sizeof(struct swapent13) * misc; 523 else 524 #endif 525 #if defined(COMPAT_50) 526 if (SCARG(uap, cmd) == SWAP_STATS50) 527 len = sizeof(struct swapent50) * misc; 528 else 529 #endif 530 len = sizeof(struct swapent) * misc; 531 sep = (struct swapent *)malloc(len, M_TEMP, M_WAITOK); 532 533 uvm_swap_stats_locked(SCARG(uap, cmd), sep, misc, retval); 534 error = copyout(sep, SCARG(uap, arg), len); 535 536 free(sep, M_TEMP); 537 UVMHIST_LOG(pdhist, "<- done SWAP_STATS", 0, 0, 0, 0); 538 goto out; 539 } 540 if (SCARG(uap, cmd) == SWAP_GETDUMPDEV) { 541 dev_t *devp = (dev_t *)SCARG(uap, arg); 542 543 error = copyout(&dumpdev, devp, sizeof(dumpdev)); 544 goto out; 545 } 546 547 /* 548 * all other requests require superuser privs. verify. 549 */ 550 if ((error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_SWAPCTL, 551 0, NULL, NULL, NULL))) 552 goto out; 553 554 if (SCARG(uap, cmd) == SWAP_DUMPOFF) { 555 /* drop the current dump device */ 556 dumpdev = NODEV; 557 dumpcdev = NODEV; 558 cpu_dumpconf(); 559 goto out; 560 } 561 562 /* 563 * at this point we expect a path name in arg. we will 564 * use namei() to gain a vnode reference (vref), and lock 565 * the vnode (VOP_LOCK). 566 * 567 * XXX: a NULL arg means use the root vnode pointer (e.g. for 568 * miniroot) 569 */ 570 if (SCARG(uap, arg) == NULL) { 571 vp = rootvp; /* miniroot */ 572 if (vget(vp, LK_EXCLUSIVE)) { 573 error = EBUSY; 574 goto out; 575 } 576 if (SCARG(uap, cmd) == SWAP_ON && 577 copystr("miniroot", userpath, SWAP_PATH_MAX, &len)) 578 panic("swapctl: miniroot copy failed"); 579 } else { 580 int space; 581 char *where; 582 583 if (SCARG(uap, cmd) == SWAP_ON) { 584 if ((error = copyinstr(SCARG(uap, arg), userpath, 585 SWAP_PATH_MAX, &len))) 586 goto out; 587 space = UIO_SYSSPACE; 588 where = userpath; 589 } else { 590 space = UIO_USERSPACE; 591 where = (char *)SCARG(uap, arg); 592 } 593 NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | TRYEMULROOT, 594 space, where); 595 if ((error = namei(&nd))) 596 goto out; 597 vp = nd.ni_vp; 598 } 599 /* note: "vp" is referenced and locked */ 600 601 error = 0; /* assume no error */ 602 switch(SCARG(uap, cmd)) { 603 604 case SWAP_DUMPDEV: 605 if (vp->v_type != VBLK) { 606 error = ENOTBLK; 607 break; 608 } 609 if (bdevsw_lookup(vp->v_rdev)) { 610 dumpdev = vp->v_rdev; 611 dumpcdev = devsw_blk2chr(dumpdev); 612 } else 613 dumpdev = NODEV; 614 cpu_dumpconf(); 615 break; 616 617 case SWAP_CTL: 618 /* 619 * get new priority, remove old entry (if any) and then 620 * reinsert it in the correct place. finally, prune out 621 * any empty priority structures. 622 */ 623 priority = SCARG(uap, misc); 624 spp = malloc(sizeof *spp, M_VMSWAP, M_WAITOK); 625 mutex_enter(&uvm_swap_data_lock); 626 if ((sdp = swaplist_find(vp, true)) == NULL) { 627 error = ENOENT; 628 } else { 629 swaplist_insert(sdp, spp, priority); 630 swaplist_trim(); 631 } 632 mutex_exit(&uvm_swap_data_lock); 633 if (error) 634 free(spp, M_VMSWAP); 635 break; 636 637 case SWAP_ON: 638 639 /* 640 * check for duplicates. if none found, then insert a 641 * dummy entry on the list to prevent someone else from 642 * trying to enable this device while we are working on 643 * it. 644 */ 645 646 priority = SCARG(uap, misc); 647 sdp = malloc(sizeof *sdp, M_VMSWAP, M_WAITOK); 648 spp = malloc(sizeof *spp, M_VMSWAP, M_WAITOK); 649 memset(sdp, 0, sizeof(*sdp)); 650 sdp->swd_flags = SWF_FAKE; 651 sdp->swd_vp = vp; 652 sdp->swd_dev = (vp->v_type == VBLK) ? vp->v_rdev : NODEV; 653 bufq_alloc(&sdp->swd_tab, "disksort", BUFQ_SORT_RAWBLOCK); 654 mutex_enter(&uvm_swap_data_lock); 655 if (swaplist_find(vp, false) != NULL) { 656 error = EBUSY; 657 mutex_exit(&uvm_swap_data_lock); 658 bufq_free(sdp->swd_tab); 659 free(sdp, M_VMSWAP); 660 free(spp, M_VMSWAP); 661 break; 662 } 663 swaplist_insert(sdp, spp, priority); 664 mutex_exit(&uvm_swap_data_lock); 665 666 sdp->swd_pathlen = len; 667 sdp->swd_path = malloc(sdp->swd_pathlen, M_VMSWAP, M_WAITOK); 668 if (copystr(userpath, sdp->swd_path, sdp->swd_pathlen, 0) != 0) 669 panic("swapctl: copystr"); 670 671 /* 672 * we've now got a FAKE placeholder in the swap list. 673 * now attempt to enable swap on it. if we fail, undo 674 * what we've done and kill the fake entry we just inserted. 675 * if swap_on is a success, it will clear the SWF_FAKE flag 676 */ 677 678 if ((error = swap_on(l, sdp)) != 0) { 679 mutex_enter(&uvm_swap_data_lock); 680 (void) swaplist_find(vp, true); /* kill fake entry */ 681 swaplist_trim(); 682 mutex_exit(&uvm_swap_data_lock); 683 bufq_free(sdp->swd_tab); 684 free(sdp->swd_path, M_VMSWAP); 685 free(sdp, M_VMSWAP); 686 break; 687 } 688 break; 689 690 case SWAP_OFF: 691 mutex_enter(&uvm_swap_data_lock); 692 if ((sdp = swaplist_find(vp, false)) == NULL) { 693 mutex_exit(&uvm_swap_data_lock); 694 error = ENXIO; 695 break; 696 } 697 698 /* 699 * If a device isn't in use or enabled, we 700 * can't stop swapping from it (again). 701 */ 702 if ((sdp->swd_flags & (SWF_INUSE|SWF_ENABLE)) == 0) { 703 mutex_exit(&uvm_swap_data_lock); 704 error = EBUSY; 705 break; 706 } 707 708 /* 709 * do the real work. 710 */ 711 error = swap_off(l, sdp); 712 break; 713 714 default: 715 error = EINVAL; 716 } 717 718 /* 719 * done! release the ref gained by namei() and unlock. 720 */ 721 vput(vp); 722 723 out: 724 free(userpath, M_TEMP); 725 rw_exit(&swap_syscall_lock); 726 727 UVMHIST_LOG(pdhist, "<- done! error=%d", error, 0, 0, 0); 728 return (error); 729 } 730 731 /* 732 * swap_stats: implements swapctl(SWAP_STATS). The function is kept 733 * away from sys_swapctl() in order to allow COMPAT_* swapctl() 734 * emulation to use it directly without going through sys_swapctl(). 735 * The problem with using sys_swapctl() there is that it involves 736 * copying the swapent array to the stackgap, and this array's size 737 * is not known at build time. Hence it would not be possible to 738 * ensure it would fit in the stackgap in any case. 739 */ 740 void 741 uvm_swap_stats(int cmd, struct swapent *sep, int sec, register_t *retval) 742 { 743 744 rw_enter(&swap_syscall_lock, RW_READER); 745 uvm_swap_stats_locked(cmd, sep, sec, retval); 746 rw_exit(&swap_syscall_lock); 747 } 748 749 static void 750 uvm_swap_stats_locked(int cmd, struct swapent *sep, int sec, register_t *retval) 751 { 752 struct swappri *spp; 753 struct swapdev *sdp; 754 int count = 0; 755 756 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 757 for (sdp = CIRCLEQ_FIRST(&spp->spi_swapdev); 758 sdp != (void *)&spp->spi_swapdev && sec-- > 0; 759 sdp = CIRCLEQ_NEXT(sdp, swd_next)) { 760 int inuse; 761 762 /* 763 * backwards compatibility for system call. 764 * For NetBSD 1.3 and 5.0, we have to use 765 * the 32 bit dev_t. For 5.0 and -current 766 * we have to add the path. 767 */ 768 inuse = btodb((uint64_t)sdp->swd_npginuse << 769 PAGE_SHIFT); 770 771 #if defined(COMPAT_13) || defined(COMPAT_50) 772 if (cmd == SWAP_STATS) { 773 #endif 774 sep->se_dev = sdp->swd_dev; 775 sep->se_flags = sdp->swd_flags; 776 sep->se_nblks = sdp->swd_nblks; 777 sep->se_inuse = inuse; 778 sep->se_priority = sdp->swd_priority; 779 memcpy(&sep->se_path, sdp->swd_path, 780 sizeof sep->se_path); 781 sep++; 782 #if defined(COMPAT_13) 783 } else if (cmd == SWAP_STATS13) { 784 struct swapent13 *sep13 = 785 (struct swapent13 *)sep; 786 787 sep13->se13_dev = sdp->swd_dev; 788 sep13->se13_flags = sdp->swd_flags; 789 sep13->se13_nblks = sdp->swd_nblks; 790 sep13->se13_inuse = inuse; 791 sep13->se13_priority = sdp->swd_priority; 792 sep = (struct swapent *)(sep13 + 1); 793 #endif 794 #if defined(COMPAT_50) 795 } else if (cmd == SWAP_STATS50) { 796 struct swapent50 *sep50 = 797 (struct swapent50 *)sep; 798 799 sep50->se50_dev = sdp->swd_dev; 800 sep50->se50_flags = sdp->swd_flags; 801 sep50->se50_nblks = sdp->swd_nblks; 802 sep50->se50_inuse = inuse; 803 sep50->se50_priority = sdp->swd_priority; 804 memcpy(&sep50->se50_path, sdp->swd_path, 805 sizeof sep50->se50_path); 806 sep = (struct swapent *)(sep50 + 1); 807 } 808 #endif 809 count++; 810 } 811 } 812 813 *retval = count; 814 return; 815 } 816 817 /* 818 * swap_on: attempt to enable a swapdev for swapping. note that the 819 * swapdev is already on the global list, but disabled (marked 820 * SWF_FAKE). 821 * 822 * => we avoid the start of the disk (to protect disk labels) 823 * => we also avoid the miniroot, if we are swapping to root. 824 * => caller should leave uvm_swap_data_lock unlocked, we may lock it 825 * if needed. 826 */ 827 static int 828 swap_on(struct lwp *l, struct swapdev *sdp) 829 { 830 struct vnode *vp; 831 int error, npages, nblocks, size; 832 long addr; 833 u_long result; 834 struct vattr va; 835 #ifdef NFS 836 extern int (**nfsv2_vnodeop_p)(void *); 837 #endif /* NFS */ 838 const struct bdevsw *bdev; 839 dev_t dev; 840 UVMHIST_FUNC("swap_on"); UVMHIST_CALLED(pdhist); 841 842 /* 843 * we want to enable swapping on sdp. the swd_vp contains 844 * the vnode we want (locked and ref'd), and the swd_dev 845 * contains the dev_t of the file, if it a block device. 846 */ 847 848 vp = sdp->swd_vp; 849 dev = sdp->swd_dev; 850 851 /* 852 * open the swap file (mostly useful for block device files to 853 * let device driver know what is up). 854 * 855 * we skip the open/close for root on swap because the root 856 * has already been opened when root was mounted (mountroot). 857 */ 858 if (vp != rootvp) { 859 if ((error = VOP_OPEN(vp, FREAD|FWRITE, l->l_cred))) 860 return (error); 861 } 862 863 /* XXX this only works for block devices */ 864 UVMHIST_LOG(pdhist, " dev=%d, major(dev)=%d", dev, major(dev), 0,0); 865 866 /* 867 * we now need to determine the size of the swap area. for 868 * block specials we can call the d_psize function. 869 * for normal files, we must stat [get attrs]. 870 * 871 * we put the result in nblks. 872 * for normal files, we also want the filesystem block size 873 * (which we get with statfs). 874 */ 875 switch (vp->v_type) { 876 case VBLK: 877 bdev = bdevsw_lookup(dev); 878 if (bdev == NULL || bdev->d_psize == NULL || 879 (nblocks = (*bdev->d_psize)(dev)) == -1) { 880 error = ENXIO; 881 goto bad; 882 } 883 break; 884 885 case VREG: 886 if ((error = VOP_GETATTR(vp, &va, l->l_cred))) 887 goto bad; 888 nblocks = (int)btodb(va.va_size); 889 if ((error = 890 VFS_STATVFS(vp->v_mount, &vp->v_mount->mnt_stat)) != 0) 891 goto bad; 892 893 sdp->swd_bsize = vp->v_mount->mnt_stat.f_iosize; 894 /* 895 * limit the max # of outstanding I/O requests we issue 896 * at any one time. take it easy on NFS servers. 897 */ 898 #ifdef NFS 899 if (vp->v_op == nfsv2_vnodeop_p) 900 sdp->swd_maxactive = 2; /* XXX */ 901 else 902 #endif /* NFS */ 903 sdp->swd_maxactive = 8; /* XXX */ 904 break; 905 906 default: 907 error = ENXIO; 908 goto bad; 909 } 910 911 /* 912 * save nblocks in a safe place and convert to pages. 913 */ 914 915 sdp->swd_nblks = nblocks; 916 npages = dbtob((uint64_t)nblocks) >> PAGE_SHIFT; 917 918 /* 919 * for block special files, we want to make sure that leave 920 * the disklabel and bootblocks alone, so we arrange to skip 921 * over them (arbitrarily choosing to skip PAGE_SIZE bytes). 922 * note that because of this the "size" can be less than the 923 * actual number of blocks on the device. 924 */ 925 if (vp->v_type == VBLK) { 926 /* we use pages 1 to (size - 1) [inclusive] */ 927 size = npages - 1; 928 addr = 1; 929 } else { 930 /* we use pages 0 to (size - 1) [inclusive] */ 931 size = npages; 932 addr = 0; 933 } 934 935 /* 936 * make sure we have enough blocks for a reasonable sized swap 937 * area. we want at least one page. 938 */ 939 940 if (size < 1) { 941 UVMHIST_LOG(pdhist, " size <= 1!!", 0, 0, 0, 0); 942 error = EINVAL; 943 goto bad; 944 } 945 946 UVMHIST_LOG(pdhist, " dev=%x: size=%d addr=%ld\n", dev, size, addr, 0); 947 948 /* 949 * now we need to allocate an extent to manage this swap device 950 */ 951 952 sdp->swd_blist = blist_create(npages); 953 /* mark all expect the `saved' region free. */ 954 blist_free(sdp->swd_blist, addr, size); 955 956 /* 957 * if the vnode we are swapping to is the root vnode 958 * (i.e. we are swapping to the miniroot) then we want 959 * to make sure we don't overwrite it. do a statfs to 960 * find its size and skip over it. 961 */ 962 if (vp == rootvp) { 963 struct mount *mp; 964 struct statvfs *sp; 965 int rootblocks, rootpages; 966 967 mp = rootvnode->v_mount; 968 sp = &mp->mnt_stat; 969 rootblocks = sp->f_blocks * btodb(sp->f_frsize); 970 /* 971 * XXX: sp->f_blocks isn't the total number of 972 * blocks in the filesystem, it's the number of 973 * data blocks. so, our rootblocks almost 974 * definitely underestimates the total size 975 * of the filesystem - how badly depends on the 976 * details of the filesystem type. there isn't 977 * an obvious way to deal with this cleanly 978 * and perfectly, so for now we just pad our 979 * rootblocks estimate with an extra 5 percent. 980 */ 981 rootblocks += (rootblocks >> 5) + 982 (rootblocks >> 6) + 983 (rootblocks >> 7); 984 rootpages = round_page(dbtob(rootblocks)) >> PAGE_SHIFT; 985 if (rootpages > size) 986 panic("swap_on: miniroot larger than swap?"); 987 988 if (rootpages != blist_fill(sdp->swd_blist, addr, rootpages)) { 989 panic("swap_on: unable to preserve miniroot"); 990 } 991 992 size -= rootpages; 993 printf("Preserved %d pages of miniroot ", rootpages); 994 printf("leaving %d pages of swap\n", size); 995 } 996 997 /* 998 * add a ref to vp to reflect usage as a swap device. 999 */ 1000 vref(vp); 1001 1002 /* 1003 * now add the new swapdev to the drum and enable. 1004 */ 1005 result = vmem_alloc(swapmap, npages, VM_BESTFIT | VM_SLEEP); 1006 if (result == 0) 1007 panic("swapdrum_add"); 1008 /* 1009 * If this is the first regular swap create the workqueue. 1010 * => Protected by swap_syscall_lock. 1011 */ 1012 if (vp->v_type != VBLK) { 1013 if (sw_reg_count++ == 0) { 1014 KASSERT(sw_reg_workqueue == NULL); 1015 if (workqueue_create(&sw_reg_workqueue, "swapiod", 1016 sw_reg_iodone, NULL, PRIBIO, IPL_BIO, 0) != 0) 1017 panic("%s: workqueue_create failed", __func__); 1018 } 1019 } 1020 1021 sdp->swd_drumoffset = (int)result; 1022 sdp->swd_drumsize = npages; 1023 sdp->swd_npages = size; 1024 mutex_enter(&uvm_swap_data_lock); 1025 sdp->swd_flags &= ~SWF_FAKE; /* going live */ 1026 sdp->swd_flags |= (SWF_INUSE|SWF_ENABLE); 1027 uvmexp.swpages += size; 1028 uvmexp.swpgavail += size; 1029 mutex_exit(&uvm_swap_data_lock); 1030 return (0); 1031 1032 /* 1033 * failure: clean up and return error. 1034 */ 1035 1036 bad: 1037 if (sdp->swd_blist) { 1038 blist_destroy(sdp->swd_blist); 1039 } 1040 if (vp != rootvp) { 1041 (void)VOP_CLOSE(vp, FREAD|FWRITE, l->l_cred); 1042 } 1043 return (error); 1044 } 1045 1046 /* 1047 * swap_off: stop swapping on swapdev 1048 * 1049 * => swap data should be locked, we will unlock. 1050 */ 1051 static int 1052 swap_off(struct lwp *l, struct swapdev *sdp) 1053 { 1054 int npages = sdp->swd_npages; 1055 int error = 0; 1056 1057 UVMHIST_FUNC("swap_off"); UVMHIST_CALLED(pdhist); 1058 UVMHIST_LOG(pdhist, " dev=%x, npages=%d", sdp->swd_dev,npages,0,0); 1059 1060 /* disable the swap area being removed */ 1061 sdp->swd_flags &= ~SWF_ENABLE; 1062 uvmexp.swpgavail -= npages; 1063 mutex_exit(&uvm_swap_data_lock); 1064 1065 /* 1066 * the idea is to find all the pages that are paged out to this 1067 * device, and page them all in. in uvm, swap-backed pageable 1068 * memory can take two forms: aobjs and anons. call the 1069 * swapoff hook for each subsystem to bring in pages. 1070 */ 1071 1072 if (uao_swap_off(sdp->swd_drumoffset, 1073 sdp->swd_drumoffset + sdp->swd_drumsize) || 1074 amap_swap_off(sdp->swd_drumoffset, 1075 sdp->swd_drumoffset + sdp->swd_drumsize)) { 1076 error = ENOMEM; 1077 } else if (sdp->swd_npginuse > sdp->swd_npgbad) { 1078 error = EBUSY; 1079 } 1080 1081 if (error) { 1082 mutex_enter(&uvm_swap_data_lock); 1083 sdp->swd_flags |= SWF_ENABLE; 1084 uvmexp.swpgavail += npages; 1085 mutex_exit(&uvm_swap_data_lock); 1086 1087 return error; 1088 } 1089 1090 /* 1091 * If this is the last regular swap destroy the workqueue. 1092 * => Protected by swap_syscall_lock. 1093 */ 1094 if (sdp->swd_vp->v_type != VBLK) { 1095 KASSERT(sw_reg_count > 0); 1096 KASSERT(sw_reg_workqueue != NULL); 1097 if (--sw_reg_count == 0) { 1098 workqueue_destroy(sw_reg_workqueue); 1099 sw_reg_workqueue = NULL; 1100 } 1101 } 1102 1103 /* 1104 * done with the vnode. 1105 * drop our ref on the vnode before calling VOP_CLOSE() 1106 * so that spec_close() can tell if this is the last close. 1107 */ 1108 vrele(sdp->swd_vp); 1109 if (sdp->swd_vp != rootvp) { 1110 (void) VOP_CLOSE(sdp->swd_vp, FREAD|FWRITE, l->l_cred); 1111 } 1112 1113 mutex_enter(&uvm_swap_data_lock); 1114 uvmexp.swpages -= npages; 1115 uvmexp.swpginuse -= sdp->swd_npgbad; 1116 1117 if (swaplist_find(sdp->swd_vp, true) == NULL) 1118 panic("%s: swapdev not in list", __func__); 1119 swaplist_trim(); 1120 mutex_exit(&uvm_swap_data_lock); 1121 1122 /* 1123 * free all resources! 1124 */ 1125 vmem_free(swapmap, sdp->swd_drumoffset, sdp->swd_drumsize); 1126 blist_destroy(sdp->swd_blist); 1127 bufq_free(sdp->swd_tab); 1128 free(sdp, M_VMSWAP); 1129 return (0); 1130 } 1131 1132 /* 1133 * /dev/drum interface and i/o functions 1134 */ 1135 1136 /* 1137 * swstrategy: perform I/O on the drum 1138 * 1139 * => we must map the i/o request from the drum to the correct swapdev. 1140 */ 1141 static void 1142 swstrategy(struct buf *bp) 1143 { 1144 struct swapdev *sdp; 1145 struct vnode *vp; 1146 int pageno, bn; 1147 UVMHIST_FUNC("swstrategy"); UVMHIST_CALLED(pdhist); 1148 1149 /* 1150 * convert block number to swapdev. note that swapdev can't 1151 * be yanked out from under us because we are holding resources 1152 * in it (i.e. the blocks we are doing I/O on). 1153 */ 1154 pageno = dbtob((int64_t)bp->b_blkno) >> PAGE_SHIFT; 1155 mutex_enter(&uvm_swap_data_lock); 1156 sdp = swapdrum_getsdp(pageno); 1157 mutex_exit(&uvm_swap_data_lock); 1158 if (sdp == NULL) { 1159 bp->b_error = EINVAL; 1160 biodone(bp); 1161 UVMHIST_LOG(pdhist, " failed to get swap device", 0, 0, 0, 0); 1162 return; 1163 } 1164 1165 /* 1166 * convert drum page number to block number on this swapdev. 1167 */ 1168 1169 pageno -= sdp->swd_drumoffset; /* page # on swapdev */ 1170 bn = btodb((uint64_t)pageno << PAGE_SHIFT); /* convert to diskblock */ 1171 1172 UVMHIST_LOG(pdhist, " %s: mapoff=%x bn=%x bcount=%ld", 1173 ((bp->b_flags & B_READ) == 0) ? "write" : "read", 1174 sdp->swd_drumoffset, bn, bp->b_bcount); 1175 1176 /* 1177 * for block devices we finish up here. 1178 * for regular files we have to do more work which we delegate 1179 * to sw_reg_strategy(). 1180 */ 1181 1182 vp = sdp->swd_vp; /* swapdev vnode pointer */ 1183 switch (vp->v_type) { 1184 default: 1185 panic("%s: vnode type 0x%x", __func__, vp->v_type); 1186 1187 case VBLK: 1188 1189 /* 1190 * must convert "bp" from an I/O on /dev/drum to an I/O 1191 * on the swapdev (sdp). 1192 */ 1193 bp->b_blkno = bn; /* swapdev block number */ 1194 bp->b_dev = sdp->swd_dev; /* swapdev dev_t */ 1195 1196 /* 1197 * if we are doing a write, we have to redirect the i/o on 1198 * drum's v_numoutput counter to the swapdevs. 1199 */ 1200 if ((bp->b_flags & B_READ) == 0) { 1201 mutex_enter(bp->b_objlock); 1202 vwakeup(bp); /* kills one 'v_numoutput' on drum */ 1203 mutex_exit(bp->b_objlock); 1204 mutex_enter(&vp->v_interlock); 1205 vp->v_numoutput++; /* put it on swapdev */ 1206 mutex_exit(&vp->v_interlock); 1207 } 1208 1209 /* 1210 * finally plug in swapdev vnode and start I/O 1211 */ 1212 bp->b_vp = vp; 1213 bp->b_objlock = &vp->v_interlock; 1214 VOP_STRATEGY(vp, bp); 1215 return; 1216 1217 case VREG: 1218 /* 1219 * delegate to sw_reg_strategy function. 1220 */ 1221 sw_reg_strategy(sdp, bp, bn); 1222 return; 1223 } 1224 /* NOTREACHED */ 1225 } 1226 1227 /* 1228 * swread: the read function for the drum (just a call to physio) 1229 */ 1230 /*ARGSUSED*/ 1231 static int 1232 swread(dev_t dev, struct uio *uio, int ioflag) 1233 { 1234 UVMHIST_FUNC("swread"); UVMHIST_CALLED(pdhist); 1235 1236 UVMHIST_LOG(pdhist, " dev=%x offset=%qx", dev, uio->uio_offset, 0, 0); 1237 return (physio(swstrategy, NULL, dev, B_READ, minphys, uio)); 1238 } 1239 1240 /* 1241 * swwrite: the write function for the drum (just a call to physio) 1242 */ 1243 /*ARGSUSED*/ 1244 static int 1245 swwrite(dev_t dev, struct uio *uio, int ioflag) 1246 { 1247 UVMHIST_FUNC("swwrite"); UVMHIST_CALLED(pdhist); 1248 1249 UVMHIST_LOG(pdhist, " dev=%x offset=%qx", dev, uio->uio_offset, 0, 0); 1250 return (physio(swstrategy, NULL, dev, B_WRITE, minphys, uio)); 1251 } 1252 1253 const struct bdevsw swap_bdevsw = { 1254 nullopen, nullclose, swstrategy, noioctl, nodump, nosize, D_OTHER, 1255 }; 1256 1257 const struct cdevsw swap_cdevsw = { 1258 nullopen, nullclose, swread, swwrite, noioctl, 1259 nostop, notty, nopoll, nommap, nokqfilter, D_OTHER, 1260 }; 1261 1262 /* 1263 * sw_reg_strategy: handle swap i/o to regular files 1264 */ 1265 static void 1266 sw_reg_strategy(struct swapdev *sdp, struct buf *bp, int bn) 1267 { 1268 struct vnode *vp; 1269 struct vndxfer *vnx; 1270 daddr_t nbn; 1271 char *addr; 1272 off_t byteoff; 1273 int s, off, nra, error, sz, resid; 1274 UVMHIST_FUNC("sw_reg_strategy"); UVMHIST_CALLED(pdhist); 1275 1276 /* 1277 * allocate a vndxfer head for this transfer and point it to 1278 * our buffer. 1279 */ 1280 vnx = pool_get(&vndxfer_pool, PR_WAITOK); 1281 vnx->vx_flags = VX_BUSY; 1282 vnx->vx_error = 0; 1283 vnx->vx_pending = 0; 1284 vnx->vx_bp = bp; 1285 vnx->vx_sdp = sdp; 1286 1287 /* 1288 * setup for main loop where we read filesystem blocks into 1289 * our buffer. 1290 */ 1291 error = 0; 1292 bp->b_resid = bp->b_bcount; /* nothing transfered yet! */ 1293 addr = bp->b_data; /* current position in buffer */ 1294 byteoff = dbtob((uint64_t)bn); 1295 1296 for (resid = bp->b_resid; resid; resid -= sz) { 1297 struct vndbuf *nbp; 1298 1299 /* 1300 * translate byteoffset into block number. return values: 1301 * vp = vnode of underlying device 1302 * nbn = new block number (on underlying vnode dev) 1303 * nra = num blocks we can read-ahead (excludes requested 1304 * block) 1305 */ 1306 nra = 0; 1307 error = VOP_BMAP(sdp->swd_vp, byteoff / sdp->swd_bsize, 1308 &vp, &nbn, &nra); 1309 1310 if (error == 0 && nbn == (daddr_t)-1) { 1311 /* 1312 * this used to just set error, but that doesn't 1313 * do the right thing. Instead, it causes random 1314 * memory errors. The panic() should remain until 1315 * this condition doesn't destabilize the system. 1316 */ 1317 #if 1 1318 panic("%s: swap to sparse file", __func__); 1319 #else 1320 error = EIO; /* failure */ 1321 #endif 1322 } 1323 1324 /* 1325 * punt if there was an error or a hole in the file. 1326 * we must wait for any i/o ops we have already started 1327 * to finish before returning. 1328 * 1329 * XXX we could deal with holes here but it would be 1330 * a hassle (in the write case). 1331 */ 1332 if (error) { 1333 s = splbio(); 1334 vnx->vx_error = error; /* pass error up */ 1335 goto out; 1336 } 1337 1338 /* 1339 * compute the size ("sz") of this transfer (in bytes). 1340 */ 1341 off = byteoff % sdp->swd_bsize; 1342 sz = (1 + nra) * sdp->swd_bsize - off; 1343 if (sz > resid) 1344 sz = resid; 1345 1346 UVMHIST_LOG(pdhist, "sw_reg_strategy: " 1347 "vp %p/%p offset 0x%x/0x%x", 1348 sdp->swd_vp, vp, byteoff, nbn); 1349 1350 /* 1351 * now get a buf structure. note that the vb_buf is 1352 * at the front of the nbp structure so that you can 1353 * cast pointers between the two structure easily. 1354 */ 1355 nbp = pool_get(&vndbuf_pool, PR_WAITOK); 1356 buf_init(&nbp->vb_buf); 1357 nbp->vb_buf.b_flags = bp->b_flags; 1358 nbp->vb_buf.b_cflags = bp->b_cflags; 1359 nbp->vb_buf.b_oflags = bp->b_oflags; 1360 nbp->vb_buf.b_bcount = sz; 1361 nbp->vb_buf.b_bufsize = sz; 1362 nbp->vb_buf.b_error = 0; 1363 nbp->vb_buf.b_data = addr; 1364 nbp->vb_buf.b_lblkno = 0; 1365 nbp->vb_buf.b_blkno = nbn + btodb(off); 1366 nbp->vb_buf.b_rawblkno = nbp->vb_buf.b_blkno; 1367 nbp->vb_buf.b_iodone = sw_reg_biodone; 1368 nbp->vb_buf.b_vp = vp; 1369 nbp->vb_buf.b_objlock = &vp->v_interlock; 1370 if (vp->v_type == VBLK) { 1371 nbp->vb_buf.b_dev = vp->v_rdev; 1372 } 1373 1374 nbp->vb_xfer = vnx; /* patch it back in to vnx */ 1375 1376 /* 1377 * Just sort by block number 1378 */ 1379 s = splbio(); 1380 if (vnx->vx_error != 0) { 1381 buf_destroy(&nbp->vb_buf); 1382 pool_put(&vndbuf_pool, nbp); 1383 goto out; 1384 } 1385 vnx->vx_pending++; 1386 1387 /* sort it in and start I/O if we are not over our limit */ 1388 /* XXXAD locking */ 1389 bufq_put(sdp->swd_tab, &nbp->vb_buf); 1390 sw_reg_start(sdp); 1391 splx(s); 1392 1393 /* 1394 * advance to the next I/O 1395 */ 1396 byteoff += sz; 1397 addr += sz; 1398 } 1399 1400 s = splbio(); 1401 1402 out: /* Arrive here at splbio */ 1403 vnx->vx_flags &= ~VX_BUSY; 1404 if (vnx->vx_pending == 0) { 1405 error = vnx->vx_error; 1406 pool_put(&vndxfer_pool, vnx); 1407 bp->b_error = error; 1408 biodone(bp); 1409 } 1410 splx(s); 1411 } 1412 1413 /* 1414 * sw_reg_start: start an I/O request on the requested swapdev 1415 * 1416 * => reqs are sorted by b_rawblkno (above) 1417 */ 1418 static void 1419 sw_reg_start(struct swapdev *sdp) 1420 { 1421 struct buf *bp; 1422 struct vnode *vp; 1423 UVMHIST_FUNC("sw_reg_start"); UVMHIST_CALLED(pdhist); 1424 1425 /* recursion control */ 1426 if ((sdp->swd_flags & SWF_BUSY) != 0) 1427 return; 1428 1429 sdp->swd_flags |= SWF_BUSY; 1430 1431 while (sdp->swd_active < sdp->swd_maxactive) { 1432 bp = bufq_get(sdp->swd_tab); 1433 if (bp == NULL) 1434 break; 1435 sdp->swd_active++; 1436 1437 UVMHIST_LOG(pdhist, 1438 "sw_reg_start: bp %p vp %p blkno %p cnt %lx", 1439 bp, bp->b_vp, bp->b_blkno, bp->b_bcount); 1440 vp = bp->b_vp; 1441 KASSERT(bp->b_objlock == &vp->v_interlock); 1442 if ((bp->b_flags & B_READ) == 0) { 1443 mutex_enter(&vp->v_interlock); 1444 vp->v_numoutput++; 1445 mutex_exit(&vp->v_interlock); 1446 } 1447 VOP_STRATEGY(vp, bp); 1448 } 1449 sdp->swd_flags &= ~SWF_BUSY; 1450 } 1451 1452 /* 1453 * sw_reg_biodone: one of our i/o's has completed 1454 */ 1455 static void 1456 sw_reg_biodone(struct buf *bp) 1457 { 1458 workqueue_enqueue(sw_reg_workqueue, &bp->b_work, NULL); 1459 } 1460 1461 /* 1462 * sw_reg_iodone: one of our i/o's has completed and needs post-i/o cleanup 1463 * 1464 * => note that we can recover the vndbuf struct by casting the buf ptr 1465 */ 1466 static void 1467 sw_reg_iodone(struct work *wk, void *dummy) 1468 { 1469 struct vndbuf *vbp = (void *)wk; 1470 struct vndxfer *vnx = vbp->vb_xfer; 1471 struct buf *pbp = vnx->vx_bp; /* parent buffer */ 1472 struct swapdev *sdp = vnx->vx_sdp; 1473 int s, resid, error; 1474 KASSERT(&vbp->vb_buf.b_work == wk); 1475 UVMHIST_FUNC("sw_reg_iodone"); UVMHIST_CALLED(pdhist); 1476 1477 UVMHIST_LOG(pdhist, " vbp=%p vp=%p blkno=%x addr=%p", 1478 vbp, vbp->vb_buf.b_vp, vbp->vb_buf.b_blkno, vbp->vb_buf.b_data); 1479 UVMHIST_LOG(pdhist, " cnt=%lx resid=%lx", 1480 vbp->vb_buf.b_bcount, vbp->vb_buf.b_resid, 0, 0); 1481 1482 /* 1483 * protect vbp at splbio and update. 1484 */ 1485 1486 s = splbio(); 1487 resid = vbp->vb_buf.b_bcount - vbp->vb_buf.b_resid; 1488 pbp->b_resid -= resid; 1489 vnx->vx_pending--; 1490 1491 if (vbp->vb_buf.b_error != 0) { 1492 /* pass error upward */ 1493 error = vbp->vb_buf.b_error ? vbp->vb_buf.b_error : EIO; 1494 UVMHIST_LOG(pdhist, " got error=%d !", error, 0, 0, 0); 1495 vnx->vx_error = error; 1496 } 1497 1498 /* 1499 * kill vbp structure 1500 */ 1501 buf_destroy(&vbp->vb_buf); 1502 pool_put(&vndbuf_pool, vbp); 1503 1504 /* 1505 * wrap up this transaction if it has run to completion or, in 1506 * case of an error, when all auxiliary buffers have returned. 1507 */ 1508 if (vnx->vx_error != 0) { 1509 /* pass error upward */ 1510 error = vnx->vx_error; 1511 if ((vnx->vx_flags & VX_BUSY) == 0 && vnx->vx_pending == 0) { 1512 pbp->b_error = error; 1513 biodone(pbp); 1514 pool_put(&vndxfer_pool, vnx); 1515 } 1516 } else if (pbp->b_resid == 0) { 1517 KASSERT(vnx->vx_pending == 0); 1518 if ((vnx->vx_flags & VX_BUSY) == 0) { 1519 UVMHIST_LOG(pdhist, " iodone error=%d !", 1520 pbp, vnx->vx_error, 0, 0); 1521 biodone(pbp); 1522 pool_put(&vndxfer_pool, vnx); 1523 } 1524 } 1525 1526 /* 1527 * done! start next swapdev I/O if one is pending 1528 */ 1529 sdp->swd_active--; 1530 sw_reg_start(sdp); 1531 splx(s); 1532 } 1533 1534 1535 /* 1536 * uvm_swap_alloc: allocate space on swap 1537 * 1538 * => allocation is done "round robin" down the priority list, as we 1539 * allocate in a priority we "rotate" the circle queue. 1540 * => space can be freed with uvm_swap_free 1541 * => we return the page slot number in /dev/drum (0 == invalid slot) 1542 * => we lock uvm_swap_data_lock 1543 * => XXXMRG: "LESSOK" INTERFACE NEEDED TO EXTENT SYSTEM 1544 */ 1545 int 1546 uvm_swap_alloc(int *nslots /* IN/OUT */, bool lessok) 1547 { 1548 struct swapdev *sdp; 1549 struct swappri *spp; 1550 UVMHIST_FUNC("uvm_swap_alloc"); UVMHIST_CALLED(pdhist); 1551 1552 /* 1553 * no swap devices configured yet? definite failure. 1554 */ 1555 if (uvmexp.nswapdev < 1) 1556 return 0; 1557 1558 /* 1559 * lock data lock, convert slots into blocks, and enter loop 1560 */ 1561 mutex_enter(&uvm_swap_data_lock); 1562 1563 ReTry: /* XXXMRG */ 1564 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 1565 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 1566 uint64_t result; 1567 1568 /* if it's not enabled, then we can't swap from it */ 1569 if ((sdp->swd_flags & SWF_ENABLE) == 0) 1570 continue; 1571 if (sdp->swd_npginuse + *nslots > sdp->swd_npages) 1572 continue; 1573 result = blist_alloc(sdp->swd_blist, *nslots); 1574 if (result == BLIST_NONE) { 1575 continue; 1576 } 1577 KASSERT(result < sdp->swd_drumsize); 1578 1579 /* 1580 * successful allocation! now rotate the circleq. 1581 */ 1582 CIRCLEQ_REMOVE(&spp->spi_swapdev, sdp, swd_next); 1583 CIRCLEQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next); 1584 sdp->swd_npginuse += *nslots; 1585 uvmexp.swpginuse += *nslots; 1586 mutex_exit(&uvm_swap_data_lock); 1587 /* done! return drum slot number */ 1588 UVMHIST_LOG(pdhist, 1589 "success! returning %d slots starting at %d", 1590 *nslots, result + sdp->swd_drumoffset, 0, 0); 1591 return (result + sdp->swd_drumoffset); 1592 } 1593 } 1594 1595 /* XXXMRG: BEGIN HACK */ 1596 if (*nslots > 1 && lessok) { 1597 *nslots = 1; 1598 /* XXXMRG: ugh! blist should support this for us */ 1599 goto ReTry; 1600 } 1601 /* XXXMRG: END HACK */ 1602 1603 mutex_exit(&uvm_swap_data_lock); 1604 return 0; 1605 } 1606 1607 /* 1608 * uvm_swapisfull: return true if most of available swap is allocated 1609 * and in use. we don't count some small portion as it may be inaccessible 1610 * to us at any given moment, for example if there is lock contention or if 1611 * pages are busy. 1612 */ 1613 bool 1614 uvm_swapisfull(void) 1615 { 1616 int swpgonly; 1617 bool rv; 1618 1619 mutex_enter(&uvm_swap_data_lock); 1620 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1621 swpgonly = (int)((uint64_t)uvmexp.swpgonly * 100 / 1622 uvm_swapisfull_factor); 1623 rv = (swpgonly >= uvmexp.swpgavail); 1624 mutex_exit(&uvm_swap_data_lock); 1625 1626 return (rv); 1627 } 1628 1629 /* 1630 * uvm_swap_markbad: keep track of swap ranges where we've had i/o errors 1631 * 1632 * => we lock uvm_swap_data_lock 1633 */ 1634 void 1635 uvm_swap_markbad(int startslot, int nslots) 1636 { 1637 struct swapdev *sdp; 1638 UVMHIST_FUNC("uvm_swap_markbad"); UVMHIST_CALLED(pdhist); 1639 1640 mutex_enter(&uvm_swap_data_lock); 1641 sdp = swapdrum_getsdp(startslot); 1642 KASSERT(sdp != NULL); 1643 1644 /* 1645 * we just keep track of how many pages have been marked bad 1646 * in this device, to make everything add up in swap_off(). 1647 * we assume here that the range of slots will all be within 1648 * one swap device. 1649 */ 1650 1651 KASSERT(uvmexp.swpgonly >= nslots); 1652 uvmexp.swpgonly -= nslots; 1653 sdp->swd_npgbad += nslots; 1654 UVMHIST_LOG(pdhist, "now %d bad", sdp->swd_npgbad, 0,0,0); 1655 mutex_exit(&uvm_swap_data_lock); 1656 } 1657 1658 /* 1659 * uvm_swap_free: free swap slots 1660 * 1661 * => this can be all or part of an allocation made by uvm_swap_alloc 1662 * => we lock uvm_swap_data_lock 1663 */ 1664 void 1665 uvm_swap_free(int startslot, int nslots) 1666 { 1667 struct swapdev *sdp; 1668 UVMHIST_FUNC("uvm_swap_free"); UVMHIST_CALLED(pdhist); 1669 1670 UVMHIST_LOG(pdhist, "freeing %d slots starting at %d", nslots, 1671 startslot, 0, 0); 1672 1673 /* 1674 * ignore attempts to free the "bad" slot. 1675 */ 1676 1677 if (startslot == SWSLOT_BAD) { 1678 return; 1679 } 1680 1681 /* 1682 * convert drum slot offset back to sdp, free the blocks 1683 * in the extent, and return. must hold pri lock to do 1684 * lookup and access the extent. 1685 */ 1686 1687 mutex_enter(&uvm_swap_data_lock); 1688 sdp = swapdrum_getsdp(startslot); 1689 KASSERT(uvmexp.nswapdev >= 1); 1690 KASSERT(sdp != NULL); 1691 KASSERT(sdp->swd_npginuse >= nslots); 1692 blist_free(sdp->swd_blist, startslot - sdp->swd_drumoffset, nslots); 1693 sdp->swd_npginuse -= nslots; 1694 uvmexp.swpginuse -= nslots; 1695 mutex_exit(&uvm_swap_data_lock); 1696 } 1697 1698 /* 1699 * uvm_swap_put: put any number of pages into a contig place on swap 1700 * 1701 * => can be sync or async 1702 */ 1703 1704 int 1705 uvm_swap_put(int swslot, struct vm_page **ppsp, int npages, int flags) 1706 { 1707 int error; 1708 1709 error = uvm_swap_io(ppsp, swslot, npages, B_WRITE | 1710 ((flags & PGO_SYNCIO) ? 0 : B_ASYNC)); 1711 return error; 1712 } 1713 1714 /* 1715 * uvm_swap_get: get a single page from swap 1716 * 1717 * => usually a sync op (from fault) 1718 */ 1719 1720 int 1721 uvm_swap_get(struct vm_page *page, int swslot, int flags) 1722 { 1723 int error; 1724 1725 uvmexp.nswget++; 1726 KASSERT(flags & PGO_SYNCIO); 1727 if (swslot == SWSLOT_BAD) { 1728 return EIO; 1729 } 1730 1731 error = uvm_swap_io(&page, swslot, 1, B_READ | 1732 ((flags & PGO_SYNCIO) ? 0 : B_ASYNC)); 1733 if (error == 0) { 1734 1735 /* 1736 * this page is no longer only in swap. 1737 */ 1738 1739 mutex_enter(&uvm_swap_data_lock); 1740 KASSERT(uvmexp.swpgonly > 0); 1741 uvmexp.swpgonly--; 1742 mutex_exit(&uvm_swap_data_lock); 1743 } 1744 return error; 1745 } 1746 1747 /* 1748 * uvm_swap_io: do an i/o operation to swap 1749 */ 1750 1751 static int 1752 uvm_swap_io(struct vm_page **pps, int startslot, int npages, int flags) 1753 { 1754 daddr_t startblk; 1755 struct buf *bp; 1756 vaddr_t kva; 1757 int error, mapinflags; 1758 bool write, async; 1759 UVMHIST_FUNC("uvm_swap_io"); UVMHIST_CALLED(pdhist); 1760 1761 UVMHIST_LOG(pdhist, "<- called, startslot=%d, npages=%d, flags=%d", 1762 startslot, npages, flags, 0); 1763 1764 write = (flags & B_READ) == 0; 1765 async = (flags & B_ASYNC) != 0; 1766 1767 /* 1768 * allocate a buf for the i/o. 1769 */ 1770 1771 KASSERT(curlwp != uvm.pagedaemon_lwp || (write && async)); 1772 bp = getiobuf(swapdev_vp, curlwp != uvm.pagedaemon_lwp); 1773 if (bp == NULL) { 1774 uvm_aio_aiodone_pages(pps, npages, true, ENOMEM); 1775 return ENOMEM; 1776 } 1777 1778 /* 1779 * convert starting drum slot to block number 1780 */ 1781 1782 startblk = btodb((uint64_t)startslot << PAGE_SHIFT); 1783 1784 /* 1785 * first, map the pages into the kernel. 1786 */ 1787 1788 mapinflags = !write ? 1789 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_READ : 1790 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_WRITE; 1791 kva = uvm_pagermapin(pps, npages, mapinflags); 1792 1793 /* 1794 * fill in the bp/sbp. we currently route our i/o through 1795 * /dev/drum's vnode [swapdev_vp]. 1796 */ 1797 1798 bp->b_cflags = BC_BUSY | BC_NOCACHE; 1799 bp->b_flags = (flags & (B_READ|B_ASYNC)); 1800 bp->b_proc = &proc0; /* XXX */ 1801 bp->b_vnbufs.le_next = NOLIST; 1802 bp->b_data = (void *)kva; 1803 bp->b_blkno = startblk; 1804 bp->b_bufsize = bp->b_bcount = npages << PAGE_SHIFT; 1805 1806 /* 1807 * bump v_numoutput (counter of number of active outputs). 1808 */ 1809 1810 if (write) { 1811 mutex_enter(&swapdev_vp->v_interlock); 1812 swapdev_vp->v_numoutput++; 1813 mutex_exit(&swapdev_vp->v_interlock); 1814 } 1815 1816 /* 1817 * for async ops we must set up the iodone handler. 1818 */ 1819 1820 if (async) { 1821 bp->b_iodone = uvm_aio_biodone; 1822 UVMHIST_LOG(pdhist, "doing async!", 0, 0, 0, 0); 1823 if (curlwp == uvm.pagedaemon_lwp) 1824 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); 1825 else 1826 BIO_SETPRIO(bp, BPRIO_TIMELIMITED); 1827 } else { 1828 bp->b_iodone = NULL; 1829 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); 1830 } 1831 UVMHIST_LOG(pdhist, 1832 "about to start io: data = %p blkno = 0x%x, bcount = %ld", 1833 bp->b_data, bp->b_blkno, bp->b_bcount, 0); 1834 1835 /* 1836 * now we start the I/O, and if async, return. 1837 */ 1838 1839 VOP_STRATEGY(swapdev_vp, bp); 1840 if (async) 1841 return 0; 1842 1843 /* 1844 * must be sync i/o. wait for it to finish 1845 */ 1846 1847 error = biowait(bp); 1848 1849 /* 1850 * kill the pager mapping 1851 */ 1852 1853 uvm_pagermapout(kva, npages); 1854 1855 /* 1856 * now dispose of the buf and we're done. 1857 */ 1858 1859 if (write) { 1860 mutex_enter(&swapdev_vp->v_interlock); 1861 vwakeup(bp); 1862 mutex_exit(&swapdev_vp->v_interlock); 1863 } 1864 putiobuf(bp); 1865 UVMHIST_LOG(pdhist, "<- done (sync) error=%d", error, 0, 0, 0); 1866 1867 return (error); 1868 } 1869