1 /* $NetBSD: uvm_swap.c,v 1.139 2008/05/29 14:51:27 mrg Exp $ */ 2 3 /* 4 * Copyright (c) 1995, 1996, 1997 Matthew R. Green 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * from: NetBSD: vm_swap.c,v 1.52 1997/12/02 13:47:37 pk Exp 29 * from: Id: uvm_swap.c,v 1.1.2.42 1998/02/02 20:38:06 chuck Exp 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.139 2008/05/29 14:51:27 mrg Exp $"); 34 35 #include "fs_nfs.h" 36 #include "opt_uvmhist.h" 37 #include "opt_compat_netbsd.h" 38 #include "opt_ddb.h" 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #include <sys/buf.h> 43 #include <sys/bufq.h> 44 #include <sys/conf.h> 45 #include <sys/proc.h> 46 #include <sys/namei.h> 47 #include <sys/disklabel.h> 48 #include <sys/errno.h> 49 #include <sys/kernel.h> 50 #include <sys/malloc.h> 51 #include <sys/vnode.h> 52 #include <sys/file.h> 53 #include <sys/vmem.h> 54 #include <sys/blist.h> 55 #include <sys/mount.h> 56 #include <sys/pool.h> 57 #include <sys/syscallargs.h> 58 #include <sys/swap.h> 59 #include <sys/kauth.h> 60 #include <sys/sysctl.h> 61 #include <sys/workqueue.h> 62 63 #include <uvm/uvm.h> 64 65 #include <miscfs/specfs/specdev.h> 66 67 /* 68 * uvm_swap.c: manage configuration and i/o to swap space. 69 */ 70 71 /* 72 * swap space is managed in the following way: 73 * 74 * each swap partition or file is described by a "swapdev" structure. 75 * each "swapdev" structure contains a "swapent" structure which contains 76 * information that is passed up to the user (via system calls). 77 * 78 * each swap partition is assigned a "priority" (int) which controls 79 * swap parition usage. 80 * 81 * the system maintains a global data structure describing all swap 82 * partitions/files. there is a sorted LIST of "swappri" structures 83 * which describe "swapdev"'s at that priority. this LIST is headed 84 * by the "swap_priority" global var. each "swappri" contains a 85 * CIRCLEQ of "swapdev" structures at that priority. 86 * 87 * locking: 88 * - swap_syscall_lock (krwlock_t): this lock serializes the swapctl 89 * system call and prevents the swap priority list from changing 90 * while we are in the middle of a system call (e.g. SWAP_STATS). 91 * - uvm_swap_data_lock (kmutex_t): this lock protects all swap data 92 * structures including the priority list, the swapdev structures, 93 * and the swapmap arena. 94 * 95 * each swap device has the following info: 96 * - swap device in use (could be disabled, preventing future use) 97 * - swap enabled (allows new allocations on swap) 98 * - map info in /dev/drum 99 * - vnode pointer 100 * for swap files only: 101 * - block size 102 * - max byte count in buffer 103 * - buffer 104 * 105 * userland controls and configures swap with the swapctl(2) system call. 106 * the sys_swapctl performs the following operations: 107 * [1] SWAP_NSWAP: returns the number of swap devices currently configured 108 * [2] SWAP_STATS: given a pointer to an array of swapent structures 109 * (passed in via "arg") of a size passed in via "misc" ... we load 110 * the current swap config into the array. The actual work is done 111 * in the uvm_swap_stats(9) function. 112 * [3] SWAP_ON: given a pathname in arg (could be device or file) and a 113 * priority in "misc", start swapping on it. 114 * [4] SWAP_OFF: as SWAP_ON, but stops swapping to a device 115 * [5] SWAP_CTL: changes the priority of a swap device (new priority in 116 * "misc") 117 */ 118 119 /* 120 * swapdev: describes a single swap partition/file 121 * 122 * note the following should be true: 123 * swd_inuse <= swd_nblks [number of blocks in use is <= total blocks] 124 * swd_nblks <= swd_mapsize [because mapsize includes miniroot+disklabel] 125 */ 126 struct swapdev { 127 struct oswapent swd_ose; 128 #define swd_dev swd_ose.ose_dev /* device id */ 129 #define swd_flags swd_ose.ose_flags /* flags:inuse/enable/fake */ 130 #define swd_priority swd_ose.ose_priority /* our priority */ 131 /* also: swd_ose.ose_nblks, swd_ose.ose_inuse */ 132 char *swd_path; /* saved pathname of device */ 133 int swd_pathlen; /* length of pathname */ 134 int swd_npages; /* #pages we can use */ 135 int swd_npginuse; /* #pages in use */ 136 int swd_npgbad; /* #pages bad */ 137 int swd_drumoffset; /* page0 offset in drum */ 138 int swd_drumsize; /* #pages in drum */ 139 blist_t swd_blist; /* blist for this swapdev */ 140 struct vnode *swd_vp; /* backing vnode */ 141 CIRCLEQ_ENTRY(swapdev) swd_next; /* priority circleq */ 142 143 int swd_bsize; /* blocksize (bytes) */ 144 int swd_maxactive; /* max active i/o reqs */ 145 struct bufq_state *swd_tab; /* buffer list */ 146 int swd_active; /* number of active buffers */ 147 }; 148 149 /* 150 * swap device priority entry; the list is kept sorted on `spi_priority'. 151 */ 152 struct swappri { 153 int spi_priority; /* priority */ 154 CIRCLEQ_HEAD(spi_swapdev, swapdev) spi_swapdev; 155 /* circleq of swapdevs at this priority */ 156 LIST_ENTRY(swappri) spi_swappri; /* global list of pri's */ 157 }; 158 159 /* 160 * The following two structures are used to keep track of data transfers 161 * on swap devices associated with regular files. 162 * NOTE: this code is more or less a copy of vnd.c; we use the same 163 * structure names here to ease porting.. 164 */ 165 struct vndxfer { 166 struct buf *vx_bp; /* Pointer to parent buffer */ 167 struct swapdev *vx_sdp; 168 int vx_error; 169 int vx_pending; /* # of pending aux buffers */ 170 int vx_flags; 171 #define VX_BUSY 1 172 #define VX_DEAD 2 173 }; 174 175 struct vndbuf { 176 struct buf vb_buf; 177 struct vndxfer *vb_xfer; 178 }; 179 180 181 /* 182 * We keep a of pool vndbuf's and vndxfer structures. 183 */ 184 POOL_INIT(vndxfer_pool, sizeof(struct vndxfer), 0, 0, 0, "swp vnx", NULL, 185 IPL_BIO); 186 POOL_INIT(vndbuf_pool, sizeof(struct vndbuf), 0, 0, 0, "swp vnd", NULL, 187 IPL_BIO); 188 189 /* 190 * local variables 191 */ 192 MALLOC_DEFINE(M_VMSWAP, "VM swap", "VM swap structures"); 193 static vmem_t *swapmap; /* controls the mapping of /dev/drum */ 194 195 /* list of all active swap devices [by priority] */ 196 LIST_HEAD(swap_priority, swappri); 197 static struct swap_priority swap_priority; 198 199 /* locks */ 200 static krwlock_t swap_syscall_lock; 201 202 /* workqueue and use counter for swap to regular files */ 203 static int sw_reg_count = 0; 204 static struct workqueue *sw_reg_workqueue; 205 206 /* 207 * prototypes 208 */ 209 static struct swapdev *swapdrum_getsdp(int); 210 211 static struct swapdev *swaplist_find(struct vnode *, bool); 212 static void swaplist_insert(struct swapdev *, 213 struct swappri *, int); 214 static void swaplist_trim(void); 215 216 static int swap_on(struct lwp *, struct swapdev *); 217 static int swap_off(struct lwp *, struct swapdev *); 218 219 static void uvm_swap_stats_locked(int, struct swapent *, int, register_t *); 220 221 static void sw_reg_strategy(struct swapdev *, struct buf *, int); 222 static void sw_reg_biodone(struct buf *); 223 static void sw_reg_iodone(struct work *wk, void *dummy); 224 static void sw_reg_start(struct swapdev *); 225 226 static int uvm_swap_io(struct vm_page **, int, int, int); 227 228 /* 229 * uvm_swap_init: init the swap system data structures and locks 230 * 231 * => called at boot time from init_main.c after the filesystems 232 * are brought up (which happens after uvm_init()) 233 */ 234 void 235 uvm_swap_init(void) 236 { 237 UVMHIST_FUNC("uvm_swap_init"); 238 239 UVMHIST_CALLED(pdhist); 240 /* 241 * first, init the swap list, its counter, and its lock. 242 * then get a handle on the vnode for /dev/drum by using 243 * the its dev_t number ("swapdev", from MD conf.c). 244 */ 245 246 LIST_INIT(&swap_priority); 247 uvmexp.nswapdev = 0; 248 rw_init(&swap_syscall_lock); 249 cv_init(&uvm.scheduler_cv, "schedule"); 250 mutex_init(&uvm_swap_data_lock, MUTEX_DEFAULT, IPL_NONE); 251 252 /* XXXSMP should be at IPL_VM, but for audio interrupt handlers. */ 253 mutex_init(&uvm_scheduler_mutex, MUTEX_SPIN, IPL_SCHED); 254 255 if (bdevvp(swapdev, &swapdev_vp)) 256 panic("uvm_swap_init: can't get vnode for swap device"); 257 if (vn_lock(swapdev_vp, LK_EXCLUSIVE | LK_RETRY)) 258 panic("uvm_swap_init: can't lock swap device"); 259 if (VOP_OPEN(swapdev_vp, FREAD | FWRITE, NOCRED)) 260 panic("uvm_swap_init: can't open swap device"); 261 VOP_UNLOCK(swapdev_vp, 0); 262 263 /* 264 * create swap block resource map to map /dev/drum. the range 265 * from 1 to INT_MAX allows 2 gigablocks of swap space. note 266 * that block 0 is reserved (used to indicate an allocation 267 * failure, or no allocation). 268 */ 269 swapmap = vmem_create("swapmap", 1, INT_MAX - 1, 1, NULL, NULL, NULL, 0, 270 VM_NOSLEEP, IPL_NONE); 271 if (swapmap == 0) 272 panic("uvm_swap_init: extent_create failed"); 273 274 /* 275 * done! 276 */ 277 uvm.swap_running = true; 278 uvm.swapout_enabled = 1; 279 UVMHIST_LOG(pdhist, "<- done", 0, 0, 0, 0); 280 281 sysctl_createv(NULL, 0, NULL, NULL, 282 CTLFLAG_READWRITE, 283 CTLTYPE_INT, "swapout", 284 SYSCTL_DESCR("Set 0 to disable swapout of kernel stacks"), 285 NULL, 0, &uvm.swapout_enabled, 0, CTL_VM, CTL_CREATE, CTL_EOL); 286 } 287 288 /* 289 * swaplist functions: functions that operate on the list of swap 290 * devices on the system. 291 */ 292 293 /* 294 * swaplist_insert: insert swap device "sdp" into the global list 295 * 296 * => caller must hold both swap_syscall_lock and uvm_swap_data_lock 297 * => caller must provide a newly malloc'd swappri structure (we will 298 * FREE it if we don't need it... this it to prevent malloc blocking 299 * here while adding swap) 300 */ 301 static void 302 swaplist_insert(struct swapdev *sdp, struct swappri *newspp, int priority) 303 { 304 struct swappri *spp, *pspp; 305 UVMHIST_FUNC("swaplist_insert"); UVMHIST_CALLED(pdhist); 306 307 /* 308 * find entry at or after which to insert the new device. 309 */ 310 pspp = NULL; 311 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 312 if (priority <= spp->spi_priority) 313 break; 314 pspp = spp; 315 } 316 317 /* 318 * new priority? 319 */ 320 if (spp == NULL || spp->spi_priority != priority) { 321 spp = newspp; /* use newspp! */ 322 UVMHIST_LOG(pdhist, "created new swappri = %d", 323 priority, 0, 0, 0); 324 325 spp->spi_priority = priority; 326 CIRCLEQ_INIT(&spp->spi_swapdev); 327 328 if (pspp) 329 LIST_INSERT_AFTER(pspp, spp, spi_swappri); 330 else 331 LIST_INSERT_HEAD(&swap_priority, spp, spi_swappri); 332 } else { 333 /* we don't need a new priority structure, free it */ 334 FREE(newspp, M_VMSWAP); 335 } 336 337 /* 338 * priority found (or created). now insert on the priority's 339 * circleq list and bump the total number of swapdevs. 340 */ 341 sdp->swd_priority = priority; 342 CIRCLEQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next); 343 uvmexp.nswapdev++; 344 } 345 346 /* 347 * swaplist_find: find and optionally remove a swap device from the 348 * global list. 349 * 350 * => caller must hold both swap_syscall_lock and uvm_swap_data_lock 351 * => we return the swapdev we found (and removed) 352 */ 353 static struct swapdev * 354 swaplist_find(struct vnode *vp, bool remove) 355 { 356 struct swapdev *sdp; 357 struct swappri *spp; 358 359 /* 360 * search the lists for the requested vp 361 */ 362 363 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 364 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 365 if (sdp->swd_vp == vp) { 366 if (remove) { 367 CIRCLEQ_REMOVE(&spp->spi_swapdev, 368 sdp, swd_next); 369 uvmexp.nswapdev--; 370 } 371 return(sdp); 372 } 373 } 374 } 375 return (NULL); 376 } 377 378 /* 379 * swaplist_trim: scan priority list for empty priority entries and kill 380 * them. 381 * 382 * => caller must hold both swap_syscall_lock and uvm_swap_data_lock 383 */ 384 static void 385 swaplist_trim(void) 386 { 387 struct swappri *spp, *nextspp; 388 389 for (spp = LIST_FIRST(&swap_priority); spp != NULL; spp = nextspp) { 390 nextspp = LIST_NEXT(spp, spi_swappri); 391 if (CIRCLEQ_FIRST(&spp->spi_swapdev) != 392 (void *)&spp->spi_swapdev) 393 continue; 394 LIST_REMOVE(spp, spi_swappri); 395 free(spp, M_VMSWAP); 396 } 397 } 398 399 /* 400 * swapdrum_getsdp: given a page offset in /dev/drum, convert it back 401 * to the "swapdev" that maps that section of the drum. 402 * 403 * => each swapdev takes one big contig chunk of the drum 404 * => caller must hold uvm_swap_data_lock 405 */ 406 static struct swapdev * 407 swapdrum_getsdp(int pgno) 408 { 409 struct swapdev *sdp; 410 struct swappri *spp; 411 412 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 413 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 414 if (sdp->swd_flags & SWF_FAKE) 415 continue; 416 if (pgno >= sdp->swd_drumoffset && 417 pgno < (sdp->swd_drumoffset + sdp->swd_drumsize)) { 418 return sdp; 419 } 420 } 421 } 422 return NULL; 423 } 424 425 426 /* 427 * sys_swapctl: main entry point for swapctl(2) system call 428 * [with two helper functions: swap_on and swap_off] 429 */ 430 int 431 sys_swapctl(struct lwp *l, const struct sys_swapctl_args *uap, register_t *retval) 432 { 433 /* { 434 syscallarg(int) cmd; 435 syscallarg(void *) arg; 436 syscallarg(int) misc; 437 } */ 438 struct vnode *vp; 439 struct nameidata nd; 440 struct swappri *spp; 441 struct swapdev *sdp; 442 struct swapent *sep; 443 #define SWAP_PATH_MAX (PATH_MAX + 1) 444 char *userpath; 445 size_t len; 446 int error, misc; 447 int priority; 448 UVMHIST_FUNC("sys_swapctl"); UVMHIST_CALLED(pdhist); 449 450 misc = SCARG(uap, misc); 451 452 /* 453 * ensure serialized syscall access by grabbing the swap_syscall_lock 454 */ 455 rw_enter(&swap_syscall_lock, RW_WRITER); 456 457 userpath = malloc(SWAP_PATH_MAX, M_TEMP, M_WAITOK); 458 /* 459 * we handle the non-priv NSWAP and STATS request first. 460 * 461 * SWAP_NSWAP: return number of config'd swap devices 462 * [can also be obtained with uvmexp sysctl] 463 */ 464 if (SCARG(uap, cmd) == SWAP_NSWAP) { 465 UVMHIST_LOG(pdhist, "<- done SWAP_NSWAP=%d", uvmexp.nswapdev, 466 0, 0, 0); 467 *retval = uvmexp.nswapdev; 468 error = 0; 469 goto out; 470 } 471 472 /* 473 * SWAP_STATS: get stats on current # of configured swap devs 474 * 475 * note that the swap_priority list can't change as long 476 * as we are holding the swap_syscall_lock. we don't want 477 * to grab the uvm_swap_data_lock because we may fault&sleep during 478 * copyout() and we don't want to be holding that lock then! 479 */ 480 if (SCARG(uap, cmd) == SWAP_STATS 481 #if defined(COMPAT_13) 482 || SCARG(uap, cmd) == SWAP_OSTATS 483 #endif 484 ) { 485 if ((size_t)misc > (size_t)uvmexp.nswapdev) 486 misc = uvmexp.nswapdev; 487 #if defined(COMPAT_13) 488 if (SCARG(uap, cmd) == SWAP_OSTATS) 489 len = sizeof(struct oswapent) * misc; 490 else 491 #endif 492 len = sizeof(struct swapent) * misc; 493 sep = (struct swapent *)malloc(len, M_TEMP, M_WAITOK); 494 495 uvm_swap_stats_locked(SCARG(uap, cmd), sep, misc, retval); 496 error = copyout(sep, SCARG(uap, arg), len); 497 498 free(sep, M_TEMP); 499 UVMHIST_LOG(pdhist, "<- done SWAP_STATS", 0, 0, 0, 0); 500 goto out; 501 } 502 if (SCARG(uap, cmd) == SWAP_GETDUMPDEV) { 503 dev_t *devp = (dev_t *)SCARG(uap, arg); 504 505 error = copyout(&dumpdev, devp, sizeof(dumpdev)); 506 goto out; 507 } 508 509 /* 510 * all other requests require superuser privs. verify. 511 */ 512 if ((error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_SWAPCTL, 513 0, NULL, NULL, NULL))) 514 goto out; 515 516 if (SCARG(uap, cmd) == SWAP_DUMPOFF) { 517 /* drop the current dump device */ 518 dumpdev = NODEV; 519 dumpcdev = NODEV; 520 cpu_dumpconf(); 521 goto out; 522 } 523 524 /* 525 * at this point we expect a path name in arg. we will 526 * use namei() to gain a vnode reference (vref), and lock 527 * the vnode (VOP_LOCK). 528 * 529 * XXX: a NULL arg means use the root vnode pointer (e.g. for 530 * miniroot) 531 */ 532 if (SCARG(uap, arg) == NULL) { 533 vp = rootvp; /* miniroot */ 534 if (vget(vp, LK_EXCLUSIVE)) { 535 error = EBUSY; 536 goto out; 537 } 538 if (SCARG(uap, cmd) == SWAP_ON && 539 copystr("miniroot", userpath, SWAP_PATH_MAX, &len)) 540 panic("swapctl: miniroot copy failed"); 541 } else { 542 int space; 543 char *where; 544 545 if (SCARG(uap, cmd) == SWAP_ON) { 546 if ((error = copyinstr(SCARG(uap, arg), userpath, 547 SWAP_PATH_MAX, &len))) 548 goto out; 549 space = UIO_SYSSPACE; 550 where = userpath; 551 } else { 552 space = UIO_USERSPACE; 553 where = (char *)SCARG(uap, arg); 554 } 555 NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | TRYEMULROOT, 556 space, where); 557 if ((error = namei(&nd))) 558 goto out; 559 vp = nd.ni_vp; 560 } 561 /* note: "vp" is referenced and locked */ 562 563 error = 0; /* assume no error */ 564 switch(SCARG(uap, cmd)) { 565 566 case SWAP_DUMPDEV: 567 if (vp->v_type != VBLK) { 568 error = ENOTBLK; 569 break; 570 } 571 if (bdevsw_lookup(vp->v_rdev)) { 572 dumpdev = vp->v_rdev; 573 dumpcdev = devsw_blk2chr(dumpdev); 574 } else 575 dumpdev = NODEV; 576 cpu_dumpconf(); 577 break; 578 579 case SWAP_CTL: 580 /* 581 * get new priority, remove old entry (if any) and then 582 * reinsert it in the correct place. finally, prune out 583 * any empty priority structures. 584 */ 585 priority = SCARG(uap, misc); 586 spp = malloc(sizeof *spp, M_VMSWAP, M_WAITOK); 587 mutex_enter(&uvm_swap_data_lock); 588 if ((sdp = swaplist_find(vp, true)) == NULL) { 589 error = ENOENT; 590 } else { 591 swaplist_insert(sdp, spp, priority); 592 swaplist_trim(); 593 } 594 mutex_exit(&uvm_swap_data_lock); 595 if (error) 596 free(spp, M_VMSWAP); 597 break; 598 599 case SWAP_ON: 600 601 /* 602 * check for duplicates. if none found, then insert a 603 * dummy entry on the list to prevent someone else from 604 * trying to enable this device while we are working on 605 * it. 606 */ 607 608 priority = SCARG(uap, misc); 609 sdp = malloc(sizeof *sdp, M_VMSWAP, M_WAITOK); 610 spp = malloc(sizeof *spp, M_VMSWAP, M_WAITOK); 611 memset(sdp, 0, sizeof(*sdp)); 612 sdp->swd_flags = SWF_FAKE; 613 sdp->swd_vp = vp; 614 sdp->swd_dev = (vp->v_type == VBLK) ? vp->v_rdev : NODEV; 615 bufq_alloc(&sdp->swd_tab, "disksort", BUFQ_SORT_RAWBLOCK); 616 mutex_enter(&uvm_swap_data_lock); 617 if (swaplist_find(vp, false) != NULL) { 618 error = EBUSY; 619 mutex_exit(&uvm_swap_data_lock); 620 bufq_free(sdp->swd_tab); 621 free(sdp, M_VMSWAP); 622 free(spp, M_VMSWAP); 623 break; 624 } 625 swaplist_insert(sdp, spp, priority); 626 mutex_exit(&uvm_swap_data_lock); 627 628 sdp->swd_pathlen = len; 629 sdp->swd_path = malloc(sdp->swd_pathlen, M_VMSWAP, M_WAITOK); 630 if (copystr(userpath, sdp->swd_path, sdp->swd_pathlen, 0) != 0) 631 panic("swapctl: copystr"); 632 633 /* 634 * we've now got a FAKE placeholder in the swap list. 635 * now attempt to enable swap on it. if we fail, undo 636 * what we've done and kill the fake entry we just inserted. 637 * if swap_on is a success, it will clear the SWF_FAKE flag 638 */ 639 640 if ((error = swap_on(l, sdp)) != 0) { 641 mutex_enter(&uvm_swap_data_lock); 642 (void) swaplist_find(vp, true); /* kill fake entry */ 643 swaplist_trim(); 644 mutex_exit(&uvm_swap_data_lock); 645 bufq_free(sdp->swd_tab); 646 free(sdp->swd_path, M_VMSWAP); 647 free(sdp, M_VMSWAP); 648 break; 649 } 650 break; 651 652 case SWAP_OFF: 653 mutex_enter(&uvm_swap_data_lock); 654 if ((sdp = swaplist_find(vp, false)) == NULL) { 655 mutex_exit(&uvm_swap_data_lock); 656 error = ENXIO; 657 break; 658 } 659 660 /* 661 * If a device isn't in use or enabled, we 662 * can't stop swapping from it (again). 663 */ 664 if ((sdp->swd_flags & (SWF_INUSE|SWF_ENABLE)) == 0) { 665 mutex_exit(&uvm_swap_data_lock); 666 error = EBUSY; 667 break; 668 } 669 670 /* 671 * do the real work. 672 */ 673 error = swap_off(l, sdp); 674 break; 675 676 default: 677 error = EINVAL; 678 } 679 680 /* 681 * done! release the ref gained by namei() and unlock. 682 */ 683 vput(vp); 684 685 out: 686 free(userpath, M_TEMP); 687 rw_exit(&swap_syscall_lock); 688 689 UVMHIST_LOG(pdhist, "<- done! error=%d", error, 0, 0, 0); 690 return (error); 691 } 692 693 /* 694 * swap_stats: implements swapctl(SWAP_STATS). The function is kept 695 * away from sys_swapctl() in order to allow COMPAT_* swapctl() 696 * emulation to use it directly without going through sys_swapctl(). 697 * The problem with using sys_swapctl() there is that it involves 698 * copying the swapent array to the stackgap, and this array's size 699 * is not known at build time. Hence it would not be possible to 700 * ensure it would fit in the stackgap in any case. 701 */ 702 void 703 uvm_swap_stats(int cmd, struct swapent *sep, int sec, register_t *retval) 704 { 705 706 rw_enter(&swap_syscall_lock, RW_READER); 707 uvm_swap_stats_locked(cmd, sep, sec, retval); 708 rw_exit(&swap_syscall_lock); 709 } 710 711 static void 712 uvm_swap_stats_locked(int cmd, struct swapent *sep, int sec, register_t *retval) 713 { 714 struct swappri *spp; 715 struct swapdev *sdp; 716 int count = 0; 717 718 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 719 for (sdp = CIRCLEQ_FIRST(&spp->spi_swapdev); 720 sdp != (void *)&spp->spi_swapdev && sec-- > 0; 721 sdp = CIRCLEQ_NEXT(sdp, swd_next)) { 722 /* 723 * backwards compatibility for system call. 724 * note that we use 'struct oswapent' as an 725 * overlay into both 'struct swapdev' and 726 * the userland 'struct swapent', as we 727 * want to retain backwards compatibility 728 * with NetBSD 1.3. 729 */ 730 sdp->swd_ose.ose_inuse = 731 btodb((uint64_t)sdp->swd_npginuse << 732 PAGE_SHIFT); 733 (void)memcpy(sep, &sdp->swd_ose, 734 sizeof(struct oswapent)); 735 736 /* now copy out the path if necessary */ 737 #if !defined(COMPAT_13) 738 (void) cmd; 739 #endif 740 #if defined(COMPAT_13) 741 if (cmd == SWAP_STATS) 742 #endif 743 (void)memcpy(&sep->se_path, sdp->swd_path, 744 sdp->swd_pathlen); 745 746 count++; 747 #if defined(COMPAT_13) 748 if (cmd == SWAP_OSTATS) 749 sep = (struct swapent *) 750 ((struct oswapent *)sep + 1); 751 else 752 #endif 753 sep++; 754 } 755 } 756 757 *retval = count; 758 return; 759 } 760 761 /* 762 * swap_on: attempt to enable a swapdev for swapping. note that the 763 * swapdev is already on the global list, but disabled (marked 764 * SWF_FAKE). 765 * 766 * => we avoid the start of the disk (to protect disk labels) 767 * => we also avoid the miniroot, if we are swapping to root. 768 * => caller should leave uvm_swap_data_lock unlocked, we may lock it 769 * if needed. 770 */ 771 static int 772 swap_on(struct lwp *l, struct swapdev *sdp) 773 { 774 struct vnode *vp; 775 int error, npages, nblocks, size; 776 long addr; 777 u_long result; 778 struct vattr va; 779 #ifdef NFS 780 extern int (**nfsv2_vnodeop_p)(void *); 781 #endif /* NFS */ 782 const struct bdevsw *bdev; 783 dev_t dev; 784 UVMHIST_FUNC("swap_on"); UVMHIST_CALLED(pdhist); 785 786 /* 787 * we want to enable swapping on sdp. the swd_vp contains 788 * the vnode we want (locked and ref'd), and the swd_dev 789 * contains the dev_t of the file, if it a block device. 790 */ 791 792 vp = sdp->swd_vp; 793 dev = sdp->swd_dev; 794 795 /* 796 * open the swap file (mostly useful for block device files to 797 * let device driver know what is up). 798 * 799 * we skip the open/close for root on swap because the root 800 * has already been opened when root was mounted (mountroot). 801 */ 802 if (vp != rootvp) { 803 if ((error = VOP_OPEN(vp, FREAD|FWRITE, l->l_cred))) 804 return (error); 805 } 806 807 /* XXX this only works for block devices */ 808 UVMHIST_LOG(pdhist, " dev=%d, major(dev)=%d", dev, major(dev), 0,0); 809 810 /* 811 * we now need to determine the size of the swap area. for 812 * block specials we can call the d_psize function. 813 * for normal files, we must stat [get attrs]. 814 * 815 * we put the result in nblks. 816 * for normal files, we also want the filesystem block size 817 * (which we get with statfs). 818 */ 819 switch (vp->v_type) { 820 case VBLK: 821 bdev = bdevsw_lookup(dev); 822 if (bdev == NULL || bdev->d_psize == NULL || 823 (nblocks = (*bdev->d_psize)(dev)) == -1) { 824 error = ENXIO; 825 goto bad; 826 } 827 break; 828 829 case VREG: 830 if ((error = VOP_GETATTR(vp, &va, l->l_cred))) 831 goto bad; 832 nblocks = (int)btodb(va.va_size); 833 if ((error = 834 VFS_STATVFS(vp->v_mount, &vp->v_mount->mnt_stat)) != 0) 835 goto bad; 836 837 sdp->swd_bsize = vp->v_mount->mnt_stat.f_iosize; 838 /* 839 * limit the max # of outstanding I/O requests we issue 840 * at any one time. take it easy on NFS servers. 841 */ 842 #ifdef NFS 843 if (vp->v_op == nfsv2_vnodeop_p) 844 sdp->swd_maxactive = 2; /* XXX */ 845 else 846 #endif /* NFS */ 847 sdp->swd_maxactive = 8; /* XXX */ 848 break; 849 850 default: 851 error = ENXIO; 852 goto bad; 853 } 854 855 /* 856 * save nblocks in a safe place and convert to pages. 857 */ 858 859 sdp->swd_ose.ose_nblks = nblocks; 860 npages = dbtob((uint64_t)nblocks) >> PAGE_SHIFT; 861 862 /* 863 * for block special files, we want to make sure that leave 864 * the disklabel and bootblocks alone, so we arrange to skip 865 * over them (arbitrarily choosing to skip PAGE_SIZE bytes). 866 * note that because of this the "size" can be less than the 867 * actual number of blocks on the device. 868 */ 869 if (vp->v_type == VBLK) { 870 /* we use pages 1 to (size - 1) [inclusive] */ 871 size = npages - 1; 872 addr = 1; 873 } else { 874 /* we use pages 0 to (size - 1) [inclusive] */ 875 size = npages; 876 addr = 0; 877 } 878 879 /* 880 * make sure we have enough blocks for a reasonable sized swap 881 * area. we want at least one page. 882 */ 883 884 if (size < 1) { 885 UVMHIST_LOG(pdhist, " size <= 1!!", 0, 0, 0, 0); 886 error = EINVAL; 887 goto bad; 888 } 889 890 UVMHIST_LOG(pdhist, " dev=%x: size=%d addr=%ld\n", dev, size, addr, 0); 891 892 /* 893 * now we need to allocate an extent to manage this swap device 894 */ 895 896 sdp->swd_blist = blist_create(npages); 897 /* mark all expect the `saved' region free. */ 898 blist_free(sdp->swd_blist, addr, size); 899 900 /* 901 * if the vnode we are swapping to is the root vnode 902 * (i.e. we are swapping to the miniroot) then we want 903 * to make sure we don't overwrite it. do a statfs to 904 * find its size and skip over it. 905 */ 906 if (vp == rootvp) { 907 struct mount *mp; 908 struct statvfs *sp; 909 int rootblocks, rootpages; 910 911 mp = rootvnode->v_mount; 912 sp = &mp->mnt_stat; 913 rootblocks = sp->f_blocks * btodb(sp->f_frsize); 914 /* 915 * XXX: sp->f_blocks isn't the total number of 916 * blocks in the filesystem, it's the number of 917 * data blocks. so, our rootblocks almost 918 * definitely underestimates the total size 919 * of the filesystem - how badly depends on the 920 * details of the filesystem type. there isn't 921 * an obvious way to deal with this cleanly 922 * and perfectly, so for now we just pad our 923 * rootblocks estimate with an extra 5 percent. 924 */ 925 rootblocks += (rootblocks >> 5) + 926 (rootblocks >> 6) + 927 (rootblocks >> 7); 928 rootpages = round_page(dbtob(rootblocks)) >> PAGE_SHIFT; 929 if (rootpages > size) 930 panic("swap_on: miniroot larger than swap?"); 931 932 if (rootpages != blist_fill(sdp->swd_blist, addr, rootpages)) { 933 panic("swap_on: unable to preserve miniroot"); 934 } 935 936 size -= rootpages; 937 printf("Preserved %d pages of miniroot ", rootpages); 938 printf("leaving %d pages of swap\n", size); 939 } 940 941 /* 942 * add a ref to vp to reflect usage as a swap device. 943 */ 944 vref(vp); 945 946 /* 947 * now add the new swapdev to the drum and enable. 948 */ 949 result = vmem_alloc(swapmap, npages, VM_BESTFIT | VM_SLEEP); 950 if (result == 0) 951 panic("swapdrum_add"); 952 /* 953 * If this is the first regular swap create the workqueue. 954 * => Protected by swap_syscall_lock. 955 */ 956 if (vp->v_type != VBLK) { 957 if (sw_reg_count++ == 0) { 958 KASSERT(sw_reg_workqueue == NULL); 959 if (workqueue_create(&sw_reg_workqueue, "swapiod", 960 sw_reg_iodone, NULL, PRIBIO, IPL_BIO, 0) != 0) 961 panic("swap_add: workqueue_create failed"); 962 } 963 } 964 965 sdp->swd_drumoffset = (int)result; 966 sdp->swd_drumsize = npages; 967 sdp->swd_npages = size; 968 mutex_enter(&uvm_swap_data_lock); 969 sdp->swd_flags &= ~SWF_FAKE; /* going live */ 970 sdp->swd_flags |= (SWF_INUSE|SWF_ENABLE); 971 uvmexp.swpages += size; 972 uvmexp.swpgavail += size; 973 mutex_exit(&uvm_swap_data_lock); 974 return (0); 975 976 /* 977 * failure: clean up and return error. 978 */ 979 980 bad: 981 if (sdp->swd_blist) { 982 blist_destroy(sdp->swd_blist); 983 } 984 if (vp != rootvp) { 985 (void)VOP_CLOSE(vp, FREAD|FWRITE, l->l_cred); 986 } 987 return (error); 988 } 989 990 /* 991 * swap_off: stop swapping on swapdev 992 * 993 * => swap data should be locked, we will unlock. 994 */ 995 static int 996 swap_off(struct lwp *l, struct swapdev *sdp) 997 { 998 int npages = sdp->swd_npages; 999 int error = 0; 1000 1001 UVMHIST_FUNC("swap_off"); UVMHIST_CALLED(pdhist); 1002 UVMHIST_LOG(pdhist, " dev=%x, npages=%d", sdp->swd_dev,npages,0,0); 1003 1004 /* disable the swap area being removed */ 1005 sdp->swd_flags &= ~SWF_ENABLE; 1006 uvmexp.swpgavail -= npages; 1007 mutex_exit(&uvm_swap_data_lock); 1008 1009 /* 1010 * the idea is to find all the pages that are paged out to this 1011 * device, and page them all in. in uvm, swap-backed pageable 1012 * memory can take two forms: aobjs and anons. call the 1013 * swapoff hook for each subsystem to bring in pages. 1014 */ 1015 1016 if (uao_swap_off(sdp->swd_drumoffset, 1017 sdp->swd_drumoffset + sdp->swd_drumsize) || 1018 amap_swap_off(sdp->swd_drumoffset, 1019 sdp->swd_drumoffset + sdp->swd_drumsize)) { 1020 error = ENOMEM; 1021 } else if (sdp->swd_npginuse > sdp->swd_npgbad) { 1022 error = EBUSY; 1023 } 1024 1025 if (error) { 1026 mutex_enter(&uvm_swap_data_lock); 1027 sdp->swd_flags |= SWF_ENABLE; 1028 uvmexp.swpgavail += npages; 1029 mutex_exit(&uvm_swap_data_lock); 1030 1031 return error; 1032 } 1033 1034 /* 1035 * If this is the last regular swap destroy the workqueue. 1036 * => Protected by swap_syscall_lock. 1037 */ 1038 if (sdp->swd_vp->v_type != VBLK) { 1039 KASSERT(sw_reg_count > 0); 1040 KASSERT(sw_reg_workqueue != NULL); 1041 if (--sw_reg_count == 0) { 1042 workqueue_destroy(sw_reg_workqueue); 1043 sw_reg_workqueue = NULL; 1044 } 1045 } 1046 1047 /* 1048 * done with the vnode. 1049 * drop our ref on the vnode before calling VOP_CLOSE() 1050 * so that spec_close() can tell if this is the last close. 1051 */ 1052 vrele(sdp->swd_vp); 1053 if (sdp->swd_vp != rootvp) { 1054 (void) VOP_CLOSE(sdp->swd_vp, FREAD|FWRITE, l->l_cred); 1055 } 1056 1057 mutex_enter(&uvm_swap_data_lock); 1058 uvmexp.swpages -= npages; 1059 uvmexp.swpginuse -= sdp->swd_npgbad; 1060 1061 if (swaplist_find(sdp->swd_vp, true) == NULL) 1062 panic("swap_off: swapdev not in list"); 1063 swaplist_trim(); 1064 mutex_exit(&uvm_swap_data_lock); 1065 1066 /* 1067 * free all resources! 1068 */ 1069 vmem_free(swapmap, sdp->swd_drumoffset, sdp->swd_drumsize); 1070 blist_destroy(sdp->swd_blist); 1071 bufq_free(sdp->swd_tab); 1072 free(sdp, M_VMSWAP); 1073 return (0); 1074 } 1075 1076 /* 1077 * /dev/drum interface and i/o functions 1078 */ 1079 1080 /* 1081 * swstrategy: perform I/O on the drum 1082 * 1083 * => we must map the i/o request from the drum to the correct swapdev. 1084 */ 1085 static void 1086 swstrategy(struct buf *bp) 1087 { 1088 struct swapdev *sdp; 1089 struct vnode *vp; 1090 int pageno, bn; 1091 UVMHIST_FUNC("swstrategy"); UVMHIST_CALLED(pdhist); 1092 1093 /* 1094 * convert block number to swapdev. note that swapdev can't 1095 * be yanked out from under us because we are holding resources 1096 * in it (i.e. the blocks we are doing I/O on). 1097 */ 1098 pageno = dbtob((int64_t)bp->b_blkno) >> PAGE_SHIFT; 1099 mutex_enter(&uvm_swap_data_lock); 1100 sdp = swapdrum_getsdp(pageno); 1101 mutex_exit(&uvm_swap_data_lock); 1102 if (sdp == NULL) { 1103 bp->b_error = EINVAL; 1104 biodone(bp); 1105 UVMHIST_LOG(pdhist, " failed to get swap device", 0, 0, 0, 0); 1106 return; 1107 } 1108 1109 /* 1110 * convert drum page number to block number on this swapdev. 1111 */ 1112 1113 pageno -= sdp->swd_drumoffset; /* page # on swapdev */ 1114 bn = btodb((uint64_t)pageno << PAGE_SHIFT); /* convert to diskblock */ 1115 1116 UVMHIST_LOG(pdhist, " %s: mapoff=%x bn=%x bcount=%ld", 1117 ((bp->b_flags & B_READ) == 0) ? "write" : "read", 1118 sdp->swd_drumoffset, bn, bp->b_bcount); 1119 1120 /* 1121 * for block devices we finish up here. 1122 * for regular files we have to do more work which we delegate 1123 * to sw_reg_strategy(). 1124 */ 1125 1126 vp = sdp->swd_vp; /* swapdev vnode pointer */ 1127 switch (vp->v_type) { 1128 default: 1129 panic("swstrategy: vnode type 0x%x", vp->v_type); 1130 1131 case VBLK: 1132 1133 /* 1134 * must convert "bp" from an I/O on /dev/drum to an I/O 1135 * on the swapdev (sdp). 1136 */ 1137 bp->b_blkno = bn; /* swapdev block number */ 1138 bp->b_dev = sdp->swd_dev; /* swapdev dev_t */ 1139 1140 /* 1141 * if we are doing a write, we have to redirect the i/o on 1142 * drum's v_numoutput counter to the swapdevs. 1143 */ 1144 if ((bp->b_flags & B_READ) == 0) { 1145 mutex_enter(bp->b_objlock); 1146 vwakeup(bp); /* kills one 'v_numoutput' on drum */ 1147 mutex_exit(bp->b_objlock); 1148 mutex_enter(&vp->v_interlock); 1149 vp->v_numoutput++; /* put it on swapdev */ 1150 mutex_exit(&vp->v_interlock); 1151 } 1152 1153 /* 1154 * finally plug in swapdev vnode and start I/O 1155 */ 1156 bp->b_vp = vp; 1157 bp->b_objlock = &vp->v_interlock; 1158 VOP_STRATEGY(vp, bp); 1159 return; 1160 1161 case VREG: 1162 /* 1163 * delegate to sw_reg_strategy function. 1164 */ 1165 sw_reg_strategy(sdp, bp, bn); 1166 return; 1167 } 1168 /* NOTREACHED */ 1169 } 1170 1171 /* 1172 * swread: the read function for the drum (just a call to physio) 1173 */ 1174 /*ARGSUSED*/ 1175 static int 1176 swread(dev_t dev, struct uio *uio, int ioflag) 1177 { 1178 UVMHIST_FUNC("swread"); UVMHIST_CALLED(pdhist); 1179 1180 UVMHIST_LOG(pdhist, " dev=%x offset=%qx", dev, uio->uio_offset, 0, 0); 1181 return (physio(swstrategy, NULL, dev, B_READ, minphys, uio)); 1182 } 1183 1184 /* 1185 * swwrite: the write function for the drum (just a call to physio) 1186 */ 1187 /*ARGSUSED*/ 1188 static int 1189 swwrite(dev_t dev, struct uio *uio, int ioflag) 1190 { 1191 UVMHIST_FUNC("swwrite"); UVMHIST_CALLED(pdhist); 1192 1193 UVMHIST_LOG(pdhist, " dev=%x offset=%qx", dev, uio->uio_offset, 0, 0); 1194 return (physio(swstrategy, NULL, dev, B_WRITE, minphys, uio)); 1195 } 1196 1197 const struct bdevsw swap_bdevsw = { 1198 nullopen, nullclose, swstrategy, noioctl, nodump, nosize, D_OTHER, 1199 }; 1200 1201 const struct cdevsw swap_cdevsw = { 1202 nullopen, nullclose, swread, swwrite, noioctl, 1203 nostop, notty, nopoll, nommap, nokqfilter, D_OTHER, 1204 }; 1205 1206 /* 1207 * sw_reg_strategy: handle swap i/o to regular files 1208 */ 1209 static void 1210 sw_reg_strategy(struct swapdev *sdp, struct buf *bp, int bn) 1211 { 1212 struct vnode *vp; 1213 struct vndxfer *vnx; 1214 daddr_t nbn; 1215 char *addr; 1216 off_t byteoff; 1217 int s, off, nra, error, sz, resid; 1218 UVMHIST_FUNC("sw_reg_strategy"); UVMHIST_CALLED(pdhist); 1219 1220 /* 1221 * allocate a vndxfer head for this transfer and point it to 1222 * our buffer. 1223 */ 1224 vnx = pool_get(&vndxfer_pool, PR_WAITOK); 1225 vnx->vx_flags = VX_BUSY; 1226 vnx->vx_error = 0; 1227 vnx->vx_pending = 0; 1228 vnx->vx_bp = bp; 1229 vnx->vx_sdp = sdp; 1230 1231 /* 1232 * setup for main loop where we read filesystem blocks into 1233 * our buffer. 1234 */ 1235 error = 0; 1236 bp->b_resid = bp->b_bcount; /* nothing transfered yet! */ 1237 addr = bp->b_data; /* current position in buffer */ 1238 byteoff = dbtob((uint64_t)bn); 1239 1240 for (resid = bp->b_resid; resid; resid -= sz) { 1241 struct vndbuf *nbp; 1242 1243 /* 1244 * translate byteoffset into block number. return values: 1245 * vp = vnode of underlying device 1246 * nbn = new block number (on underlying vnode dev) 1247 * nra = num blocks we can read-ahead (excludes requested 1248 * block) 1249 */ 1250 nra = 0; 1251 error = VOP_BMAP(sdp->swd_vp, byteoff / sdp->swd_bsize, 1252 &vp, &nbn, &nra); 1253 1254 if (error == 0 && nbn == (daddr_t)-1) { 1255 /* 1256 * this used to just set error, but that doesn't 1257 * do the right thing. Instead, it causes random 1258 * memory errors. The panic() should remain until 1259 * this condition doesn't destabilize the system. 1260 */ 1261 #if 1 1262 panic("sw_reg_strategy: swap to sparse file"); 1263 #else 1264 error = EIO; /* failure */ 1265 #endif 1266 } 1267 1268 /* 1269 * punt if there was an error or a hole in the file. 1270 * we must wait for any i/o ops we have already started 1271 * to finish before returning. 1272 * 1273 * XXX we could deal with holes here but it would be 1274 * a hassle (in the write case). 1275 */ 1276 if (error) { 1277 s = splbio(); 1278 vnx->vx_error = error; /* pass error up */ 1279 goto out; 1280 } 1281 1282 /* 1283 * compute the size ("sz") of this transfer (in bytes). 1284 */ 1285 off = byteoff % sdp->swd_bsize; 1286 sz = (1 + nra) * sdp->swd_bsize - off; 1287 if (sz > resid) 1288 sz = resid; 1289 1290 UVMHIST_LOG(pdhist, "sw_reg_strategy: " 1291 "vp %p/%p offset 0x%x/0x%x", 1292 sdp->swd_vp, vp, byteoff, nbn); 1293 1294 /* 1295 * now get a buf structure. note that the vb_buf is 1296 * at the front of the nbp structure so that you can 1297 * cast pointers between the two structure easily. 1298 */ 1299 nbp = pool_get(&vndbuf_pool, PR_WAITOK); 1300 buf_init(&nbp->vb_buf); 1301 nbp->vb_buf.b_flags = bp->b_flags; 1302 nbp->vb_buf.b_cflags = bp->b_cflags; 1303 nbp->vb_buf.b_oflags = bp->b_oflags; 1304 nbp->vb_buf.b_bcount = sz; 1305 nbp->vb_buf.b_bufsize = sz; 1306 nbp->vb_buf.b_error = 0; 1307 nbp->vb_buf.b_data = addr; 1308 nbp->vb_buf.b_lblkno = 0; 1309 nbp->vb_buf.b_blkno = nbn + btodb(off); 1310 nbp->vb_buf.b_rawblkno = nbp->vb_buf.b_blkno; 1311 nbp->vb_buf.b_iodone = sw_reg_biodone; 1312 nbp->vb_buf.b_vp = vp; 1313 nbp->vb_buf.b_objlock = &vp->v_interlock; 1314 if (vp->v_type == VBLK) { 1315 nbp->vb_buf.b_dev = vp->v_rdev; 1316 } 1317 1318 nbp->vb_xfer = vnx; /* patch it back in to vnx */ 1319 1320 /* 1321 * Just sort by block number 1322 */ 1323 s = splbio(); 1324 if (vnx->vx_error != 0) { 1325 buf_destroy(&nbp->vb_buf); 1326 pool_put(&vndbuf_pool, nbp); 1327 goto out; 1328 } 1329 vnx->vx_pending++; 1330 1331 /* sort it in and start I/O if we are not over our limit */ 1332 /* XXXAD locking */ 1333 BUFQ_PUT(sdp->swd_tab, &nbp->vb_buf); 1334 sw_reg_start(sdp); 1335 splx(s); 1336 1337 /* 1338 * advance to the next I/O 1339 */ 1340 byteoff += sz; 1341 addr += sz; 1342 } 1343 1344 s = splbio(); 1345 1346 out: /* Arrive here at splbio */ 1347 vnx->vx_flags &= ~VX_BUSY; 1348 if (vnx->vx_pending == 0) { 1349 error = vnx->vx_error; 1350 pool_put(&vndxfer_pool, vnx); 1351 bp->b_error = error; 1352 biodone(bp); 1353 } 1354 splx(s); 1355 } 1356 1357 /* 1358 * sw_reg_start: start an I/O request on the requested swapdev 1359 * 1360 * => reqs are sorted by b_rawblkno (above) 1361 */ 1362 static void 1363 sw_reg_start(struct swapdev *sdp) 1364 { 1365 struct buf *bp; 1366 struct vnode *vp; 1367 UVMHIST_FUNC("sw_reg_start"); UVMHIST_CALLED(pdhist); 1368 1369 /* recursion control */ 1370 if ((sdp->swd_flags & SWF_BUSY) != 0) 1371 return; 1372 1373 sdp->swd_flags |= SWF_BUSY; 1374 1375 while (sdp->swd_active < sdp->swd_maxactive) { 1376 bp = BUFQ_GET(sdp->swd_tab); 1377 if (bp == NULL) 1378 break; 1379 sdp->swd_active++; 1380 1381 UVMHIST_LOG(pdhist, 1382 "sw_reg_start: bp %p vp %p blkno %p cnt %lx", 1383 bp, bp->b_vp, bp->b_blkno, bp->b_bcount); 1384 vp = bp->b_vp; 1385 KASSERT(bp->b_objlock == &vp->v_interlock); 1386 if ((bp->b_flags & B_READ) == 0) { 1387 mutex_enter(&vp->v_interlock); 1388 vp->v_numoutput++; 1389 mutex_exit(&vp->v_interlock); 1390 } 1391 VOP_STRATEGY(vp, bp); 1392 } 1393 sdp->swd_flags &= ~SWF_BUSY; 1394 } 1395 1396 /* 1397 * sw_reg_biodone: one of our i/o's has completed 1398 */ 1399 static void 1400 sw_reg_biodone(struct buf *bp) 1401 { 1402 workqueue_enqueue(sw_reg_workqueue, &bp->b_work, NULL); 1403 } 1404 1405 /* 1406 * sw_reg_iodone: one of our i/o's has completed and needs post-i/o cleanup 1407 * 1408 * => note that we can recover the vndbuf struct by casting the buf ptr 1409 */ 1410 static void 1411 sw_reg_iodone(struct work *wk, void *dummy) 1412 { 1413 struct vndbuf *vbp = (void *)wk; 1414 struct vndxfer *vnx = vbp->vb_xfer; 1415 struct buf *pbp = vnx->vx_bp; /* parent buffer */ 1416 struct swapdev *sdp = vnx->vx_sdp; 1417 int s, resid, error; 1418 KASSERT(&vbp->vb_buf.b_work == wk); 1419 UVMHIST_FUNC("sw_reg_iodone"); UVMHIST_CALLED(pdhist); 1420 1421 UVMHIST_LOG(pdhist, " vbp=%p vp=%p blkno=%x addr=%p", 1422 vbp, vbp->vb_buf.b_vp, vbp->vb_buf.b_blkno, vbp->vb_buf.b_data); 1423 UVMHIST_LOG(pdhist, " cnt=%lx resid=%lx", 1424 vbp->vb_buf.b_bcount, vbp->vb_buf.b_resid, 0, 0); 1425 1426 /* 1427 * protect vbp at splbio and update. 1428 */ 1429 1430 s = splbio(); 1431 resid = vbp->vb_buf.b_bcount - vbp->vb_buf.b_resid; 1432 pbp->b_resid -= resid; 1433 vnx->vx_pending--; 1434 1435 if (vbp->vb_buf.b_error != 0) { 1436 /* pass error upward */ 1437 error = vbp->vb_buf.b_error ? vbp->vb_buf.b_error : EIO; 1438 UVMHIST_LOG(pdhist, " got error=%d !", error, 0, 0, 0); 1439 vnx->vx_error = error; 1440 } 1441 1442 /* 1443 * kill vbp structure 1444 */ 1445 buf_destroy(&vbp->vb_buf); 1446 pool_put(&vndbuf_pool, vbp); 1447 1448 /* 1449 * wrap up this transaction if it has run to completion or, in 1450 * case of an error, when all auxiliary buffers have returned. 1451 */ 1452 if (vnx->vx_error != 0) { 1453 /* pass error upward */ 1454 error = vnx->vx_error; 1455 if ((vnx->vx_flags & VX_BUSY) == 0 && vnx->vx_pending == 0) { 1456 pbp->b_error = error; 1457 biodone(pbp); 1458 pool_put(&vndxfer_pool, vnx); 1459 } 1460 } else if (pbp->b_resid == 0) { 1461 KASSERT(vnx->vx_pending == 0); 1462 if ((vnx->vx_flags & VX_BUSY) == 0) { 1463 UVMHIST_LOG(pdhist, " iodone error=%d !", 1464 pbp, vnx->vx_error, 0, 0); 1465 biodone(pbp); 1466 pool_put(&vndxfer_pool, vnx); 1467 } 1468 } 1469 1470 /* 1471 * done! start next swapdev I/O if one is pending 1472 */ 1473 sdp->swd_active--; 1474 sw_reg_start(sdp); 1475 splx(s); 1476 } 1477 1478 1479 /* 1480 * uvm_swap_alloc: allocate space on swap 1481 * 1482 * => allocation is done "round robin" down the priority list, as we 1483 * allocate in a priority we "rotate" the circle queue. 1484 * => space can be freed with uvm_swap_free 1485 * => we return the page slot number in /dev/drum (0 == invalid slot) 1486 * => we lock uvm_swap_data_lock 1487 * => XXXMRG: "LESSOK" INTERFACE NEEDED TO EXTENT SYSTEM 1488 */ 1489 int 1490 uvm_swap_alloc(int *nslots /* IN/OUT */, bool lessok) 1491 { 1492 struct swapdev *sdp; 1493 struct swappri *spp; 1494 UVMHIST_FUNC("uvm_swap_alloc"); UVMHIST_CALLED(pdhist); 1495 1496 /* 1497 * no swap devices configured yet? definite failure. 1498 */ 1499 if (uvmexp.nswapdev < 1) 1500 return 0; 1501 1502 /* 1503 * lock data lock, convert slots into blocks, and enter loop 1504 */ 1505 mutex_enter(&uvm_swap_data_lock); 1506 1507 ReTry: /* XXXMRG */ 1508 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 1509 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 1510 uint64_t result; 1511 1512 /* if it's not enabled, then we can't swap from it */ 1513 if ((sdp->swd_flags & SWF_ENABLE) == 0) 1514 continue; 1515 if (sdp->swd_npginuse + *nslots > sdp->swd_npages) 1516 continue; 1517 result = blist_alloc(sdp->swd_blist, *nslots); 1518 if (result == BLIST_NONE) { 1519 continue; 1520 } 1521 KASSERT(result < sdp->swd_drumsize); 1522 1523 /* 1524 * successful allocation! now rotate the circleq. 1525 */ 1526 CIRCLEQ_REMOVE(&spp->spi_swapdev, sdp, swd_next); 1527 CIRCLEQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next); 1528 sdp->swd_npginuse += *nslots; 1529 uvmexp.swpginuse += *nslots; 1530 mutex_exit(&uvm_swap_data_lock); 1531 /* done! return drum slot number */ 1532 UVMHIST_LOG(pdhist, 1533 "success! returning %d slots starting at %d", 1534 *nslots, result + sdp->swd_drumoffset, 0, 0); 1535 return (result + sdp->swd_drumoffset); 1536 } 1537 } 1538 1539 /* XXXMRG: BEGIN HACK */ 1540 if (*nslots > 1 && lessok) { 1541 *nslots = 1; 1542 /* XXXMRG: ugh! blist should support this for us */ 1543 goto ReTry; 1544 } 1545 /* XXXMRG: END HACK */ 1546 1547 mutex_exit(&uvm_swap_data_lock); 1548 return 0; 1549 } 1550 1551 bool 1552 uvm_swapisfull(void) 1553 { 1554 bool rv; 1555 1556 mutex_enter(&uvm_swap_data_lock); 1557 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1558 rv = (uvmexp.swpgonly >= uvmexp.swpgavail); 1559 mutex_exit(&uvm_swap_data_lock); 1560 1561 return (rv); 1562 } 1563 1564 /* 1565 * uvm_swap_markbad: keep track of swap ranges where we've had i/o errors 1566 * 1567 * => we lock uvm_swap_data_lock 1568 */ 1569 void 1570 uvm_swap_markbad(int startslot, int nslots) 1571 { 1572 struct swapdev *sdp; 1573 UVMHIST_FUNC("uvm_swap_markbad"); UVMHIST_CALLED(pdhist); 1574 1575 mutex_enter(&uvm_swap_data_lock); 1576 sdp = swapdrum_getsdp(startslot); 1577 KASSERT(sdp != NULL); 1578 1579 /* 1580 * we just keep track of how many pages have been marked bad 1581 * in this device, to make everything add up in swap_off(). 1582 * we assume here that the range of slots will all be within 1583 * one swap device. 1584 */ 1585 1586 KASSERT(uvmexp.swpgonly >= nslots); 1587 uvmexp.swpgonly -= nslots; 1588 sdp->swd_npgbad += nslots; 1589 UVMHIST_LOG(pdhist, "now %d bad", sdp->swd_npgbad, 0,0,0); 1590 mutex_exit(&uvm_swap_data_lock); 1591 } 1592 1593 /* 1594 * uvm_swap_free: free swap slots 1595 * 1596 * => this can be all or part of an allocation made by uvm_swap_alloc 1597 * => we lock uvm_swap_data_lock 1598 */ 1599 void 1600 uvm_swap_free(int startslot, int nslots) 1601 { 1602 struct swapdev *sdp; 1603 UVMHIST_FUNC("uvm_swap_free"); UVMHIST_CALLED(pdhist); 1604 1605 UVMHIST_LOG(pdhist, "freeing %d slots starting at %d", nslots, 1606 startslot, 0, 0); 1607 1608 /* 1609 * ignore attempts to free the "bad" slot. 1610 */ 1611 1612 if (startslot == SWSLOT_BAD) { 1613 return; 1614 } 1615 1616 /* 1617 * convert drum slot offset back to sdp, free the blocks 1618 * in the extent, and return. must hold pri lock to do 1619 * lookup and access the extent. 1620 */ 1621 1622 mutex_enter(&uvm_swap_data_lock); 1623 sdp = swapdrum_getsdp(startslot); 1624 KASSERT(uvmexp.nswapdev >= 1); 1625 KASSERT(sdp != NULL); 1626 KASSERT(sdp->swd_npginuse >= nslots); 1627 blist_free(sdp->swd_blist, startslot - sdp->swd_drumoffset, nslots); 1628 sdp->swd_npginuse -= nslots; 1629 uvmexp.swpginuse -= nslots; 1630 mutex_exit(&uvm_swap_data_lock); 1631 } 1632 1633 /* 1634 * uvm_swap_put: put any number of pages into a contig place on swap 1635 * 1636 * => can be sync or async 1637 */ 1638 1639 int 1640 uvm_swap_put(int swslot, struct vm_page **ppsp, int npages, int flags) 1641 { 1642 int error; 1643 1644 error = uvm_swap_io(ppsp, swslot, npages, B_WRITE | 1645 ((flags & PGO_SYNCIO) ? 0 : B_ASYNC)); 1646 return error; 1647 } 1648 1649 /* 1650 * uvm_swap_get: get a single page from swap 1651 * 1652 * => usually a sync op (from fault) 1653 */ 1654 1655 int 1656 uvm_swap_get(struct vm_page *page, int swslot, int flags) 1657 { 1658 int error; 1659 1660 uvmexp.nswget++; 1661 KASSERT(flags & PGO_SYNCIO); 1662 if (swslot == SWSLOT_BAD) { 1663 return EIO; 1664 } 1665 1666 error = uvm_swap_io(&page, swslot, 1, B_READ | 1667 ((flags & PGO_SYNCIO) ? 0 : B_ASYNC)); 1668 if (error == 0) { 1669 1670 /* 1671 * this page is no longer only in swap. 1672 */ 1673 1674 mutex_enter(&uvm_swap_data_lock); 1675 KASSERT(uvmexp.swpgonly > 0); 1676 uvmexp.swpgonly--; 1677 mutex_exit(&uvm_swap_data_lock); 1678 } 1679 return error; 1680 } 1681 1682 /* 1683 * uvm_swap_io: do an i/o operation to swap 1684 */ 1685 1686 static int 1687 uvm_swap_io(struct vm_page **pps, int startslot, int npages, int flags) 1688 { 1689 daddr_t startblk; 1690 struct buf *bp; 1691 vaddr_t kva; 1692 int error, mapinflags; 1693 bool write, async; 1694 UVMHIST_FUNC("uvm_swap_io"); UVMHIST_CALLED(pdhist); 1695 1696 UVMHIST_LOG(pdhist, "<- called, startslot=%d, npages=%d, flags=%d", 1697 startslot, npages, flags, 0); 1698 1699 write = (flags & B_READ) == 0; 1700 async = (flags & B_ASYNC) != 0; 1701 1702 /* 1703 * allocate a buf for the i/o. 1704 */ 1705 1706 KASSERT(curlwp != uvm.pagedaemon_lwp || (write && async)); 1707 bp = getiobuf(swapdev_vp, curlwp != uvm.pagedaemon_lwp); 1708 if (bp == NULL) { 1709 uvm_aio_aiodone_pages(pps, npages, true, ENOMEM); 1710 return ENOMEM; 1711 } 1712 1713 /* 1714 * convert starting drum slot to block number 1715 */ 1716 1717 startblk = btodb((uint64_t)startslot << PAGE_SHIFT); 1718 1719 /* 1720 * first, map the pages into the kernel. 1721 */ 1722 1723 mapinflags = !write ? 1724 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_READ : 1725 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_WRITE; 1726 kva = uvm_pagermapin(pps, npages, mapinflags); 1727 1728 /* 1729 * fill in the bp/sbp. we currently route our i/o through 1730 * /dev/drum's vnode [swapdev_vp]. 1731 */ 1732 1733 bp->b_cflags = BC_BUSY | BC_NOCACHE; 1734 bp->b_flags = (flags & (B_READ|B_ASYNC)); 1735 bp->b_proc = &proc0; /* XXX */ 1736 bp->b_vnbufs.le_next = NOLIST; 1737 bp->b_data = (void *)kva; 1738 bp->b_blkno = startblk; 1739 bp->b_bufsize = bp->b_bcount = npages << PAGE_SHIFT; 1740 1741 /* 1742 * bump v_numoutput (counter of number of active outputs). 1743 */ 1744 1745 if (write) { 1746 mutex_enter(&swapdev_vp->v_interlock); 1747 swapdev_vp->v_numoutput++; 1748 mutex_exit(&swapdev_vp->v_interlock); 1749 } 1750 1751 /* 1752 * for async ops we must set up the iodone handler. 1753 */ 1754 1755 if (async) { 1756 bp->b_iodone = uvm_aio_biodone; 1757 UVMHIST_LOG(pdhist, "doing async!", 0, 0, 0, 0); 1758 if (curlwp == uvm.pagedaemon_lwp) 1759 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); 1760 else 1761 BIO_SETPRIO(bp, BPRIO_TIMELIMITED); 1762 } else { 1763 bp->b_iodone = NULL; 1764 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); 1765 } 1766 UVMHIST_LOG(pdhist, 1767 "about to start io: data = %p blkno = 0x%x, bcount = %ld", 1768 bp->b_data, bp->b_blkno, bp->b_bcount, 0); 1769 1770 /* 1771 * now we start the I/O, and if async, return. 1772 */ 1773 1774 VOP_STRATEGY(swapdev_vp, bp); 1775 if (async) 1776 return 0; 1777 1778 /* 1779 * must be sync i/o. wait for it to finish 1780 */ 1781 1782 error = biowait(bp); 1783 1784 /* 1785 * kill the pager mapping 1786 */ 1787 1788 uvm_pagermapout(kva, npages); 1789 1790 /* 1791 * now dispose of the buf and we're done. 1792 */ 1793 1794 if (write) { 1795 mutex_enter(&swapdev_vp->v_interlock); 1796 vwakeup(bp); 1797 mutex_exit(&swapdev_vp->v_interlock); 1798 } 1799 putiobuf(bp); 1800 UVMHIST_LOG(pdhist, "<- done (sync) error=%d", error, 0, 0, 0); 1801 1802 return (error); 1803 } 1804