1 /* $NetBSD: uvm_swap.c,v 1.150 2010/03/02 21:32:29 pooka Exp $ */ 2 3 /* 4 * Copyright (c) 1995, 1996, 1997, 2009 Matthew R. Green 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * from: NetBSD: vm_swap.c,v 1.52 1997/12/02 13:47:37 pk Exp 29 * from: Id: uvm_swap.c,v 1.1.2.42 1998/02/02 20:38:06 chuck Exp 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.150 2010/03/02 21:32:29 pooka Exp $"); 34 35 #include "opt_uvmhist.h" 36 #include "opt_compat_netbsd.h" 37 #include "opt_ddb.h" 38 39 #include <sys/param.h> 40 #include <sys/systm.h> 41 #include <sys/buf.h> 42 #include <sys/bufq.h> 43 #include <sys/conf.h> 44 #include <sys/proc.h> 45 #include <sys/namei.h> 46 #include <sys/disklabel.h> 47 #include <sys/errno.h> 48 #include <sys/kernel.h> 49 #include <sys/malloc.h> 50 #include <sys/vnode.h> 51 #include <sys/file.h> 52 #include <sys/vmem.h> 53 #include <sys/blist.h> 54 #include <sys/mount.h> 55 #include <sys/pool.h> 56 #include <sys/syscallargs.h> 57 #include <sys/swap.h> 58 #include <sys/kauth.h> 59 #include <sys/sysctl.h> 60 #include <sys/workqueue.h> 61 62 #include <uvm/uvm.h> 63 64 #include <miscfs/specfs/specdev.h> 65 66 /* 67 * uvm_swap.c: manage configuration and i/o to swap space. 68 */ 69 70 /* 71 * swap space is managed in the following way: 72 * 73 * each swap partition or file is described by a "swapdev" structure. 74 * each "swapdev" structure contains a "swapent" structure which contains 75 * information that is passed up to the user (via system calls). 76 * 77 * each swap partition is assigned a "priority" (int) which controls 78 * swap parition usage. 79 * 80 * the system maintains a global data structure describing all swap 81 * partitions/files. there is a sorted LIST of "swappri" structures 82 * which describe "swapdev"'s at that priority. this LIST is headed 83 * by the "swap_priority" global var. each "swappri" contains a 84 * CIRCLEQ of "swapdev" structures at that priority. 85 * 86 * locking: 87 * - swap_syscall_lock (krwlock_t): this lock serializes the swapctl 88 * system call and prevents the swap priority list from changing 89 * while we are in the middle of a system call (e.g. SWAP_STATS). 90 * - uvm_swap_data_lock (kmutex_t): this lock protects all swap data 91 * structures including the priority list, the swapdev structures, 92 * and the swapmap arena. 93 * 94 * each swap device has the following info: 95 * - swap device in use (could be disabled, preventing future use) 96 * - swap enabled (allows new allocations on swap) 97 * - map info in /dev/drum 98 * - vnode pointer 99 * for swap files only: 100 * - block size 101 * - max byte count in buffer 102 * - buffer 103 * 104 * userland controls and configures swap with the swapctl(2) system call. 105 * the sys_swapctl performs the following operations: 106 * [1] SWAP_NSWAP: returns the number of swap devices currently configured 107 * [2] SWAP_STATS: given a pointer to an array of swapent structures 108 * (passed in via "arg") of a size passed in via "misc" ... we load 109 * the current swap config into the array. The actual work is done 110 * in the uvm_swap_stats(9) function. 111 * [3] SWAP_ON: given a pathname in arg (could be device or file) and a 112 * priority in "misc", start swapping on it. 113 * [4] SWAP_OFF: as SWAP_ON, but stops swapping to a device 114 * [5] SWAP_CTL: changes the priority of a swap device (new priority in 115 * "misc") 116 */ 117 118 /* 119 * swapdev: describes a single swap partition/file 120 * 121 * note the following should be true: 122 * swd_inuse <= swd_nblks [number of blocks in use is <= total blocks] 123 * swd_nblks <= swd_mapsize [because mapsize includes miniroot+disklabel] 124 */ 125 struct swapdev { 126 dev_t swd_dev; /* device id */ 127 int swd_flags; /* flags:inuse/enable/fake */ 128 int swd_priority; /* our priority */ 129 int swd_nblks; /* blocks in this device */ 130 char *swd_path; /* saved pathname of device */ 131 int swd_pathlen; /* length of pathname */ 132 int swd_npages; /* #pages we can use */ 133 int swd_npginuse; /* #pages in use */ 134 int swd_npgbad; /* #pages bad */ 135 int swd_drumoffset; /* page0 offset in drum */ 136 int swd_drumsize; /* #pages in drum */ 137 blist_t swd_blist; /* blist for this swapdev */ 138 struct vnode *swd_vp; /* backing vnode */ 139 CIRCLEQ_ENTRY(swapdev) swd_next; /* priority circleq */ 140 141 int swd_bsize; /* blocksize (bytes) */ 142 int swd_maxactive; /* max active i/o reqs */ 143 struct bufq_state *swd_tab; /* buffer list */ 144 int swd_active; /* number of active buffers */ 145 }; 146 147 /* 148 * swap device priority entry; the list is kept sorted on `spi_priority'. 149 */ 150 struct swappri { 151 int spi_priority; /* priority */ 152 CIRCLEQ_HEAD(spi_swapdev, swapdev) spi_swapdev; 153 /* circleq of swapdevs at this priority */ 154 LIST_ENTRY(swappri) spi_swappri; /* global list of pri's */ 155 }; 156 157 /* 158 * The following two structures are used to keep track of data transfers 159 * on swap devices associated with regular files. 160 * NOTE: this code is more or less a copy of vnd.c; we use the same 161 * structure names here to ease porting.. 162 */ 163 struct vndxfer { 164 struct buf *vx_bp; /* Pointer to parent buffer */ 165 struct swapdev *vx_sdp; 166 int vx_error; 167 int vx_pending; /* # of pending aux buffers */ 168 int vx_flags; 169 #define VX_BUSY 1 170 #define VX_DEAD 2 171 }; 172 173 struct vndbuf { 174 struct buf vb_buf; 175 struct vndxfer *vb_xfer; 176 }; 177 178 /* 179 * NetBSD 1.3 swapctl(SWAP_STATS, ...) swapent structure; uses 32 bit 180 * dev_t and has no se_path[] member. 181 */ 182 struct swapent13 { 183 int32_t se13_dev; /* device id */ 184 int se13_flags; /* flags */ 185 int se13_nblks; /* total blocks */ 186 int se13_inuse; /* blocks in use */ 187 int se13_priority; /* priority of this device */ 188 }; 189 190 /* 191 * NetBSD 5.0 swapctl(SWAP_STATS, ...) swapent structure; uses 32 bit 192 * dev_t. 193 */ 194 struct swapent50 { 195 int32_t se50_dev; /* device id */ 196 int se50_flags; /* flags */ 197 int se50_nblks; /* total blocks */ 198 int se50_inuse; /* blocks in use */ 199 int se50_priority; /* priority of this device */ 200 char se50_path[PATH_MAX+1]; /* path name */ 201 }; 202 203 /* 204 * We keep a of pool vndbuf's and vndxfer structures. 205 */ 206 static struct pool vndxfer_pool, vndbuf_pool; 207 208 /* 209 * local variables 210 */ 211 MALLOC_DEFINE(M_VMSWAP, "VM swap", "VM swap structures"); 212 static vmem_t *swapmap; /* controls the mapping of /dev/drum */ 213 214 /* list of all active swap devices [by priority] */ 215 LIST_HEAD(swap_priority, swappri); 216 static struct swap_priority swap_priority; 217 218 /* locks */ 219 static krwlock_t swap_syscall_lock; 220 221 /* workqueue and use counter for swap to regular files */ 222 static int sw_reg_count = 0; 223 static struct workqueue *sw_reg_workqueue; 224 225 /* tuneables */ 226 u_int uvm_swapisfull_factor = 99; 227 228 /* 229 * prototypes 230 */ 231 static struct swapdev *swapdrum_getsdp(int); 232 233 static struct swapdev *swaplist_find(struct vnode *, bool); 234 static void swaplist_insert(struct swapdev *, 235 struct swappri *, int); 236 static void swaplist_trim(void); 237 238 static int swap_on(struct lwp *, struct swapdev *); 239 static int swap_off(struct lwp *, struct swapdev *); 240 241 static void uvm_swap_stats_locked(int, struct swapent *, int, register_t *); 242 243 static void sw_reg_strategy(struct swapdev *, struct buf *, int); 244 static void sw_reg_biodone(struct buf *); 245 static void sw_reg_iodone(struct work *wk, void *dummy); 246 static void sw_reg_start(struct swapdev *); 247 248 static int uvm_swap_io(struct vm_page **, int, int, int); 249 250 /* 251 * uvm_swap_init: init the swap system data structures and locks 252 * 253 * => called at boot time from init_main.c after the filesystems 254 * are brought up (which happens after uvm_init()) 255 */ 256 void 257 uvm_swap_init(void) 258 { 259 UVMHIST_FUNC("uvm_swap_init"); 260 261 UVMHIST_CALLED(pdhist); 262 /* 263 * first, init the swap list, its counter, and its lock. 264 * then get a handle on the vnode for /dev/drum by using 265 * the its dev_t number ("swapdev", from MD conf.c). 266 */ 267 268 LIST_INIT(&swap_priority); 269 uvmexp.nswapdev = 0; 270 rw_init(&swap_syscall_lock); 271 mutex_init(&uvm_swap_data_lock, MUTEX_DEFAULT, IPL_NONE); 272 273 if (bdevvp(swapdev, &swapdev_vp)) 274 panic("%s: can't get vnode for swap device", __func__); 275 if (vn_lock(swapdev_vp, LK_EXCLUSIVE | LK_RETRY)) 276 panic("%s: can't lock swap device", __func__); 277 if (VOP_OPEN(swapdev_vp, FREAD | FWRITE, NOCRED)) 278 panic("%s: can't open swap device", __func__); 279 VOP_UNLOCK(swapdev_vp, 0); 280 281 /* 282 * create swap block resource map to map /dev/drum. the range 283 * from 1 to INT_MAX allows 2 gigablocks of swap space. note 284 * that block 0 is reserved (used to indicate an allocation 285 * failure, or no allocation). 286 */ 287 swapmap = vmem_create("swapmap", 1, INT_MAX - 1, 1, NULL, NULL, NULL, 0, 288 VM_NOSLEEP, IPL_NONE); 289 if (swapmap == 0) { 290 panic("%s: vmem_create failed", __func__); 291 } 292 293 pool_init(&vndxfer_pool, sizeof(struct vndxfer), 0, 0, 0, "swp vnx", 294 NULL, IPL_BIO); 295 pool_init(&vndbuf_pool, sizeof(struct vndbuf), 0, 0, 0, "swp vnd", 296 NULL, IPL_BIO); 297 298 UVMHIST_LOG(pdhist, "<- done", 0, 0, 0, 0); 299 } 300 301 /* 302 * swaplist functions: functions that operate on the list of swap 303 * devices on the system. 304 */ 305 306 /* 307 * swaplist_insert: insert swap device "sdp" into the global list 308 * 309 * => caller must hold both swap_syscall_lock and uvm_swap_data_lock 310 * => caller must provide a newly malloc'd swappri structure (we will 311 * FREE it if we don't need it... this it to prevent malloc blocking 312 * here while adding swap) 313 */ 314 static void 315 swaplist_insert(struct swapdev *sdp, struct swappri *newspp, int priority) 316 { 317 struct swappri *spp, *pspp; 318 UVMHIST_FUNC("swaplist_insert"); UVMHIST_CALLED(pdhist); 319 320 /* 321 * find entry at or after which to insert the new device. 322 */ 323 pspp = NULL; 324 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 325 if (priority <= spp->spi_priority) 326 break; 327 pspp = spp; 328 } 329 330 /* 331 * new priority? 332 */ 333 if (spp == NULL || spp->spi_priority != priority) { 334 spp = newspp; /* use newspp! */ 335 UVMHIST_LOG(pdhist, "created new swappri = %d", 336 priority, 0, 0, 0); 337 338 spp->spi_priority = priority; 339 CIRCLEQ_INIT(&spp->spi_swapdev); 340 341 if (pspp) 342 LIST_INSERT_AFTER(pspp, spp, spi_swappri); 343 else 344 LIST_INSERT_HEAD(&swap_priority, spp, spi_swappri); 345 } else { 346 /* we don't need a new priority structure, free it */ 347 free(newspp, M_VMSWAP); 348 } 349 350 /* 351 * priority found (or created). now insert on the priority's 352 * circleq list and bump the total number of swapdevs. 353 */ 354 sdp->swd_priority = priority; 355 CIRCLEQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next); 356 uvmexp.nswapdev++; 357 } 358 359 /* 360 * swaplist_find: find and optionally remove a swap device from the 361 * global list. 362 * 363 * => caller must hold both swap_syscall_lock and uvm_swap_data_lock 364 * => we return the swapdev we found (and removed) 365 */ 366 static struct swapdev * 367 swaplist_find(struct vnode *vp, bool remove) 368 { 369 struct swapdev *sdp; 370 struct swappri *spp; 371 372 /* 373 * search the lists for the requested vp 374 */ 375 376 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 377 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 378 if (sdp->swd_vp == vp) { 379 if (remove) { 380 CIRCLEQ_REMOVE(&spp->spi_swapdev, 381 sdp, swd_next); 382 uvmexp.nswapdev--; 383 } 384 return(sdp); 385 } 386 } 387 } 388 return (NULL); 389 } 390 391 /* 392 * swaplist_trim: scan priority list for empty priority entries and kill 393 * them. 394 * 395 * => caller must hold both swap_syscall_lock and uvm_swap_data_lock 396 */ 397 static void 398 swaplist_trim(void) 399 { 400 struct swappri *spp, *nextspp; 401 402 for (spp = LIST_FIRST(&swap_priority); spp != NULL; spp = nextspp) { 403 nextspp = LIST_NEXT(spp, spi_swappri); 404 if (CIRCLEQ_FIRST(&spp->spi_swapdev) != 405 (void *)&spp->spi_swapdev) 406 continue; 407 LIST_REMOVE(spp, spi_swappri); 408 free(spp, M_VMSWAP); 409 } 410 } 411 412 /* 413 * swapdrum_getsdp: given a page offset in /dev/drum, convert it back 414 * to the "swapdev" that maps that section of the drum. 415 * 416 * => each swapdev takes one big contig chunk of the drum 417 * => caller must hold uvm_swap_data_lock 418 */ 419 static struct swapdev * 420 swapdrum_getsdp(int pgno) 421 { 422 struct swapdev *sdp; 423 struct swappri *spp; 424 425 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 426 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 427 if (sdp->swd_flags & SWF_FAKE) 428 continue; 429 if (pgno >= sdp->swd_drumoffset && 430 pgno < (sdp->swd_drumoffset + sdp->swd_drumsize)) { 431 return sdp; 432 } 433 } 434 } 435 return NULL; 436 } 437 438 439 /* 440 * sys_swapctl: main entry point for swapctl(2) system call 441 * [with two helper functions: swap_on and swap_off] 442 */ 443 int 444 sys_swapctl(struct lwp *l, const struct sys_swapctl_args *uap, register_t *retval) 445 { 446 /* { 447 syscallarg(int) cmd; 448 syscallarg(void *) arg; 449 syscallarg(int) misc; 450 } */ 451 struct vnode *vp; 452 struct nameidata nd; 453 struct swappri *spp; 454 struct swapdev *sdp; 455 struct swapent *sep; 456 #define SWAP_PATH_MAX (PATH_MAX + 1) 457 char *userpath; 458 size_t len; 459 int error, misc; 460 int priority; 461 UVMHIST_FUNC("sys_swapctl"); UVMHIST_CALLED(pdhist); 462 463 misc = SCARG(uap, misc); 464 465 /* 466 * ensure serialized syscall access by grabbing the swap_syscall_lock 467 */ 468 rw_enter(&swap_syscall_lock, RW_WRITER); 469 470 userpath = malloc(SWAP_PATH_MAX, M_TEMP, M_WAITOK); 471 /* 472 * we handle the non-priv NSWAP and STATS request first. 473 * 474 * SWAP_NSWAP: return number of config'd swap devices 475 * [can also be obtained with uvmexp sysctl] 476 */ 477 if (SCARG(uap, cmd) == SWAP_NSWAP) { 478 UVMHIST_LOG(pdhist, "<- done SWAP_NSWAP=%d", uvmexp.nswapdev, 479 0, 0, 0); 480 *retval = uvmexp.nswapdev; 481 error = 0; 482 goto out; 483 } 484 485 /* 486 * SWAP_STATS: get stats on current # of configured swap devs 487 * 488 * note that the swap_priority list can't change as long 489 * as we are holding the swap_syscall_lock. we don't want 490 * to grab the uvm_swap_data_lock because we may fault&sleep during 491 * copyout() and we don't want to be holding that lock then! 492 */ 493 if (SCARG(uap, cmd) == SWAP_STATS 494 #if defined(COMPAT_50) 495 || SCARG(uap, cmd) == SWAP_STATS50 496 #endif 497 #if defined(COMPAT_13) 498 || SCARG(uap, cmd) == SWAP_STATS13 499 #endif 500 ) { 501 if ((size_t)misc > (size_t)uvmexp.nswapdev) 502 misc = uvmexp.nswapdev; 503 #if defined(COMPAT_13) 504 if (SCARG(uap, cmd) == SWAP_STATS13) 505 len = sizeof(struct swapent13) * misc; 506 else 507 #endif 508 #if defined(COMPAT_50) 509 if (SCARG(uap, cmd) == SWAP_STATS50) 510 len = sizeof(struct swapent50) * misc; 511 else 512 #endif 513 len = sizeof(struct swapent) * misc; 514 sep = (struct swapent *)malloc(len, M_TEMP, M_WAITOK); 515 516 uvm_swap_stats_locked(SCARG(uap, cmd), sep, misc, retval); 517 error = copyout(sep, SCARG(uap, arg), len); 518 519 free(sep, M_TEMP); 520 UVMHIST_LOG(pdhist, "<- done SWAP_STATS", 0, 0, 0, 0); 521 goto out; 522 } 523 if (SCARG(uap, cmd) == SWAP_GETDUMPDEV) { 524 dev_t *devp = (dev_t *)SCARG(uap, arg); 525 526 error = copyout(&dumpdev, devp, sizeof(dumpdev)); 527 goto out; 528 } 529 530 /* 531 * all other requests require superuser privs. verify. 532 */ 533 if ((error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_SWAPCTL, 534 0, NULL, NULL, NULL))) 535 goto out; 536 537 if (SCARG(uap, cmd) == SWAP_DUMPOFF) { 538 /* drop the current dump device */ 539 dumpdev = NODEV; 540 dumpcdev = NODEV; 541 cpu_dumpconf(); 542 goto out; 543 } 544 545 /* 546 * at this point we expect a path name in arg. we will 547 * use namei() to gain a vnode reference (vref), and lock 548 * the vnode (VOP_LOCK). 549 * 550 * XXX: a NULL arg means use the root vnode pointer (e.g. for 551 * miniroot) 552 */ 553 if (SCARG(uap, arg) == NULL) { 554 vp = rootvp; /* miniroot */ 555 if (vget(vp, LK_EXCLUSIVE)) { 556 error = EBUSY; 557 goto out; 558 } 559 if (SCARG(uap, cmd) == SWAP_ON && 560 copystr("miniroot", userpath, SWAP_PATH_MAX, &len)) 561 panic("swapctl: miniroot copy failed"); 562 } else { 563 int space; 564 char *where; 565 566 if (SCARG(uap, cmd) == SWAP_ON) { 567 if ((error = copyinstr(SCARG(uap, arg), userpath, 568 SWAP_PATH_MAX, &len))) 569 goto out; 570 space = UIO_SYSSPACE; 571 where = userpath; 572 } else { 573 space = UIO_USERSPACE; 574 where = (char *)SCARG(uap, arg); 575 } 576 NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | TRYEMULROOT, 577 space, where); 578 if ((error = namei(&nd))) 579 goto out; 580 vp = nd.ni_vp; 581 } 582 /* note: "vp" is referenced and locked */ 583 584 error = 0; /* assume no error */ 585 switch(SCARG(uap, cmd)) { 586 587 case SWAP_DUMPDEV: 588 if (vp->v_type != VBLK) { 589 error = ENOTBLK; 590 break; 591 } 592 if (bdevsw_lookup(vp->v_rdev)) { 593 dumpdev = vp->v_rdev; 594 dumpcdev = devsw_blk2chr(dumpdev); 595 } else 596 dumpdev = NODEV; 597 cpu_dumpconf(); 598 break; 599 600 case SWAP_CTL: 601 /* 602 * get new priority, remove old entry (if any) and then 603 * reinsert it in the correct place. finally, prune out 604 * any empty priority structures. 605 */ 606 priority = SCARG(uap, misc); 607 spp = malloc(sizeof *spp, M_VMSWAP, M_WAITOK); 608 mutex_enter(&uvm_swap_data_lock); 609 if ((sdp = swaplist_find(vp, true)) == NULL) { 610 error = ENOENT; 611 } else { 612 swaplist_insert(sdp, spp, priority); 613 swaplist_trim(); 614 } 615 mutex_exit(&uvm_swap_data_lock); 616 if (error) 617 free(spp, M_VMSWAP); 618 break; 619 620 case SWAP_ON: 621 622 /* 623 * check for duplicates. if none found, then insert a 624 * dummy entry on the list to prevent someone else from 625 * trying to enable this device while we are working on 626 * it. 627 */ 628 629 priority = SCARG(uap, misc); 630 sdp = malloc(sizeof *sdp, M_VMSWAP, M_WAITOK); 631 spp = malloc(sizeof *spp, M_VMSWAP, M_WAITOK); 632 memset(sdp, 0, sizeof(*sdp)); 633 sdp->swd_flags = SWF_FAKE; 634 sdp->swd_vp = vp; 635 sdp->swd_dev = (vp->v_type == VBLK) ? vp->v_rdev : NODEV; 636 bufq_alloc(&sdp->swd_tab, "disksort", BUFQ_SORT_RAWBLOCK); 637 mutex_enter(&uvm_swap_data_lock); 638 if (swaplist_find(vp, false) != NULL) { 639 error = EBUSY; 640 mutex_exit(&uvm_swap_data_lock); 641 bufq_free(sdp->swd_tab); 642 free(sdp, M_VMSWAP); 643 free(spp, M_VMSWAP); 644 break; 645 } 646 swaplist_insert(sdp, spp, priority); 647 mutex_exit(&uvm_swap_data_lock); 648 649 sdp->swd_pathlen = len; 650 sdp->swd_path = malloc(sdp->swd_pathlen, M_VMSWAP, M_WAITOK); 651 if (copystr(userpath, sdp->swd_path, sdp->swd_pathlen, 0) != 0) 652 panic("swapctl: copystr"); 653 654 /* 655 * we've now got a FAKE placeholder in the swap list. 656 * now attempt to enable swap on it. if we fail, undo 657 * what we've done and kill the fake entry we just inserted. 658 * if swap_on is a success, it will clear the SWF_FAKE flag 659 */ 660 661 if ((error = swap_on(l, sdp)) != 0) { 662 mutex_enter(&uvm_swap_data_lock); 663 (void) swaplist_find(vp, true); /* kill fake entry */ 664 swaplist_trim(); 665 mutex_exit(&uvm_swap_data_lock); 666 bufq_free(sdp->swd_tab); 667 free(sdp->swd_path, M_VMSWAP); 668 free(sdp, M_VMSWAP); 669 break; 670 } 671 break; 672 673 case SWAP_OFF: 674 mutex_enter(&uvm_swap_data_lock); 675 if ((sdp = swaplist_find(vp, false)) == NULL) { 676 mutex_exit(&uvm_swap_data_lock); 677 error = ENXIO; 678 break; 679 } 680 681 /* 682 * If a device isn't in use or enabled, we 683 * can't stop swapping from it (again). 684 */ 685 if ((sdp->swd_flags & (SWF_INUSE|SWF_ENABLE)) == 0) { 686 mutex_exit(&uvm_swap_data_lock); 687 error = EBUSY; 688 break; 689 } 690 691 /* 692 * do the real work. 693 */ 694 error = swap_off(l, sdp); 695 break; 696 697 default: 698 error = EINVAL; 699 } 700 701 /* 702 * done! release the ref gained by namei() and unlock. 703 */ 704 vput(vp); 705 706 out: 707 free(userpath, M_TEMP); 708 rw_exit(&swap_syscall_lock); 709 710 UVMHIST_LOG(pdhist, "<- done! error=%d", error, 0, 0, 0); 711 return (error); 712 } 713 714 /* 715 * swap_stats: implements swapctl(SWAP_STATS). The function is kept 716 * away from sys_swapctl() in order to allow COMPAT_* swapctl() 717 * emulation to use it directly without going through sys_swapctl(). 718 * The problem with using sys_swapctl() there is that it involves 719 * copying the swapent array to the stackgap, and this array's size 720 * is not known at build time. Hence it would not be possible to 721 * ensure it would fit in the stackgap in any case. 722 */ 723 void 724 uvm_swap_stats(int cmd, struct swapent *sep, int sec, register_t *retval) 725 { 726 727 rw_enter(&swap_syscall_lock, RW_READER); 728 uvm_swap_stats_locked(cmd, sep, sec, retval); 729 rw_exit(&swap_syscall_lock); 730 } 731 732 static void 733 uvm_swap_stats_locked(int cmd, struct swapent *sep, int sec, register_t *retval) 734 { 735 struct swappri *spp; 736 struct swapdev *sdp; 737 int count = 0; 738 739 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 740 for (sdp = CIRCLEQ_FIRST(&spp->spi_swapdev); 741 sdp != (void *)&spp->spi_swapdev && sec-- > 0; 742 sdp = CIRCLEQ_NEXT(sdp, swd_next)) { 743 int inuse; 744 745 /* 746 * backwards compatibility for system call. 747 * For NetBSD 1.3 and 5.0, we have to use 748 * the 32 bit dev_t. For 5.0 and -current 749 * we have to add the path. 750 */ 751 inuse = btodb((uint64_t)sdp->swd_npginuse << 752 PAGE_SHIFT); 753 754 #if defined(COMPAT_13) || defined(COMPAT_50) 755 if (cmd == SWAP_STATS) { 756 #endif 757 sep->se_dev = sdp->swd_dev; 758 sep->se_flags = sdp->swd_flags; 759 sep->se_nblks = sdp->swd_nblks; 760 sep->se_inuse = inuse; 761 sep->se_priority = sdp->swd_priority; 762 memcpy(&sep->se_path, sdp->swd_path, 763 sizeof sep->se_path); 764 sep++; 765 #if defined(COMPAT_13) 766 } else if (cmd == SWAP_STATS13) { 767 struct swapent13 *sep13 = 768 (struct swapent13 *)sep; 769 770 sep13->se13_dev = sdp->swd_dev; 771 sep13->se13_flags = sdp->swd_flags; 772 sep13->se13_nblks = sdp->swd_nblks; 773 sep13->se13_inuse = inuse; 774 sep13->se13_priority = sdp->swd_priority; 775 sep = (struct swapent *)(sep13 + 1); 776 #endif 777 #if defined(COMPAT_50) 778 } else if (cmd == SWAP_STATS50) { 779 struct swapent50 *sep50 = 780 (struct swapent50 *)sep; 781 782 sep50->se50_dev = sdp->swd_dev; 783 sep50->se50_flags = sdp->swd_flags; 784 sep50->se50_nblks = sdp->swd_nblks; 785 sep50->se50_inuse = inuse; 786 sep50->se50_priority = sdp->swd_priority; 787 memcpy(&sep50->se50_path, sdp->swd_path, 788 sizeof sep50->se50_path); 789 sep = (struct swapent *)(sep50 + 1); 790 #endif 791 #if defined(COMPAT_13) || defined(COMPAT_50) 792 } 793 #endif 794 count++; 795 } 796 } 797 798 *retval = count; 799 return; 800 } 801 802 /* 803 * swap_on: attempt to enable a swapdev for swapping. note that the 804 * swapdev is already on the global list, but disabled (marked 805 * SWF_FAKE). 806 * 807 * => we avoid the start of the disk (to protect disk labels) 808 * => we also avoid the miniroot, if we are swapping to root. 809 * => caller should leave uvm_swap_data_lock unlocked, we may lock it 810 * if needed. 811 */ 812 static int 813 swap_on(struct lwp *l, struct swapdev *sdp) 814 { 815 struct vnode *vp; 816 int error, npages, nblocks, size; 817 long addr; 818 u_long result; 819 struct vattr va; 820 const struct bdevsw *bdev; 821 dev_t dev; 822 UVMHIST_FUNC("swap_on"); UVMHIST_CALLED(pdhist); 823 824 /* 825 * we want to enable swapping on sdp. the swd_vp contains 826 * the vnode we want (locked and ref'd), and the swd_dev 827 * contains the dev_t of the file, if it a block device. 828 */ 829 830 vp = sdp->swd_vp; 831 dev = sdp->swd_dev; 832 833 /* 834 * open the swap file (mostly useful for block device files to 835 * let device driver know what is up). 836 * 837 * we skip the open/close for root on swap because the root 838 * has already been opened when root was mounted (mountroot). 839 */ 840 if (vp != rootvp) { 841 if ((error = VOP_OPEN(vp, FREAD|FWRITE, l->l_cred))) 842 return (error); 843 } 844 845 /* XXX this only works for block devices */ 846 UVMHIST_LOG(pdhist, " dev=%d, major(dev)=%d", dev, major(dev), 0,0); 847 848 /* 849 * we now need to determine the size of the swap area. for 850 * block specials we can call the d_psize function. 851 * for normal files, we must stat [get attrs]. 852 * 853 * we put the result in nblks. 854 * for normal files, we also want the filesystem block size 855 * (which we get with statfs). 856 */ 857 switch (vp->v_type) { 858 case VBLK: 859 bdev = bdevsw_lookup(dev); 860 if (bdev == NULL || bdev->d_psize == NULL || 861 (nblocks = (*bdev->d_psize)(dev)) == -1) { 862 error = ENXIO; 863 goto bad; 864 } 865 break; 866 867 case VREG: 868 if ((error = VOP_GETATTR(vp, &va, l->l_cred))) 869 goto bad; 870 nblocks = (int)btodb(va.va_size); 871 sdp->swd_bsize = 1 << vp->v_mount->mnt_fs_bshift; 872 /* 873 * limit the max # of outstanding I/O requests we issue 874 * at any one time. take it easy on NFS servers. 875 */ 876 if (vp->v_tag == VT_NFS) 877 sdp->swd_maxactive = 2; /* XXX */ 878 else 879 sdp->swd_maxactive = 8; /* XXX */ 880 break; 881 882 default: 883 error = ENXIO; 884 goto bad; 885 } 886 887 /* 888 * save nblocks in a safe place and convert to pages. 889 */ 890 891 sdp->swd_nblks = nblocks; 892 npages = dbtob((uint64_t)nblocks) >> PAGE_SHIFT; 893 894 /* 895 * for block special files, we want to make sure that leave 896 * the disklabel and bootblocks alone, so we arrange to skip 897 * over them (arbitrarily choosing to skip PAGE_SIZE bytes). 898 * note that because of this the "size" can be less than the 899 * actual number of blocks on the device. 900 */ 901 if (vp->v_type == VBLK) { 902 /* we use pages 1 to (size - 1) [inclusive] */ 903 size = npages - 1; 904 addr = 1; 905 } else { 906 /* we use pages 0 to (size - 1) [inclusive] */ 907 size = npages; 908 addr = 0; 909 } 910 911 /* 912 * make sure we have enough blocks for a reasonable sized swap 913 * area. we want at least one page. 914 */ 915 916 if (size < 1) { 917 UVMHIST_LOG(pdhist, " size <= 1!!", 0, 0, 0, 0); 918 error = EINVAL; 919 goto bad; 920 } 921 922 UVMHIST_LOG(pdhist, " dev=%x: size=%d addr=%ld\n", dev, size, addr, 0); 923 924 /* 925 * now we need to allocate an extent to manage this swap device 926 */ 927 928 sdp->swd_blist = blist_create(npages); 929 /* mark all expect the `saved' region free. */ 930 blist_free(sdp->swd_blist, addr, size); 931 932 /* 933 * if the vnode we are swapping to is the root vnode 934 * (i.e. we are swapping to the miniroot) then we want 935 * to make sure we don't overwrite it. do a statfs to 936 * find its size and skip over it. 937 */ 938 if (vp == rootvp) { 939 struct mount *mp; 940 struct statvfs *sp; 941 int rootblocks, rootpages; 942 943 mp = rootvnode->v_mount; 944 sp = &mp->mnt_stat; 945 rootblocks = sp->f_blocks * btodb(sp->f_frsize); 946 /* 947 * XXX: sp->f_blocks isn't the total number of 948 * blocks in the filesystem, it's the number of 949 * data blocks. so, our rootblocks almost 950 * definitely underestimates the total size 951 * of the filesystem - how badly depends on the 952 * details of the filesystem type. there isn't 953 * an obvious way to deal with this cleanly 954 * and perfectly, so for now we just pad our 955 * rootblocks estimate with an extra 5 percent. 956 */ 957 rootblocks += (rootblocks >> 5) + 958 (rootblocks >> 6) + 959 (rootblocks >> 7); 960 rootpages = round_page(dbtob(rootblocks)) >> PAGE_SHIFT; 961 if (rootpages > size) 962 panic("swap_on: miniroot larger than swap?"); 963 964 if (rootpages != blist_fill(sdp->swd_blist, addr, rootpages)) { 965 panic("swap_on: unable to preserve miniroot"); 966 } 967 968 size -= rootpages; 969 printf("Preserved %d pages of miniroot ", rootpages); 970 printf("leaving %d pages of swap\n", size); 971 } 972 973 /* 974 * add a ref to vp to reflect usage as a swap device. 975 */ 976 vref(vp); 977 978 /* 979 * now add the new swapdev to the drum and enable. 980 */ 981 result = vmem_alloc(swapmap, npages, VM_BESTFIT | VM_SLEEP); 982 if (result == 0) 983 panic("swapdrum_add"); 984 /* 985 * If this is the first regular swap create the workqueue. 986 * => Protected by swap_syscall_lock. 987 */ 988 if (vp->v_type != VBLK) { 989 if (sw_reg_count++ == 0) { 990 KASSERT(sw_reg_workqueue == NULL); 991 if (workqueue_create(&sw_reg_workqueue, "swapiod", 992 sw_reg_iodone, NULL, PRIBIO, IPL_BIO, 0) != 0) 993 panic("%s: workqueue_create failed", __func__); 994 } 995 } 996 997 sdp->swd_drumoffset = (int)result; 998 sdp->swd_drumsize = npages; 999 sdp->swd_npages = size; 1000 mutex_enter(&uvm_swap_data_lock); 1001 sdp->swd_flags &= ~SWF_FAKE; /* going live */ 1002 sdp->swd_flags |= (SWF_INUSE|SWF_ENABLE); 1003 uvmexp.swpages += size; 1004 uvmexp.swpgavail += size; 1005 mutex_exit(&uvm_swap_data_lock); 1006 return (0); 1007 1008 /* 1009 * failure: clean up and return error. 1010 */ 1011 1012 bad: 1013 if (sdp->swd_blist) { 1014 blist_destroy(sdp->swd_blist); 1015 } 1016 if (vp != rootvp) { 1017 (void)VOP_CLOSE(vp, FREAD|FWRITE, l->l_cred); 1018 } 1019 return (error); 1020 } 1021 1022 /* 1023 * swap_off: stop swapping on swapdev 1024 * 1025 * => swap data should be locked, we will unlock. 1026 */ 1027 static int 1028 swap_off(struct lwp *l, struct swapdev *sdp) 1029 { 1030 int npages = sdp->swd_npages; 1031 int error = 0; 1032 1033 UVMHIST_FUNC("swap_off"); UVMHIST_CALLED(pdhist); 1034 UVMHIST_LOG(pdhist, " dev=%x, npages=%d", sdp->swd_dev,npages,0,0); 1035 1036 /* disable the swap area being removed */ 1037 sdp->swd_flags &= ~SWF_ENABLE; 1038 uvmexp.swpgavail -= npages; 1039 mutex_exit(&uvm_swap_data_lock); 1040 1041 /* 1042 * the idea is to find all the pages that are paged out to this 1043 * device, and page them all in. in uvm, swap-backed pageable 1044 * memory can take two forms: aobjs and anons. call the 1045 * swapoff hook for each subsystem to bring in pages. 1046 */ 1047 1048 if (uao_swap_off(sdp->swd_drumoffset, 1049 sdp->swd_drumoffset + sdp->swd_drumsize) || 1050 amap_swap_off(sdp->swd_drumoffset, 1051 sdp->swd_drumoffset + sdp->swd_drumsize)) { 1052 error = ENOMEM; 1053 } else if (sdp->swd_npginuse > sdp->swd_npgbad) { 1054 error = EBUSY; 1055 } 1056 1057 if (error) { 1058 mutex_enter(&uvm_swap_data_lock); 1059 sdp->swd_flags |= SWF_ENABLE; 1060 uvmexp.swpgavail += npages; 1061 mutex_exit(&uvm_swap_data_lock); 1062 1063 return error; 1064 } 1065 1066 /* 1067 * If this is the last regular swap destroy the workqueue. 1068 * => Protected by swap_syscall_lock. 1069 */ 1070 if (sdp->swd_vp->v_type != VBLK) { 1071 KASSERT(sw_reg_count > 0); 1072 KASSERT(sw_reg_workqueue != NULL); 1073 if (--sw_reg_count == 0) { 1074 workqueue_destroy(sw_reg_workqueue); 1075 sw_reg_workqueue = NULL; 1076 } 1077 } 1078 1079 /* 1080 * done with the vnode. 1081 * drop our ref on the vnode before calling VOP_CLOSE() 1082 * so that spec_close() can tell if this is the last close. 1083 */ 1084 vrele(sdp->swd_vp); 1085 if (sdp->swd_vp != rootvp) { 1086 (void) VOP_CLOSE(sdp->swd_vp, FREAD|FWRITE, l->l_cred); 1087 } 1088 1089 mutex_enter(&uvm_swap_data_lock); 1090 uvmexp.swpages -= npages; 1091 uvmexp.swpginuse -= sdp->swd_npgbad; 1092 1093 if (swaplist_find(sdp->swd_vp, true) == NULL) 1094 panic("%s: swapdev not in list", __func__); 1095 swaplist_trim(); 1096 mutex_exit(&uvm_swap_data_lock); 1097 1098 /* 1099 * free all resources! 1100 */ 1101 vmem_free(swapmap, sdp->swd_drumoffset, sdp->swd_drumsize); 1102 blist_destroy(sdp->swd_blist); 1103 bufq_free(sdp->swd_tab); 1104 free(sdp, M_VMSWAP); 1105 return (0); 1106 } 1107 1108 /* 1109 * /dev/drum interface and i/o functions 1110 */ 1111 1112 /* 1113 * swstrategy: perform I/O on the drum 1114 * 1115 * => we must map the i/o request from the drum to the correct swapdev. 1116 */ 1117 static void 1118 swstrategy(struct buf *bp) 1119 { 1120 struct swapdev *sdp; 1121 struct vnode *vp; 1122 int pageno, bn; 1123 UVMHIST_FUNC("swstrategy"); UVMHIST_CALLED(pdhist); 1124 1125 /* 1126 * convert block number to swapdev. note that swapdev can't 1127 * be yanked out from under us because we are holding resources 1128 * in it (i.e. the blocks we are doing I/O on). 1129 */ 1130 pageno = dbtob((int64_t)bp->b_blkno) >> PAGE_SHIFT; 1131 mutex_enter(&uvm_swap_data_lock); 1132 sdp = swapdrum_getsdp(pageno); 1133 mutex_exit(&uvm_swap_data_lock); 1134 if (sdp == NULL) { 1135 bp->b_error = EINVAL; 1136 biodone(bp); 1137 UVMHIST_LOG(pdhist, " failed to get swap device", 0, 0, 0, 0); 1138 return; 1139 } 1140 1141 /* 1142 * convert drum page number to block number on this swapdev. 1143 */ 1144 1145 pageno -= sdp->swd_drumoffset; /* page # on swapdev */ 1146 bn = btodb((uint64_t)pageno << PAGE_SHIFT); /* convert to diskblock */ 1147 1148 UVMHIST_LOG(pdhist, " %s: mapoff=%x bn=%x bcount=%ld", 1149 ((bp->b_flags & B_READ) == 0) ? "write" : "read", 1150 sdp->swd_drumoffset, bn, bp->b_bcount); 1151 1152 /* 1153 * for block devices we finish up here. 1154 * for regular files we have to do more work which we delegate 1155 * to sw_reg_strategy(). 1156 */ 1157 1158 vp = sdp->swd_vp; /* swapdev vnode pointer */ 1159 switch (vp->v_type) { 1160 default: 1161 panic("%s: vnode type 0x%x", __func__, vp->v_type); 1162 1163 case VBLK: 1164 1165 /* 1166 * must convert "bp" from an I/O on /dev/drum to an I/O 1167 * on the swapdev (sdp). 1168 */ 1169 bp->b_blkno = bn; /* swapdev block number */ 1170 bp->b_dev = sdp->swd_dev; /* swapdev dev_t */ 1171 1172 /* 1173 * if we are doing a write, we have to redirect the i/o on 1174 * drum's v_numoutput counter to the swapdevs. 1175 */ 1176 if ((bp->b_flags & B_READ) == 0) { 1177 mutex_enter(bp->b_objlock); 1178 vwakeup(bp); /* kills one 'v_numoutput' on drum */ 1179 mutex_exit(bp->b_objlock); 1180 mutex_enter(&vp->v_interlock); 1181 vp->v_numoutput++; /* put it on swapdev */ 1182 mutex_exit(&vp->v_interlock); 1183 } 1184 1185 /* 1186 * finally plug in swapdev vnode and start I/O 1187 */ 1188 bp->b_vp = vp; 1189 bp->b_objlock = &vp->v_interlock; 1190 VOP_STRATEGY(vp, bp); 1191 return; 1192 1193 case VREG: 1194 /* 1195 * delegate to sw_reg_strategy function. 1196 */ 1197 sw_reg_strategy(sdp, bp, bn); 1198 return; 1199 } 1200 /* NOTREACHED */ 1201 } 1202 1203 /* 1204 * swread: the read function for the drum (just a call to physio) 1205 */ 1206 /*ARGSUSED*/ 1207 static int 1208 swread(dev_t dev, struct uio *uio, int ioflag) 1209 { 1210 UVMHIST_FUNC("swread"); UVMHIST_CALLED(pdhist); 1211 1212 UVMHIST_LOG(pdhist, " dev=%x offset=%qx", dev, uio->uio_offset, 0, 0); 1213 return (physio(swstrategy, NULL, dev, B_READ, minphys, uio)); 1214 } 1215 1216 /* 1217 * swwrite: the write function for the drum (just a call to physio) 1218 */ 1219 /*ARGSUSED*/ 1220 static int 1221 swwrite(dev_t dev, struct uio *uio, int ioflag) 1222 { 1223 UVMHIST_FUNC("swwrite"); UVMHIST_CALLED(pdhist); 1224 1225 UVMHIST_LOG(pdhist, " dev=%x offset=%qx", dev, uio->uio_offset, 0, 0); 1226 return (physio(swstrategy, NULL, dev, B_WRITE, minphys, uio)); 1227 } 1228 1229 const struct bdevsw swap_bdevsw = { 1230 nullopen, nullclose, swstrategy, noioctl, nodump, nosize, D_OTHER, 1231 }; 1232 1233 const struct cdevsw swap_cdevsw = { 1234 nullopen, nullclose, swread, swwrite, noioctl, 1235 nostop, notty, nopoll, nommap, nokqfilter, D_OTHER, 1236 }; 1237 1238 /* 1239 * sw_reg_strategy: handle swap i/o to regular files 1240 */ 1241 static void 1242 sw_reg_strategy(struct swapdev *sdp, struct buf *bp, int bn) 1243 { 1244 struct vnode *vp; 1245 struct vndxfer *vnx; 1246 daddr_t nbn; 1247 char *addr; 1248 off_t byteoff; 1249 int s, off, nra, error, sz, resid; 1250 UVMHIST_FUNC("sw_reg_strategy"); UVMHIST_CALLED(pdhist); 1251 1252 /* 1253 * allocate a vndxfer head for this transfer and point it to 1254 * our buffer. 1255 */ 1256 vnx = pool_get(&vndxfer_pool, PR_WAITOK); 1257 vnx->vx_flags = VX_BUSY; 1258 vnx->vx_error = 0; 1259 vnx->vx_pending = 0; 1260 vnx->vx_bp = bp; 1261 vnx->vx_sdp = sdp; 1262 1263 /* 1264 * setup for main loop where we read filesystem blocks into 1265 * our buffer. 1266 */ 1267 error = 0; 1268 bp->b_resid = bp->b_bcount; /* nothing transfered yet! */ 1269 addr = bp->b_data; /* current position in buffer */ 1270 byteoff = dbtob((uint64_t)bn); 1271 1272 for (resid = bp->b_resid; resid; resid -= sz) { 1273 struct vndbuf *nbp; 1274 1275 /* 1276 * translate byteoffset into block number. return values: 1277 * vp = vnode of underlying device 1278 * nbn = new block number (on underlying vnode dev) 1279 * nra = num blocks we can read-ahead (excludes requested 1280 * block) 1281 */ 1282 nra = 0; 1283 error = VOP_BMAP(sdp->swd_vp, byteoff / sdp->swd_bsize, 1284 &vp, &nbn, &nra); 1285 1286 if (error == 0 && nbn == (daddr_t)-1) { 1287 /* 1288 * this used to just set error, but that doesn't 1289 * do the right thing. Instead, it causes random 1290 * memory errors. The panic() should remain until 1291 * this condition doesn't destabilize the system. 1292 */ 1293 #if 1 1294 panic("%s: swap to sparse file", __func__); 1295 #else 1296 error = EIO; /* failure */ 1297 #endif 1298 } 1299 1300 /* 1301 * punt if there was an error or a hole in the file. 1302 * we must wait for any i/o ops we have already started 1303 * to finish before returning. 1304 * 1305 * XXX we could deal with holes here but it would be 1306 * a hassle (in the write case). 1307 */ 1308 if (error) { 1309 s = splbio(); 1310 vnx->vx_error = error; /* pass error up */ 1311 goto out; 1312 } 1313 1314 /* 1315 * compute the size ("sz") of this transfer (in bytes). 1316 */ 1317 off = byteoff % sdp->swd_bsize; 1318 sz = (1 + nra) * sdp->swd_bsize - off; 1319 if (sz > resid) 1320 sz = resid; 1321 1322 UVMHIST_LOG(pdhist, "sw_reg_strategy: " 1323 "vp %p/%p offset 0x%x/0x%x", 1324 sdp->swd_vp, vp, byteoff, nbn); 1325 1326 /* 1327 * now get a buf structure. note that the vb_buf is 1328 * at the front of the nbp structure so that you can 1329 * cast pointers between the two structure easily. 1330 */ 1331 nbp = pool_get(&vndbuf_pool, PR_WAITOK); 1332 buf_init(&nbp->vb_buf); 1333 nbp->vb_buf.b_flags = bp->b_flags; 1334 nbp->vb_buf.b_cflags = bp->b_cflags; 1335 nbp->vb_buf.b_oflags = bp->b_oflags; 1336 nbp->vb_buf.b_bcount = sz; 1337 nbp->vb_buf.b_bufsize = sz; 1338 nbp->vb_buf.b_error = 0; 1339 nbp->vb_buf.b_data = addr; 1340 nbp->vb_buf.b_lblkno = 0; 1341 nbp->vb_buf.b_blkno = nbn + btodb(off); 1342 nbp->vb_buf.b_rawblkno = nbp->vb_buf.b_blkno; 1343 nbp->vb_buf.b_iodone = sw_reg_biodone; 1344 nbp->vb_buf.b_vp = vp; 1345 nbp->vb_buf.b_objlock = &vp->v_interlock; 1346 if (vp->v_type == VBLK) { 1347 nbp->vb_buf.b_dev = vp->v_rdev; 1348 } 1349 1350 nbp->vb_xfer = vnx; /* patch it back in to vnx */ 1351 1352 /* 1353 * Just sort by block number 1354 */ 1355 s = splbio(); 1356 if (vnx->vx_error != 0) { 1357 buf_destroy(&nbp->vb_buf); 1358 pool_put(&vndbuf_pool, nbp); 1359 goto out; 1360 } 1361 vnx->vx_pending++; 1362 1363 /* sort it in and start I/O if we are not over our limit */ 1364 /* XXXAD locking */ 1365 bufq_put(sdp->swd_tab, &nbp->vb_buf); 1366 sw_reg_start(sdp); 1367 splx(s); 1368 1369 /* 1370 * advance to the next I/O 1371 */ 1372 byteoff += sz; 1373 addr += sz; 1374 } 1375 1376 s = splbio(); 1377 1378 out: /* Arrive here at splbio */ 1379 vnx->vx_flags &= ~VX_BUSY; 1380 if (vnx->vx_pending == 0) { 1381 error = vnx->vx_error; 1382 pool_put(&vndxfer_pool, vnx); 1383 bp->b_error = error; 1384 biodone(bp); 1385 } 1386 splx(s); 1387 } 1388 1389 /* 1390 * sw_reg_start: start an I/O request on the requested swapdev 1391 * 1392 * => reqs are sorted by b_rawblkno (above) 1393 */ 1394 static void 1395 sw_reg_start(struct swapdev *sdp) 1396 { 1397 struct buf *bp; 1398 struct vnode *vp; 1399 UVMHIST_FUNC("sw_reg_start"); UVMHIST_CALLED(pdhist); 1400 1401 /* recursion control */ 1402 if ((sdp->swd_flags & SWF_BUSY) != 0) 1403 return; 1404 1405 sdp->swd_flags |= SWF_BUSY; 1406 1407 while (sdp->swd_active < sdp->swd_maxactive) { 1408 bp = bufq_get(sdp->swd_tab); 1409 if (bp == NULL) 1410 break; 1411 sdp->swd_active++; 1412 1413 UVMHIST_LOG(pdhist, 1414 "sw_reg_start: bp %p vp %p blkno %p cnt %lx", 1415 bp, bp->b_vp, bp->b_blkno, bp->b_bcount); 1416 vp = bp->b_vp; 1417 KASSERT(bp->b_objlock == &vp->v_interlock); 1418 if ((bp->b_flags & B_READ) == 0) { 1419 mutex_enter(&vp->v_interlock); 1420 vp->v_numoutput++; 1421 mutex_exit(&vp->v_interlock); 1422 } 1423 VOP_STRATEGY(vp, bp); 1424 } 1425 sdp->swd_flags &= ~SWF_BUSY; 1426 } 1427 1428 /* 1429 * sw_reg_biodone: one of our i/o's has completed 1430 */ 1431 static void 1432 sw_reg_biodone(struct buf *bp) 1433 { 1434 workqueue_enqueue(sw_reg_workqueue, &bp->b_work, NULL); 1435 } 1436 1437 /* 1438 * sw_reg_iodone: one of our i/o's has completed and needs post-i/o cleanup 1439 * 1440 * => note that we can recover the vndbuf struct by casting the buf ptr 1441 */ 1442 static void 1443 sw_reg_iodone(struct work *wk, void *dummy) 1444 { 1445 struct vndbuf *vbp = (void *)wk; 1446 struct vndxfer *vnx = vbp->vb_xfer; 1447 struct buf *pbp = vnx->vx_bp; /* parent buffer */ 1448 struct swapdev *sdp = vnx->vx_sdp; 1449 int s, resid, error; 1450 KASSERT(&vbp->vb_buf.b_work == wk); 1451 UVMHIST_FUNC("sw_reg_iodone"); UVMHIST_CALLED(pdhist); 1452 1453 UVMHIST_LOG(pdhist, " vbp=%p vp=%p blkno=%x addr=%p", 1454 vbp, vbp->vb_buf.b_vp, vbp->vb_buf.b_blkno, vbp->vb_buf.b_data); 1455 UVMHIST_LOG(pdhist, " cnt=%lx resid=%lx", 1456 vbp->vb_buf.b_bcount, vbp->vb_buf.b_resid, 0, 0); 1457 1458 /* 1459 * protect vbp at splbio and update. 1460 */ 1461 1462 s = splbio(); 1463 resid = vbp->vb_buf.b_bcount - vbp->vb_buf.b_resid; 1464 pbp->b_resid -= resid; 1465 vnx->vx_pending--; 1466 1467 if (vbp->vb_buf.b_error != 0) { 1468 /* pass error upward */ 1469 error = vbp->vb_buf.b_error ? vbp->vb_buf.b_error : EIO; 1470 UVMHIST_LOG(pdhist, " got error=%d !", error, 0, 0, 0); 1471 vnx->vx_error = error; 1472 } 1473 1474 /* 1475 * kill vbp structure 1476 */ 1477 buf_destroy(&vbp->vb_buf); 1478 pool_put(&vndbuf_pool, vbp); 1479 1480 /* 1481 * wrap up this transaction if it has run to completion or, in 1482 * case of an error, when all auxiliary buffers have returned. 1483 */ 1484 if (vnx->vx_error != 0) { 1485 /* pass error upward */ 1486 error = vnx->vx_error; 1487 if ((vnx->vx_flags & VX_BUSY) == 0 && vnx->vx_pending == 0) { 1488 pbp->b_error = error; 1489 biodone(pbp); 1490 pool_put(&vndxfer_pool, vnx); 1491 } 1492 } else if (pbp->b_resid == 0) { 1493 KASSERT(vnx->vx_pending == 0); 1494 if ((vnx->vx_flags & VX_BUSY) == 0) { 1495 UVMHIST_LOG(pdhist, " iodone error=%d !", 1496 pbp, vnx->vx_error, 0, 0); 1497 biodone(pbp); 1498 pool_put(&vndxfer_pool, vnx); 1499 } 1500 } 1501 1502 /* 1503 * done! start next swapdev I/O if one is pending 1504 */ 1505 sdp->swd_active--; 1506 sw_reg_start(sdp); 1507 splx(s); 1508 } 1509 1510 1511 /* 1512 * uvm_swap_alloc: allocate space on swap 1513 * 1514 * => allocation is done "round robin" down the priority list, as we 1515 * allocate in a priority we "rotate" the circle queue. 1516 * => space can be freed with uvm_swap_free 1517 * => we return the page slot number in /dev/drum (0 == invalid slot) 1518 * => we lock uvm_swap_data_lock 1519 * => XXXMRG: "LESSOK" INTERFACE NEEDED TO EXTENT SYSTEM 1520 */ 1521 int 1522 uvm_swap_alloc(int *nslots /* IN/OUT */, bool lessok) 1523 { 1524 struct swapdev *sdp; 1525 struct swappri *spp; 1526 UVMHIST_FUNC("uvm_swap_alloc"); UVMHIST_CALLED(pdhist); 1527 1528 /* 1529 * no swap devices configured yet? definite failure. 1530 */ 1531 if (uvmexp.nswapdev < 1) 1532 return 0; 1533 1534 /* 1535 * lock data lock, convert slots into blocks, and enter loop 1536 */ 1537 mutex_enter(&uvm_swap_data_lock); 1538 1539 ReTry: /* XXXMRG */ 1540 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 1541 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 1542 uint64_t result; 1543 1544 /* if it's not enabled, then we can't swap from it */ 1545 if ((sdp->swd_flags & SWF_ENABLE) == 0) 1546 continue; 1547 if (sdp->swd_npginuse + *nslots > sdp->swd_npages) 1548 continue; 1549 result = blist_alloc(sdp->swd_blist, *nslots); 1550 if (result == BLIST_NONE) { 1551 continue; 1552 } 1553 KASSERT(result < sdp->swd_drumsize); 1554 1555 /* 1556 * successful allocation! now rotate the circleq. 1557 */ 1558 CIRCLEQ_REMOVE(&spp->spi_swapdev, sdp, swd_next); 1559 CIRCLEQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next); 1560 sdp->swd_npginuse += *nslots; 1561 uvmexp.swpginuse += *nslots; 1562 mutex_exit(&uvm_swap_data_lock); 1563 /* done! return drum slot number */ 1564 UVMHIST_LOG(pdhist, 1565 "success! returning %d slots starting at %d", 1566 *nslots, result + sdp->swd_drumoffset, 0, 0); 1567 return (result + sdp->swd_drumoffset); 1568 } 1569 } 1570 1571 /* XXXMRG: BEGIN HACK */ 1572 if (*nslots > 1 && lessok) { 1573 *nslots = 1; 1574 /* XXXMRG: ugh! blist should support this for us */ 1575 goto ReTry; 1576 } 1577 /* XXXMRG: END HACK */ 1578 1579 mutex_exit(&uvm_swap_data_lock); 1580 return 0; 1581 } 1582 1583 /* 1584 * uvm_swapisfull: return true if most of available swap is allocated 1585 * and in use. we don't count some small portion as it may be inaccessible 1586 * to us at any given moment, for example if there is lock contention or if 1587 * pages are busy. 1588 */ 1589 bool 1590 uvm_swapisfull(void) 1591 { 1592 int swpgonly; 1593 bool rv; 1594 1595 mutex_enter(&uvm_swap_data_lock); 1596 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1597 swpgonly = (int)((uint64_t)uvmexp.swpgonly * 100 / 1598 uvm_swapisfull_factor); 1599 rv = (swpgonly >= uvmexp.swpgavail); 1600 mutex_exit(&uvm_swap_data_lock); 1601 1602 return (rv); 1603 } 1604 1605 /* 1606 * uvm_swap_markbad: keep track of swap ranges where we've had i/o errors 1607 * 1608 * => we lock uvm_swap_data_lock 1609 */ 1610 void 1611 uvm_swap_markbad(int startslot, int nslots) 1612 { 1613 struct swapdev *sdp; 1614 UVMHIST_FUNC("uvm_swap_markbad"); UVMHIST_CALLED(pdhist); 1615 1616 mutex_enter(&uvm_swap_data_lock); 1617 sdp = swapdrum_getsdp(startslot); 1618 KASSERT(sdp != NULL); 1619 1620 /* 1621 * we just keep track of how many pages have been marked bad 1622 * in this device, to make everything add up in swap_off(). 1623 * we assume here that the range of slots will all be within 1624 * one swap device. 1625 */ 1626 1627 KASSERT(uvmexp.swpgonly >= nslots); 1628 uvmexp.swpgonly -= nslots; 1629 sdp->swd_npgbad += nslots; 1630 UVMHIST_LOG(pdhist, "now %d bad", sdp->swd_npgbad, 0,0,0); 1631 mutex_exit(&uvm_swap_data_lock); 1632 } 1633 1634 /* 1635 * uvm_swap_free: free swap slots 1636 * 1637 * => this can be all or part of an allocation made by uvm_swap_alloc 1638 * => we lock uvm_swap_data_lock 1639 */ 1640 void 1641 uvm_swap_free(int startslot, int nslots) 1642 { 1643 struct swapdev *sdp; 1644 UVMHIST_FUNC("uvm_swap_free"); UVMHIST_CALLED(pdhist); 1645 1646 UVMHIST_LOG(pdhist, "freeing %d slots starting at %d", nslots, 1647 startslot, 0, 0); 1648 1649 /* 1650 * ignore attempts to free the "bad" slot. 1651 */ 1652 1653 if (startslot == SWSLOT_BAD) { 1654 return; 1655 } 1656 1657 /* 1658 * convert drum slot offset back to sdp, free the blocks 1659 * in the extent, and return. must hold pri lock to do 1660 * lookup and access the extent. 1661 */ 1662 1663 mutex_enter(&uvm_swap_data_lock); 1664 sdp = swapdrum_getsdp(startslot); 1665 KASSERT(uvmexp.nswapdev >= 1); 1666 KASSERT(sdp != NULL); 1667 KASSERT(sdp->swd_npginuse >= nslots); 1668 blist_free(sdp->swd_blist, startslot - sdp->swd_drumoffset, nslots); 1669 sdp->swd_npginuse -= nslots; 1670 uvmexp.swpginuse -= nslots; 1671 mutex_exit(&uvm_swap_data_lock); 1672 } 1673 1674 /* 1675 * uvm_swap_put: put any number of pages into a contig place on swap 1676 * 1677 * => can be sync or async 1678 */ 1679 1680 int 1681 uvm_swap_put(int swslot, struct vm_page **ppsp, int npages, int flags) 1682 { 1683 int error; 1684 1685 error = uvm_swap_io(ppsp, swslot, npages, B_WRITE | 1686 ((flags & PGO_SYNCIO) ? 0 : B_ASYNC)); 1687 return error; 1688 } 1689 1690 /* 1691 * uvm_swap_get: get a single page from swap 1692 * 1693 * => usually a sync op (from fault) 1694 */ 1695 1696 int 1697 uvm_swap_get(struct vm_page *page, int swslot, int flags) 1698 { 1699 int error; 1700 1701 uvmexp.nswget++; 1702 KASSERT(flags & PGO_SYNCIO); 1703 if (swslot == SWSLOT_BAD) { 1704 return EIO; 1705 } 1706 1707 error = uvm_swap_io(&page, swslot, 1, B_READ | 1708 ((flags & PGO_SYNCIO) ? 0 : B_ASYNC)); 1709 if (error == 0) { 1710 1711 /* 1712 * this page is no longer only in swap. 1713 */ 1714 1715 mutex_enter(&uvm_swap_data_lock); 1716 KASSERT(uvmexp.swpgonly > 0); 1717 uvmexp.swpgonly--; 1718 mutex_exit(&uvm_swap_data_lock); 1719 } 1720 return error; 1721 } 1722 1723 /* 1724 * uvm_swap_io: do an i/o operation to swap 1725 */ 1726 1727 static int 1728 uvm_swap_io(struct vm_page **pps, int startslot, int npages, int flags) 1729 { 1730 daddr_t startblk; 1731 struct buf *bp; 1732 vaddr_t kva; 1733 int error, mapinflags; 1734 bool write, async; 1735 UVMHIST_FUNC("uvm_swap_io"); UVMHIST_CALLED(pdhist); 1736 1737 UVMHIST_LOG(pdhist, "<- called, startslot=%d, npages=%d, flags=%d", 1738 startslot, npages, flags, 0); 1739 1740 write = (flags & B_READ) == 0; 1741 async = (flags & B_ASYNC) != 0; 1742 1743 /* 1744 * allocate a buf for the i/o. 1745 */ 1746 1747 KASSERT(curlwp != uvm.pagedaemon_lwp || (write && async)); 1748 bp = getiobuf(swapdev_vp, curlwp != uvm.pagedaemon_lwp); 1749 if (bp == NULL) { 1750 uvm_aio_aiodone_pages(pps, npages, true, ENOMEM); 1751 return ENOMEM; 1752 } 1753 1754 /* 1755 * convert starting drum slot to block number 1756 */ 1757 1758 startblk = btodb((uint64_t)startslot << PAGE_SHIFT); 1759 1760 /* 1761 * first, map the pages into the kernel. 1762 */ 1763 1764 mapinflags = !write ? 1765 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_READ : 1766 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_WRITE; 1767 kva = uvm_pagermapin(pps, npages, mapinflags); 1768 1769 /* 1770 * fill in the bp/sbp. we currently route our i/o through 1771 * /dev/drum's vnode [swapdev_vp]. 1772 */ 1773 1774 bp->b_cflags = BC_BUSY | BC_NOCACHE; 1775 bp->b_flags = (flags & (B_READ|B_ASYNC)); 1776 bp->b_proc = &proc0; /* XXX */ 1777 bp->b_vnbufs.le_next = NOLIST; 1778 bp->b_data = (void *)kva; 1779 bp->b_blkno = startblk; 1780 bp->b_bufsize = bp->b_bcount = npages << PAGE_SHIFT; 1781 1782 /* 1783 * bump v_numoutput (counter of number of active outputs). 1784 */ 1785 1786 if (write) { 1787 mutex_enter(&swapdev_vp->v_interlock); 1788 swapdev_vp->v_numoutput++; 1789 mutex_exit(&swapdev_vp->v_interlock); 1790 } 1791 1792 /* 1793 * for async ops we must set up the iodone handler. 1794 */ 1795 1796 if (async) { 1797 bp->b_iodone = uvm_aio_biodone; 1798 UVMHIST_LOG(pdhist, "doing async!", 0, 0, 0, 0); 1799 if (curlwp == uvm.pagedaemon_lwp) 1800 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); 1801 else 1802 BIO_SETPRIO(bp, BPRIO_TIMELIMITED); 1803 } else { 1804 bp->b_iodone = NULL; 1805 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); 1806 } 1807 UVMHIST_LOG(pdhist, 1808 "about to start io: data = %p blkno = 0x%x, bcount = %ld", 1809 bp->b_data, bp->b_blkno, bp->b_bcount, 0); 1810 1811 /* 1812 * now we start the I/O, and if async, return. 1813 */ 1814 1815 VOP_STRATEGY(swapdev_vp, bp); 1816 if (async) 1817 return 0; 1818 1819 /* 1820 * must be sync i/o. wait for it to finish 1821 */ 1822 1823 error = biowait(bp); 1824 1825 /* 1826 * kill the pager mapping 1827 */ 1828 1829 uvm_pagermapout(kva, npages); 1830 1831 /* 1832 * now dispose of the buf and we're done. 1833 */ 1834 1835 if (write) { 1836 mutex_enter(&swapdev_vp->v_interlock); 1837 vwakeup(bp); 1838 mutex_exit(&swapdev_vp->v_interlock); 1839 } 1840 putiobuf(bp); 1841 UVMHIST_LOG(pdhist, "<- done (sync) error=%d", error, 0, 0, 0); 1842 1843 return (error); 1844 } 1845