1 /* $NetBSD: uvm_swap.c,v 1.165 2013/11/23 14:50:40 christos Exp $ */ 2 3 /* 4 * Copyright (c) 1995, 1996, 1997, 2009 Matthew R. Green 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * from: NetBSD: vm_swap.c,v 1.52 1997/12/02 13:47:37 pk Exp 29 * from: Id: uvm_swap.c,v 1.1.2.42 1998/02/02 20:38:06 chuck Exp 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.165 2013/11/23 14:50:40 christos Exp $"); 34 35 #include "opt_uvmhist.h" 36 #include "opt_compat_netbsd.h" 37 #include "opt_ddb.h" 38 39 #include <sys/param.h> 40 #include <sys/systm.h> 41 #include <sys/buf.h> 42 #include <sys/bufq.h> 43 #include <sys/conf.h> 44 #include <sys/proc.h> 45 #include <sys/namei.h> 46 #include <sys/disklabel.h> 47 #include <sys/errno.h> 48 #include <sys/kernel.h> 49 #include <sys/vnode.h> 50 #include <sys/file.h> 51 #include <sys/vmem.h> 52 #include <sys/blist.h> 53 #include <sys/mount.h> 54 #include <sys/pool.h> 55 #include <sys/kmem.h> 56 #include <sys/syscallargs.h> 57 #include <sys/swap.h> 58 #include <sys/kauth.h> 59 #include <sys/sysctl.h> 60 #include <sys/workqueue.h> 61 62 #include <uvm/uvm.h> 63 64 #include <miscfs/specfs/specdev.h> 65 66 /* 67 * uvm_swap.c: manage configuration and i/o to swap space. 68 */ 69 70 /* 71 * swap space is managed in the following way: 72 * 73 * each swap partition or file is described by a "swapdev" structure. 74 * each "swapdev" structure contains a "swapent" structure which contains 75 * information that is passed up to the user (via system calls). 76 * 77 * each swap partition is assigned a "priority" (int) which controls 78 * swap parition usage. 79 * 80 * the system maintains a global data structure describing all swap 81 * partitions/files. there is a sorted LIST of "swappri" structures 82 * which describe "swapdev"'s at that priority. this LIST is headed 83 * by the "swap_priority" global var. each "swappri" contains a 84 * TAILQ of "swapdev" structures at that priority. 85 * 86 * locking: 87 * - swap_syscall_lock (krwlock_t): this lock serializes the swapctl 88 * system call and prevents the swap priority list from changing 89 * while we are in the middle of a system call (e.g. SWAP_STATS). 90 * - uvm_swap_data_lock (kmutex_t): this lock protects all swap data 91 * structures including the priority list, the swapdev structures, 92 * and the swapmap arena. 93 * 94 * each swap device has the following info: 95 * - swap device in use (could be disabled, preventing future use) 96 * - swap enabled (allows new allocations on swap) 97 * - map info in /dev/drum 98 * - vnode pointer 99 * for swap files only: 100 * - block size 101 * - max byte count in buffer 102 * - buffer 103 * 104 * userland controls and configures swap with the swapctl(2) system call. 105 * the sys_swapctl performs the following operations: 106 * [1] SWAP_NSWAP: returns the number of swap devices currently configured 107 * [2] SWAP_STATS: given a pointer to an array of swapent structures 108 * (passed in via "arg") of a size passed in via "misc" ... we load 109 * the current swap config into the array. The actual work is done 110 * in the uvm_swap_stats() function. 111 * [3] SWAP_ON: given a pathname in arg (could be device or file) and a 112 * priority in "misc", start swapping on it. 113 * [4] SWAP_OFF: as SWAP_ON, but stops swapping to a device 114 * [5] SWAP_CTL: changes the priority of a swap device (new priority in 115 * "misc") 116 */ 117 118 /* 119 * swapdev: describes a single swap partition/file 120 * 121 * note the following should be true: 122 * swd_inuse <= swd_nblks [number of blocks in use is <= total blocks] 123 * swd_nblks <= swd_mapsize [because mapsize includes miniroot+disklabel] 124 */ 125 struct swapdev { 126 dev_t swd_dev; /* device id */ 127 int swd_flags; /* flags:inuse/enable/fake */ 128 int swd_priority; /* our priority */ 129 int swd_nblks; /* blocks in this device */ 130 char *swd_path; /* saved pathname of device */ 131 int swd_pathlen; /* length of pathname */ 132 int swd_npages; /* #pages we can use */ 133 int swd_npginuse; /* #pages in use */ 134 int swd_npgbad; /* #pages bad */ 135 int swd_drumoffset; /* page0 offset in drum */ 136 int swd_drumsize; /* #pages in drum */ 137 blist_t swd_blist; /* blist for this swapdev */ 138 struct vnode *swd_vp; /* backing vnode */ 139 TAILQ_ENTRY(swapdev) swd_next; /* priority tailq */ 140 141 int swd_bsize; /* blocksize (bytes) */ 142 int swd_maxactive; /* max active i/o reqs */ 143 struct bufq_state *swd_tab; /* buffer list */ 144 int swd_active; /* number of active buffers */ 145 }; 146 147 /* 148 * swap device priority entry; the list is kept sorted on `spi_priority'. 149 */ 150 struct swappri { 151 int spi_priority; /* priority */ 152 TAILQ_HEAD(spi_swapdev, swapdev) spi_swapdev; 153 /* tailq of swapdevs at this priority */ 154 LIST_ENTRY(swappri) spi_swappri; /* global list of pri's */ 155 }; 156 157 /* 158 * The following two structures are used to keep track of data transfers 159 * on swap devices associated with regular files. 160 * NOTE: this code is more or less a copy of vnd.c; we use the same 161 * structure names here to ease porting.. 162 */ 163 struct vndxfer { 164 struct buf *vx_bp; /* Pointer to parent buffer */ 165 struct swapdev *vx_sdp; 166 int vx_error; 167 int vx_pending; /* # of pending aux buffers */ 168 int vx_flags; 169 #define VX_BUSY 1 170 #define VX_DEAD 2 171 }; 172 173 struct vndbuf { 174 struct buf vb_buf; 175 struct vndxfer *vb_xfer; 176 }; 177 178 /* 179 * NetBSD 1.3 swapctl(SWAP_STATS, ...) swapent structure; uses 32 bit 180 * dev_t and has no se_path[] member. 181 */ 182 struct swapent13 { 183 int32_t se13_dev; /* device id */ 184 int se13_flags; /* flags */ 185 int se13_nblks; /* total blocks */ 186 int se13_inuse; /* blocks in use */ 187 int se13_priority; /* priority of this device */ 188 }; 189 190 /* 191 * NetBSD 5.0 swapctl(SWAP_STATS, ...) swapent structure; uses 32 bit 192 * dev_t. 193 */ 194 struct swapent50 { 195 int32_t se50_dev; /* device id */ 196 int se50_flags; /* flags */ 197 int se50_nblks; /* total blocks */ 198 int se50_inuse; /* blocks in use */ 199 int se50_priority; /* priority of this device */ 200 char se50_path[PATH_MAX+1]; /* path name */ 201 }; 202 203 /* 204 * We keep a of pool vndbuf's and vndxfer structures. 205 */ 206 static struct pool vndxfer_pool, vndbuf_pool; 207 208 /* 209 * local variables 210 */ 211 static vmem_t *swapmap; /* controls the mapping of /dev/drum */ 212 213 /* list of all active swap devices [by priority] */ 214 LIST_HEAD(swap_priority, swappri); 215 static struct swap_priority swap_priority; 216 217 /* locks */ 218 static krwlock_t swap_syscall_lock; 219 220 /* workqueue and use counter for swap to regular files */ 221 static int sw_reg_count = 0; 222 static struct workqueue *sw_reg_workqueue; 223 224 /* tuneables */ 225 u_int uvm_swapisfull_factor = 99; 226 227 /* 228 * prototypes 229 */ 230 static struct swapdev *swapdrum_getsdp(int); 231 232 static struct swapdev *swaplist_find(struct vnode *, bool); 233 static void swaplist_insert(struct swapdev *, 234 struct swappri *, int); 235 static void swaplist_trim(void); 236 237 static int swap_on(struct lwp *, struct swapdev *); 238 static int swap_off(struct lwp *, struct swapdev *); 239 240 static void uvm_swap_stats(int, struct swapent *, int, register_t *); 241 242 static void sw_reg_strategy(struct swapdev *, struct buf *, int); 243 static void sw_reg_biodone(struct buf *); 244 static void sw_reg_iodone(struct work *wk, void *dummy); 245 static void sw_reg_start(struct swapdev *); 246 247 static int uvm_swap_io(struct vm_page **, int, int, int); 248 249 /* 250 * uvm_swap_init: init the swap system data structures and locks 251 * 252 * => called at boot time from init_main.c after the filesystems 253 * are brought up (which happens after uvm_init()) 254 */ 255 void 256 uvm_swap_init(void) 257 { 258 UVMHIST_FUNC("uvm_swap_init"); 259 260 UVMHIST_CALLED(pdhist); 261 /* 262 * first, init the swap list, its counter, and its lock. 263 * then get a handle on the vnode for /dev/drum by using 264 * the its dev_t number ("swapdev", from MD conf.c). 265 */ 266 267 LIST_INIT(&swap_priority); 268 uvmexp.nswapdev = 0; 269 rw_init(&swap_syscall_lock); 270 mutex_init(&uvm_swap_data_lock, MUTEX_DEFAULT, IPL_NONE); 271 272 if (bdevvp(swapdev, &swapdev_vp)) 273 panic("%s: can't get vnode for swap device", __func__); 274 if (vn_lock(swapdev_vp, LK_EXCLUSIVE | LK_RETRY)) 275 panic("%s: can't lock swap device", __func__); 276 if (VOP_OPEN(swapdev_vp, FREAD | FWRITE, NOCRED)) 277 panic("%s: can't open swap device", __func__); 278 VOP_UNLOCK(swapdev_vp); 279 280 /* 281 * create swap block resource map to map /dev/drum. the range 282 * from 1 to INT_MAX allows 2 gigablocks of swap space. note 283 * that block 0 is reserved (used to indicate an allocation 284 * failure, or no allocation). 285 */ 286 swapmap = vmem_create("swapmap", 1, INT_MAX - 1, 1, NULL, NULL, NULL, 0, 287 VM_NOSLEEP, IPL_NONE); 288 if (swapmap == 0) { 289 panic("%s: vmem_create failed", __func__); 290 } 291 292 pool_init(&vndxfer_pool, sizeof(struct vndxfer), 0, 0, 0, "swp vnx", 293 NULL, IPL_BIO); 294 pool_init(&vndbuf_pool, sizeof(struct vndbuf), 0, 0, 0, "swp vnd", 295 NULL, IPL_BIO); 296 297 UVMHIST_LOG(pdhist, "<- done", 0, 0, 0, 0); 298 } 299 300 /* 301 * swaplist functions: functions that operate on the list of swap 302 * devices on the system. 303 */ 304 305 /* 306 * swaplist_insert: insert swap device "sdp" into the global list 307 * 308 * => caller must hold both swap_syscall_lock and uvm_swap_data_lock 309 * => caller must provide a newly allocated swappri structure (we will 310 * FREE it if we don't need it... this it to prevent allocation 311 * blocking here while adding swap) 312 */ 313 static void 314 swaplist_insert(struct swapdev *sdp, struct swappri *newspp, int priority) 315 { 316 struct swappri *spp, *pspp; 317 UVMHIST_FUNC("swaplist_insert"); UVMHIST_CALLED(pdhist); 318 319 /* 320 * find entry at or after which to insert the new device. 321 */ 322 pspp = NULL; 323 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 324 if (priority <= spp->spi_priority) 325 break; 326 pspp = spp; 327 } 328 329 /* 330 * new priority? 331 */ 332 if (spp == NULL || spp->spi_priority != priority) { 333 spp = newspp; /* use newspp! */ 334 UVMHIST_LOG(pdhist, "created new swappri = %d", 335 priority, 0, 0, 0); 336 337 spp->spi_priority = priority; 338 TAILQ_INIT(&spp->spi_swapdev); 339 340 if (pspp) 341 LIST_INSERT_AFTER(pspp, spp, spi_swappri); 342 else 343 LIST_INSERT_HEAD(&swap_priority, spp, spi_swappri); 344 } else { 345 /* we don't need a new priority structure, free it */ 346 kmem_free(newspp, sizeof(*newspp)); 347 } 348 349 /* 350 * priority found (or created). now insert on the priority's 351 * tailq list and bump the total number of swapdevs. 352 */ 353 sdp->swd_priority = priority; 354 TAILQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next); 355 uvmexp.nswapdev++; 356 } 357 358 /* 359 * swaplist_find: find and optionally remove a swap device from the 360 * global list. 361 * 362 * => caller must hold both swap_syscall_lock and uvm_swap_data_lock 363 * => we return the swapdev we found (and removed) 364 */ 365 static struct swapdev * 366 swaplist_find(struct vnode *vp, bool remove) 367 { 368 struct swapdev *sdp; 369 struct swappri *spp; 370 371 /* 372 * search the lists for the requested vp 373 */ 374 375 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 376 TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 377 if (sdp->swd_vp == vp) { 378 if (remove) { 379 TAILQ_REMOVE(&spp->spi_swapdev, 380 sdp, swd_next); 381 uvmexp.nswapdev--; 382 } 383 return(sdp); 384 } 385 } 386 } 387 return (NULL); 388 } 389 390 /* 391 * swaplist_trim: scan priority list for empty priority entries and kill 392 * them. 393 * 394 * => caller must hold both swap_syscall_lock and uvm_swap_data_lock 395 */ 396 static void 397 swaplist_trim(void) 398 { 399 struct swappri *spp, *nextspp; 400 401 LIST_FOREACH_SAFE(spp, &swap_priority, spi_swappri, nextspp) { 402 if (TAILQ_EMPTY(&spp->spi_swapdev)) 403 continue; 404 LIST_REMOVE(spp, spi_swappri); 405 kmem_free(spp, sizeof(*spp)); 406 } 407 } 408 409 /* 410 * swapdrum_getsdp: given a page offset in /dev/drum, convert it back 411 * to the "swapdev" that maps that section of the drum. 412 * 413 * => each swapdev takes one big contig chunk of the drum 414 * => caller must hold uvm_swap_data_lock 415 */ 416 static struct swapdev * 417 swapdrum_getsdp(int pgno) 418 { 419 struct swapdev *sdp; 420 struct swappri *spp; 421 422 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 423 TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 424 if (sdp->swd_flags & SWF_FAKE) 425 continue; 426 if (pgno >= sdp->swd_drumoffset && 427 pgno < (sdp->swd_drumoffset + sdp->swd_drumsize)) { 428 return sdp; 429 } 430 } 431 } 432 return NULL; 433 } 434 435 436 /* 437 * sys_swapctl: main entry point for swapctl(2) system call 438 * [with two helper functions: swap_on and swap_off] 439 */ 440 int 441 sys_swapctl(struct lwp *l, const struct sys_swapctl_args *uap, register_t *retval) 442 { 443 /* { 444 syscallarg(int) cmd; 445 syscallarg(void *) arg; 446 syscallarg(int) misc; 447 } */ 448 struct vnode *vp; 449 struct nameidata nd; 450 struct swappri *spp; 451 struct swapdev *sdp; 452 struct swapent *sep; 453 #define SWAP_PATH_MAX (PATH_MAX + 1) 454 char *userpath; 455 size_t len = 0; 456 int error, misc; 457 int priority; 458 UVMHIST_FUNC("sys_swapctl"); UVMHIST_CALLED(pdhist); 459 460 /* 461 * we handle the non-priv NSWAP and STATS request first. 462 * 463 * SWAP_NSWAP: return number of config'd swap devices 464 * [can also be obtained with uvmexp sysctl] 465 */ 466 if (SCARG(uap, cmd) == SWAP_NSWAP) { 467 const int nswapdev = uvmexp.nswapdev; 468 UVMHIST_LOG(pdhist, "<- done SWAP_NSWAP=%d", nswapdev, 0, 0, 0); 469 *retval = nswapdev; 470 return 0; 471 } 472 473 misc = SCARG(uap, misc); 474 userpath = kmem_alloc(SWAP_PATH_MAX, KM_SLEEP); 475 476 /* 477 * ensure serialized syscall access by grabbing the swap_syscall_lock 478 */ 479 rw_enter(&swap_syscall_lock, RW_WRITER); 480 481 /* 482 * SWAP_STATS: get stats on current # of configured swap devs 483 * 484 * note that the swap_priority list can't change as long 485 * as we are holding the swap_syscall_lock. we don't want 486 * to grab the uvm_swap_data_lock because we may fault&sleep during 487 * copyout() and we don't want to be holding that lock then! 488 */ 489 if (SCARG(uap, cmd) == SWAP_STATS 490 #if defined(COMPAT_50) 491 || SCARG(uap, cmd) == SWAP_STATS50 492 #endif 493 #if defined(COMPAT_13) 494 || SCARG(uap, cmd) == SWAP_STATS13 495 #endif 496 ) { 497 if ((size_t)misc > (size_t)uvmexp.nswapdev) 498 misc = uvmexp.nswapdev; 499 500 if (misc == 0) { 501 error = EINVAL; 502 goto out; 503 } 504 KASSERT(misc > 0); 505 #if defined(COMPAT_13) 506 if (SCARG(uap, cmd) == SWAP_STATS13) 507 len = sizeof(struct swapent13) * misc; 508 else 509 #endif 510 #if defined(COMPAT_50) 511 if (SCARG(uap, cmd) == SWAP_STATS50) 512 len = sizeof(struct swapent50) * misc; 513 else 514 #endif 515 len = sizeof(struct swapent) * misc; 516 sep = (struct swapent *)kmem_alloc(len, KM_SLEEP); 517 518 uvm_swap_stats(SCARG(uap, cmd), sep, misc, retval); 519 error = copyout(sep, SCARG(uap, arg), len); 520 521 kmem_free(sep, len); 522 UVMHIST_LOG(pdhist, "<- done SWAP_STATS", 0, 0, 0, 0); 523 goto out; 524 } 525 if (SCARG(uap, cmd) == SWAP_GETDUMPDEV) { 526 dev_t *devp = (dev_t *)SCARG(uap, arg); 527 528 error = copyout(&dumpdev, devp, sizeof(dumpdev)); 529 goto out; 530 } 531 532 /* 533 * all other requests require superuser privs. verify. 534 */ 535 if ((error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_SWAPCTL, 536 0, NULL, NULL, NULL))) 537 goto out; 538 539 if (SCARG(uap, cmd) == SWAP_DUMPOFF) { 540 /* drop the current dump device */ 541 dumpdev = NODEV; 542 dumpcdev = NODEV; 543 cpu_dumpconf(); 544 goto out; 545 } 546 547 /* 548 * at this point we expect a path name in arg. we will 549 * use namei() to gain a vnode reference (vref), and lock 550 * the vnode (VOP_LOCK). 551 * 552 * XXX: a NULL arg means use the root vnode pointer (e.g. for 553 * miniroot) 554 */ 555 if (SCARG(uap, arg) == NULL) { 556 vp = rootvp; /* miniroot */ 557 vref(vp); 558 if (vn_lock(vp, LK_EXCLUSIVE)) { 559 vrele(vp); 560 error = EBUSY; 561 goto out; 562 } 563 if (SCARG(uap, cmd) == SWAP_ON && 564 copystr("miniroot", userpath, SWAP_PATH_MAX, &len)) 565 panic("swapctl: miniroot copy failed"); 566 KASSERT(len > 0); 567 } else { 568 struct pathbuf *pb; 569 570 /* 571 * This used to allow copying in one extra byte 572 * (SWAP_PATH_MAX instead of PATH_MAX) for SWAP_ON. 573 * This was completely pointless because if anyone 574 * used that extra byte namei would fail with 575 * ENAMETOOLONG anyway, so I've removed the excess 576 * logic. - dholland 20100215 577 */ 578 579 error = pathbuf_copyin(SCARG(uap, arg), &pb); 580 if (error) { 581 goto out; 582 } 583 if (SCARG(uap, cmd) == SWAP_ON) { 584 /* get a copy of the string */ 585 pathbuf_copystring(pb, userpath, SWAP_PATH_MAX); 586 len = strlen(userpath) + 1; 587 } 588 NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | TRYEMULROOT, pb); 589 if ((error = namei(&nd))) { 590 pathbuf_destroy(pb); 591 goto out; 592 } 593 vp = nd.ni_vp; 594 pathbuf_destroy(pb); 595 } 596 /* note: "vp" is referenced and locked */ 597 598 error = 0; /* assume no error */ 599 switch(SCARG(uap, cmd)) { 600 601 case SWAP_DUMPDEV: 602 if (vp->v_type != VBLK) { 603 error = ENOTBLK; 604 break; 605 } 606 if (bdevsw_lookup(vp->v_rdev)) { 607 dumpdev = vp->v_rdev; 608 dumpcdev = devsw_blk2chr(dumpdev); 609 } else 610 dumpdev = NODEV; 611 cpu_dumpconf(); 612 break; 613 614 case SWAP_CTL: 615 /* 616 * get new priority, remove old entry (if any) and then 617 * reinsert it in the correct place. finally, prune out 618 * any empty priority structures. 619 */ 620 priority = SCARG(uap, misc); 621 spp = kmem_alloc(sizeof(*spp), KM_SLEEP); 622 mutex_enter(&uvm_swap_data_lock); 623 if ((sdp = swaplist_find(vp, true)) == NULL) { 624 error = ENOENT; 625 } else { 626 swaplist_insert(sdp, spp, priority); 627 swaplist_trim(); 628 } 629 mutex_exit(&uvm_swap_data_lock); 630 if (error) 631 kmem_free(spp, sizeof(*spp)); 632 break; 633 634 case SWAP_ON: 635 636 /* 637 * check for duplicates. if none found, then insert a 638 * dummy entry on the list to prevent someone else from 639 * trying to enable this device while we are working on 640 * it. 641 */ 642 643 priority = SCARG(uap, misc); 644 sdp = kmem_zalloc(sizeof(*sdp), KM_SLEEP); 645 spp = kmem_alloc(sizeof(*spp), KM_SLEEP); 646 sdp->swd_flags = SWF_FAKE; 647 sdp->swd_vp = vp; 648 sdp->swd_dev = (vp->v_type == VBLK) ? vp->v_rdev : NODEV; 649 bufq_alloc(&sdp->swd_tab, "disksort", BUFQ_SORT_RAWBLOCK); 650 mutex_enter(&uvm_swap_data_lock); 651 if (swaplist_find(vp, false) != NULL) { 652 error = EBUSY; 653 mutex_exit(&uvm_swap_data_lock); 654 bufq_free(sdp->swd_tab); 655 kmem_free(sdp, sizeof(*sdp)); 656 kmem_free(spp, sizeof(*spp)); 657 break; 658 } 659 swaplist_insert(sdp, spp, priority); 660 mutex_exit(&uvm_swap_data_lock); 661 662 KASSERT(len > 0); 663 sdp->swd_pathlen = len; 664 sdp->swd_path = kmem_alloc(len, KM_SLEEP); 665 if (copystr(userpath, sdp->swd_path, len, 0) != 0) 666 panic("swapctl: copystr"); 667 668 /* 669 * we've now got a FAKE placeholder in the swap list. 670 * now attempt to enable swap on it. if we fail, undo 671 * what we've done and kill the fake entry we just inserted. 672 * if swap_on is a success, it will clear the SWF_FAKE flag 673 */ 674 675 if ((error = swap_on(l, sdp)) != 0) { 676 mutex_enter(&uvm_swap_data_lock); 677 (void) swaplist_find(vp, true); /* kill fake entry */ 678 swaplist_trim(); 679 mutex_exit(&uvm_swap_data_lock); 680 bufq_free(sdp->swd_tab); 681 kmem_free(sdp->swd_path, sdp->swd_pathlen); 682 kmem_free(sdp, sizeof(*sdp)); 683 break; 684 } 685 break; 686 687 case SWAP_OFF: 688 mutex_enter(&uvm_swap_data_lock); 689 if ((sdp = swaplist_find(vp, false)) == NULL) { 690 mutex_exit(&uvm_swap_data_lock); 691 error = ENXIO; 692 break; 693 } 694 695 /* 696 * If a device isn't in use or enabled, we 697 * can't stop swapping from it (again). 698 */ 699 if ((sdp->swd_flags & (SWF_INUSE|SWF_ENABLE)) == 0) { 700 mutex_exit(&uvm_swap_data_lock); 701 error = EBUSY; 702 break; 703 } 704 705 /* 706 * do the real work. 707 */ 708 error = swap_off(l, sdp); 709 break; 710 711 default: 712 error = EINVAL; 713 } 714 715 /* 716 * done! release the ref gained by namei() and unlock. 717 */ 718 vput(vp); 719 out: 720 rw_exit(&swap_syscall_lock); 721 kmem_free(userpath, SWAP_PATH_MAX); 722 723 UVMHIST_LOG(pdhist, "<- done! error=%d", error, 0, 0, 0); 724 return (error); 725 } 726 727 /* 728 * uvm_swap_stats: implements swapctl(SWAP_STATS). The function is kept 729 * away from sys_swapctl() in order to allow COMPAT_* swapctl() 730 * emulation to use it directly without going through sys_swapctl(). 731 * The problem with using sys_swapctl() there is that it involves 732 * copying the swapent array to the stackgap, and this array's size 733 * is not known at build time. Hence it would not be possible to 734 * ensure it would fit in the stackgap in any case. 735 */ 736 static void 737 uvm_swap_stats(int cmd, struct swapent *sep, int sec, register_t *retval) 738 { 739 struct swappri *spp; 740 struct swapdev *sdp; 741 int count = 0; 742 743 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 744 TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 745 int inuse; 746 747 if (sec-- <= 0) 748 break; 749 750 /* 751 * backwards compatibility for system call. 752 * For NetBSD 1.3 and 5.0, we have to use 753 * the 32 bit dev_t. For 5.0 and -current 754 * we have to add the path. 755 */ 756 inuse = btodb((uint64_t)sdp->swd_npginuse << 757 PAGE_SHIFT); 758 759 #if defined(COMPAT_13) || defined(COMPAT_50) 760 if (cmd == SWAP_STATS) { 761 #endif 762 sep->se_dev = sdp->swd_dev; 763 sep->se_flags = sdp->swd_flags; 764 sep->se_nblks = sdp->swd_nblks; 765 sep->se_inuse = inuse; 766 sep->se_priority = sdp->swd_priority; 767 KASSERT(sdp->swd_pathlen < 768 sizeof(sep->se_path)); 769 strcpy(sep->se_path, sdp->swd_path); 770 sep++; 771 #if defined(COMPAT_13) 772 } else if (cmd == SWAP_STATS13) { 773 struct swapent13 *sep13 = 774 (struct swapent13 *)sep; 775 776 sep13->se13_dev = sdp->swd_dev; 777 sep13->se13_flags = sdp->swd_flags; 778 sep13->se13_nblks = sdp->swd_nblks; 779 sep13->se13_inuse = inuse; 780 sep13->se13_priority = sdp->swd_priority; 781 sep = (struct swapent *)(sep13 + 1); 782 #endif 783 #if defined(COMPAT_50) 784 } else if (cmd == SWAP_STATS50) { 785 struct swapent50 *sep50 = 786 (struct swapent50 *)sep; 787 788 sep50->se50_dev = sdp->swd_dev; 789 sep50->se50_flags = sdp->swd_flags; 790 sep50->se50_nblks = sdp->swd_nblks; 791 sep50->se50_inuse = inuse; 792 sep50->se50_priority = sdp->swd_priority; 793 KASSERT(sdp->swd_pathlen < 794 sizeof(sep50->se50_path)); 795 strcpy(sep50->se50_path, sdp->swd_path); 796 sep = (struct swapent *)(sep50 + 1); 797 #endif 798 #if defined(COMPAT_13) || defined(COMPAT_50) 799 } 800 #endif 801 count++; 802 } 803 } 804 *retval = count; 805 } 806 807 /* 808 * swap_on: attempt to enable a swapdev for swapping. note that the 809 * swapdev is already on the global list, but disabled (marked 810 * SWF_FAKE). 811 * 812 * => we avoid the start of the disk (to protect disk labels) 813 * => we also avoid the miniroot, if we are swapping to root. 814 * => caller should leave uvm_swap_data_lock unlocked, we may lock it 815 * if needed. 816 */ 817 static int 818 swap_on(struct lwp *l, struct swapdev *sdp) 819 { 820 struct vnode *vp; 821 int error, npages, nblocks, size; 822 long addr; 823 vmem_addr_t result; 824 struct vattr va; 825 dev_t dev; 826 UVMHIST_FUNC("swap_on"); UVMHIST_CALLED(pdhist); 827 828 /* 829 * we want to enable swapping on sdp. the swd_vp contains 830 * the vnode we want (locked and ref'd), and the swd_dev 831 * contains the dev_t of the file, if it a block device. 832 */ 833 834 vp = sdp->swd_vp; 835 dev = sdp->swd_dev; 836 837 /* 838 * open the swap file (mostly useful for block device files to 839 * let device driver know what is up). 840 * 841 * we skip the open/close for root on swap because the root 842 * has already been opened when root was mounted (mountroot). 843 */ 844 if (vp != rootvp) { 845 if ((error = VOP_OPEN(vp, FREAD|FWRITE, l->l_cred))) 846 return (error); 847 } 848 849 /* XXX this only works for block devices */ 850 UVMHIST_LOG(pdhist, " dev=%d, major(dev)=%d", dev, major(dev), 0,0); 851 852 /* 853 * we now need to determine the size of the swap area. for 854 * block specials we can call the d_psize function. 855 * for normal files, we must stat [get attrs]. 856 * 857 * we put the result in nblks. 858 * for normal files, we also want the filesystem block size 859 * (which we get with statfs). 860 */ 861 switch (vp->v_type) { 862 case VBLK: 863 if ((nblocks = bdev_size(dev)) == -1) { 864 error = ENXIO; 865 goto bad; 866 } 867 break; 868 869 case VREG: 870 if ((error = VOP_GETATTR(vp, &va, l->l_cred))) 871 goto bad; 872 nblocks = (int)btodb(va.va_size); 873 sdp->swd_bsize = 1 << vp->v_mount->mnt_fs_bshift; 874 /* 875 * limit the max # of outstanding I/O requests we issue 876 * at any one time. take it easy on NFS servers. 877 */ 878 if (vp->v_tag == VT_NFS) 879 sdp->swd_maxactive = 2; /* XXX */ 880 else 881 sdp->swd_maxactive = 8; /* XXX */ 882 break; 883 884 default: 885 error = ENXIO; 886 goto bad; 887 } 888 889 /* 890 * save nblocks in a safe place and convert to pages. 891 */ 892 893 sdp->swd_nblks = nblocks; 894 npages = dbtob((uint64_t)nblocks) >> PAGE_SHIFT; 895 896 /* 897 * for block special files, we want to make sure that leave 898 * the disklabel and bootblocks alone, so we arrange to skip 899 * over them (arbitrarily choosing to skip PAGE_SIZE bytes). 900 * note that because of this the "size" can be less than the 901 * actual number of blocks on the device. 902 */ 903 if (vp->v_type == VBLK) { 904 /* we use pages 1 to (size - 1) [inclusive] */ 905 size = npages - 1; 906 addr = 1; 907 } else { 908 /* we use pages 0 to (size - 1) [inclusive] */ 909 size = npages; 910 addr = 0; 911 } 912 913 /* 914 * make sure we have enough blocks for a reasonable sized swap 915 * area. we want at least one page. 916 */ 917 918 if (size < 1) { 919 UVMHIST_LOG(pdhist, " size <= 1!!", 0, 0, 0, 0); 920 error = EINVAL; 921 goto bad; 922 } 923 924 UVMHIST_LOG(pdhist, " dev=%x: size=%d addr=%ld\n", dev, size, addr, 0); 925 926 /* 927 * now we need to allocate an extent to manage this swap device 928 */ 929 930 sdp->swd_blist = blist_create(npages); 931 /* mark all expect the `saved' region free. */ 932 blist_free(sdp->swd_blist, addr, size); 933 934 /* 935 * if the vnode we are swapping to is the root vnode 936 * (i.e. we are swapping to the miniroot) then we want 937 * to make sure we don't overwrite it. do a statfs to 938 * find its size and skip over it. 939 */ 940 if (vp == rootvp) { 941 struct mount *mp; 942 struct statvfs *sp; 943 int rootblocks, rootpages; 944 945 mp = rootvnode->v_mount; 946 sp = &mp->mnt_stat; 947 rootblocks = sp->f_blocks * btodb(sp->f_frsize); 948 /* 949 * XXX: sp->f_blocks isn't the total number of 950 * blocks in the filesystem, it's the number of 951 * data blocks. so, our rootblocks almost 952 * definitely underestimates the total size 953 * of the filesystem - how badly depends on the 954 * details of the filesystem type. there isn't 955 * an obvious way to deal with this cleanly 956 * and perfectly, so for now we just pad our 957 * rootblocks estimate with an extra 5 percent. 958 */ 959 rootblocks += (rootblocks >> 5) + 960 (rootblocks >> 6) + 961 (rootblocks >> 7); 962 rootpages = round_page(dbtob(rootblocks)) >> PAGE_SHIFT; 963 if (rootpages > size) 964 panic("swap_on: miniroot larger than swap?"); 965 966 if (rootpages != blist_fill(sdp->swd_blist, addr, rootpages)) { 967 panic("swap_on: unable to preserve miniroot"); 968 } 969 970 size -= rootpages; 971 printf("Preserved %d pages of miniroot ", rootpages); 972 printf("leaving %d pages of swap\n", size); 973 } 974 975 /* 976 * add a ref to vp to reflect usage as a swap device. 977 */ 978 vref(vp); 979 980 /* 981 * now add the new swapdev to the drum and enable. 982 */ 983 error = vmem_alloc(swapmap, npages, VM_BESTFIT | VM_SLEEP, &result); 984 if (error != 0) 985 panic("swapdrum_add"); 986 /* 987 * If this is the first regular swap create the workqueue. 988 * => Protected by swap_syscall_lock. 989 */ 990 if (vp->v_type != VBLK) { 991 if (sw_reg_count++ == 0) { 992 KASSERT(sw_reg_workqueue == NULL); 993 if (workqueue_create(&sw_reg_workqueue, "swapiod", 994 sw_reg_iodone, NULL, PRIBIO, IPL_BIO, 0) != 0) 995 panic("%s: workqueue_create failed", __func__); 996 } 997 } 998 999 sdp->swd_drumoffset = (int)result; 1000 sdp->swd_drumsize = npages; 1001 sdp->swd_npages = size; 1002 mutex_enter(&uvm_swap_data_lock); 1003 sdp->swd_flags &= ~SWF_FAKE; /* going live */ 1004 sdp->swd_flags |= (SWF_INUSE|SWF_ENABLE); 1005 uvmexp.swpages += size; 1006 uvmexp.swpgavail += size; 1007 mutex_exit(&uvm_swap_data_lock); 1008 return (0); 1009 1010 /* 1011 * failure: clean up and return error. 1012 */ 1013 1014 bad: 1015 if (sdp->swd_blist) { 1016 blist_destroy(sdp->swd_blist); 1017 } 1018 if (vp != rootvp) { 1019 (void)VOP_CLOSE(vp, FREAD|FWRITE, l->l_cred); 1020 } 1021 return (error); 1022 } 1023 1024 /* 1025 * swap_off: stop swapping on swapdev 1026 * 1027 * => swap data should be locked, we will unlock. 1028 */ 1029 static int 1030 swap_off(struct lwp *l, struct swapdev *sdp) 1031 { 1032 int npages = sdp->swd_npages; 1033 int error = 0; 1034 1035 UVMHIST_FUNC("swap_off"); UVMHIST_CALLED(pdhist); 1036 UVMHIST_LOG(pdhist, " dev=%x, npages=%d", sdp->swd_dev,npages,0,0); 1037 1038 /* disable the swap area being removed */ 1039 sdp->swd_flags &= ~SWF_ENABLE; 1040 uvmexp.swpgavail -= npages; 1041 mutex_exit(&uvm_swap_data_lock); 1042 1043 /* 1044 * the idea is to find all the pages that are paged out to this 1045 * device, and page them all in. in uvm, swap-backed pageable 1046 * memory can take two forms: aobjs and anons. call the 1047 * swapoff hook for each subsystem to bring in pages. 1048 */ 1049 1050 if (uao_swap_off(sdp->swd_drumoffset, 1051 sdp->swd_drumoffset + sdp->swd_drumsize) || 1052 amap_swap_off(sdp->swd_drumoffset, 1053 sdp->swd_drumoffset + sdp->swd_drumsize)) { 1054 error = ENOMEM; 1055 } else if (sdp->swd_npginuse > sdp->swd_npgbad) { 1056 error = EBUSY; 1057 } 1058 1059 if (error) { 1060 mutex_enter(&uvm_swap_data_lock); 1061 sdp->swd_flags |= SWF_ENABLE; 1062 uvmexp.swpgavail += npages; 1063 mutex_exit(&uvm_swap_data_lock); 1064 1065 return error; 1066 } 1067 1068 /* 1069 * If this is the last regular swap destroy the workqueue. 1070 * => Protected by swap_syscall_lock. 1071 */ 1072 if (sdp->swd_vp->v_type != VBLK) { 1073 KASSERT(sw_reg_count > 0); 1074 KASSERT(sw_reg_workqueue != NULL); 1075 if (--sw_reg_count == 0) { 1076 workqueue_destroy(sw_reg_workqueue); 1077 sw_reg_workqueue = NULL; 1078 } 1079 } 1080 1081 /* 1082 * done with the vnode. 1083 * drop our ref on the vnode before calling VOP_CLOSE() 1084 * so that spec_close() can tell if this is the last close. 1085 */ 1086 vrele(sdp->swd_vp); 1087 if (sdp->swd_vp != rootvp) { 1088 (void) VOP_CLOSE(sdp->swd_vp, FREAD|FWRITE, l->l_cred); 1089 } 1090 1091 mutex_enter(&uvm_swap_data_lock); 1092 uvmexp.swpages -= npages; 1093 uvmexp.swpginuse -= sdp->swd_npgbad; 1094 1095 if (swaplist_find(sdp->swd_vp, true) == NULL) 1096 panic("%s: swapdev not in list", __func__); 1097 swaplist_trim(); 1098 mutex_exit(&uvm_swap_data_lock); 1099 1100 /* 1101 * free all resources! 1102 */ 1103 vmem_free(swapmap, sdp->swd_drumoffset, sdp->swd_drumsize); 1104 blist_destroy(sdp->swd_blist); 1105 bufq_free(sdp->swd_tab); 1106 kmem_free(sdp, sizeof(*sdp)); 1107 return (0); 1108 } 1109 1110 void 1111 uvm_swap_shutdown(struct lwp *l) 1112 { 1113 struct swapdev *sdp; 1114 struct swappri *spp; 1115 struct vnode *vp; 1116 int error; 1117 1118 printf("turning of swap..."); 1119 rw_enter(&swap_syscall_lock, RW_WRITER); 1120 mutex_enter(&uvm_swap_data_lock); 1121 again: 1122 LIST_FOREACH(spp, &swap_priority, spi_swappri) 1123 TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 1124 if (sdp->swd_flags & SWF_FAKE) 1125 continue; 1126 if ((sdp->swd_flags & (SWF_INUSE|SWF_ENABLE)) == 0) 1127 continue; 1128 #ifdef DEBUG 1129 printf("\nturning off swap on %s...", 1130 sdp->swd_path); 1131 #endif 1132 if (vn_lock(vp = sdp->swd_vp, LK_EXCLUSIVE)) { 1133 error = EBUSY; 1134 vp = NULL; 1135 } else 1136 error = 0; 1137 if (!error) { 1138 error = swap_off(l, sdp); 1139 mutex_enter(&uvm_swap_data_lock); 1140 } 1141 if (error) { 1142 printf("stopping swap on %s failed " 1143 "with error %d\n", sdp->swd_path, error); 1144 TAILQ_REMOVE(&spp->spi_swapdev, sdp, 1145 swd_next); 1146 uvmexp.nswapdev--; 1147 swaplist_trim(); 1148 if (vp) 1149 vput(vp); 1150 } 1151 goto again; 1152 } 1153 printf(" done\n"); 1154 mutex_exit(&uvm_swap_data_lock); 1155 rw_exit(&swap_syscall_lock); 1156 } 1157 1158 1159 /* 1160 * /dev/drum interface and i/o functions 1161 */ 1162 1163 /* 1164 * swstrategy: perform I/O on the drum 1165 * 1166 * => we must map the i/o request from the drum to the correct swapdev. 1167 */ 1168 static void 1169 swstrategy(struct buf *bp) 1170 { 1171 struct swapdev *sdp; 1172 struct vnode *vp; 1173 int pageno, bn; 1174 UVMHIST_FUNC("swstrategy"); UVMHIST_CALLED(pdhist); 1175 1176 /* 1177 * convert block number to swapdev. note that swapdev can't 1178 * be yanked out from under us because we are holding resources 1179 * in it (i.e. the blocks we are doing I/O on). 1180 */ 1181 pageno = dbtob((int64_t)bp->b_blkno) >> PAGE_SHIFT; 1182 mutex_enter(&uvm_swap_data_lock); 1183 sdp = swapdrum_getsdp(pageno); 1184 mutex_exit(&uvm_swap_data_lock); 1185 if (sdp == NULL) { 1186 bp->b_error = EINVAL; 1187 bp->b_resid = bp->b_bcount; 1188 biodone(bp); 1189 UVMHIST_LOG(pdhist, " failed to get swap device", 0, 0, 0, 0); 1190 return; 1191 } 1192 1193 /* 1194 * convert drum page number to block number on this swapdev. 1195 */ 1196 1197 pageno -= sdp->swd_drumoffset; /* page # on swapdev */ 1198 bn = btodb((uint64_t)pageno << PAGE_SHIFT); /* convert to diskblock */ 1199 1200 UVMHIST_LOG(pdhist, " %s: mapoff=%x bn=%x bcount=%ld", 1201 ((bp->b_flags & B_READ) == 0) ? "write" : "read", 1202 sdp->swd_drumoffset, bn, bp->b_bcount); 1203 1204 /* 1205 * for block devices we finish up here. 1206 * for regular files we have to do more work which we delegate 1207 * to sw_reg_strategy(). 1208 */ 1209 1210 vp = sdp->swd_vp; /* swapdev vnode pointer */ 1211 switch (vp->v_type) { 1212 default: 1213 panic("%s: vnode type 0x%x", __func__, vp->v_type); 1214 1215 case VBLK: 1216 1217 /* 1218 * must convert "bp" from an I/O on /dev/drum to an I/O 1219 * on the swapdev (sdp). 1220 */ 1221 bp->b_blkno = bn; /* swapdev block number */ 1222 bp->b_dev = sdp->swd_dev; /* swapdev dev_t */ 1223 1224 /* 1225 * if we are doing a write, we have to redirect the i/o on 1226 * drum's v_numoutput counter to the swapdevs. 1227 */ 1228 if ((bp->b_flags & B_READ) == 0) { 1229 mutex_enter(bp->b_objlock); 1230 vwakeup(bp); /* kills one 'v_numoutput' on drum */ 1231 mutex_exit(bp->b_objlock); 1232 mutex_enter(vp->v_interlock); 1233 vp->v_numoutput++; /* put it on swapdev */ 1234 mutex_exit(vp->v_interlock); 1235 } 1236 1237 /* 1238 * finally plug in swapdev vnode and start I/O 1239 */ 1240 bp->b_vp = vp; 1241 bp->b_objlock = vp->v_interlock; 1242 VOP_STRATEGY(vp, bp); 1243 return; 1244 1245 case VREG: 1246 /* 1247 * delegate to sw_reg_strategy function. 1248 */ 1249 sw_reg_strategy(sdp, bp, bn); 1250 return; 1251 } 1252 /* NOTREACHED */ 1253 } 1254 1255 /* 1256 * swread: the read function for the drum (just a call to physio) 1257 */ 1258 /*ARGSUSED*/ 1259 static int 1260 swread(dev_t dev, struct uio *uio, int ioflag) 1261 { 1262 UVMHIST_FUNC("swread"); UVMHIST_CALLED(pdhist); 1263 1264 UVMHIST_LOG(pdhist, " dev=%x offset=%qx", dev, uio->uio_offset, 0, 0); 1265 return (physio(swstrategy, NULL, dev, B_READ, minphys, uio)); 1266 } 1267 1268 /* 1269 * swwrite: the write function for the drum (just a call to physio) 1270 */ 1271 /*ARGSUSED*/ 1272 static int 1273 swwrite(dev_t dev, struct uio *uio, int ioflag) 1274 { 1275 UVMHIST_FUNC("swwrite"); UVMHIST_CALLED(pdhist); 1276 1277 UVMHIST_LOG(pdhist, " dev=%x offset=%qx", dev, uio->uio_offset, 0, 0); 1278 return (physio(swstrategy, NULL, dev, B_WRITE, minphys, uio)); 1279 } 1280 1281 const struct bdevsw swap_bdevsw = { 1282 nullopen, nullclose, swstrategy, noioctl, nodump, nosize, D_OTHER, 1283 }; 1284 1285 const struct cdevsw swap_cdevsw = { 1286 nullopen, nullclose, swread, swwrite, noioctl, 1287 nostop, notty, nopoll, nommap, nokqfilter, D_OTHER, 1288 }; 1289 1290 /* 1291 * sw_reg_strategy: handle swap i/o to regular files 1292 */ 1293 static void 1294 sw_reg_strategy(struct swapdev *sdp, struct buf *bp, int bn) 1295 { 1296 struct vnode *vp; 1297 struct vndxfer *vnx; 1298 daddr_t nbn; 1299 char *addr; 1300 off_t byteoff; 1301 int s, off, nra, error, sz, resid; 1302 UVMHIST_FUNC("sw_reg_strategy"); UVMHIST_CALLED(pdhist); 1303 1304 /* 1305 * allocate a vndxfer head for this transfer and point it to 1306 * our buffer. 1307 */ 1308 vnx = pool_get(&vndxfer_pool, PR_WAITOK); 1309 vnx->vx_flags = VX_BUSY; 1310 vnx->vx_error = 0; 1311 vnx->vx_pending = 0; 1312 vnx->vx_bp = bp; 1313 vnx->vx_sdp = sdp; 1314 1315 /* 1316 * setup for main loop where we read filesystem blocks into 1317 * our buffer. 1318 */ 1319 error = 0; 1320 bp->b_resid = bp->b_bcount; /* nothing transfered yet! */ 1321 addr = bp->b_data; /* current position in buffer */ 1322 byteoff = dbtob((uint64_t)bn); 1323 1324 for (resid = bp->b_resid; resid; resid -= sz) { 1325 struct vndbuf *nbp; 1326 1327 /* 1328 * translate byteoffset into block number. return values: 1329 * vp = vnode of underlying device 1330 * nbn = new block number (on underlying vnode dev) 1331 * nra = num blocks we can read-ahead (excludes requested 1332 * block) 1333 */ 1334 nra = 0; 1335 error = VOP_BMAP(sdp->swd_vp, byteoff / sdp->swd_bsize, 1336 &vp, &nbn, &nra); 1337 1338 if (error == 0 && nbn == (daddr_t)-1) { 1339 /* 1340 * this used to just set error, but that doesn't 1341 * do the right thing. Instead, it causes random 1342 * memory errors. The panic() should remain until 1343 * this condition doesn't destabilize the system. 1344 */ 1345 #if 1 1346 panic("%s: swap to sparse file", __func__); 1347 #else 1348 error = EIO; /* failure */ 1349 #endif 1350 } 1351 1352 /* 1353 * punt if there was an error or a hole in the file. 1354 * we must wait for any i/o ops we have already started 1355 * to finish before returning. 1356 * 1357 * XXX we could deal with holes here but it would be 1358 * a hassle (in the write case). 1359 */ 1360 if (error) { 1361 s = splbio(); 1362 vnx->vx_error = error; /* pass error up */ 1363 goto out; 1364 } 1365 1366 /* 1367 * compute the size ("sz") of this transfer (in bytes). 1368 */ 1369 off = byteoff % sdp->swd_bsize; 1370 sz = (1 + nra) * sdp->swd_bsize - off; 1371 if (sz > resid) 1372 sz = resid; 1373 1374 UVMHIST_LOG(pdhist, "sw_reg_strategy: " 1375 "vp %p/%p offset 0x%x/0x%x", 1376 sdp->swd_vp, vp, byteoff, nbn); 1377 1378 /* 1379 * now get a buf structure. note that the vb_buf is 1380 * at the front of the nbp structure so that you can 1381 * cast pointers between the two structure easily. 1382 */ 1383 nbp = pool_get(&vndbuf_pool, PR_WAITOK); 1384 buf_init(&nbp->vb_buf); 1385 nbp->vb_buf.b_flags = bp->b_flags; 1386 nbp->vb_buf.b_cflags = bp->b_cflags; 1387 nbp->vb_buf.b_oflags = bp->b_oflags; 1388 nbp->vb_buf.b_bcount = sz; 1389 nbp->vb_buf.b_bufsize = sz; 1390 nbp->vb_buf.b_error = 0; 1391 nbp->vb_buf.b_data = addr; 1392 nbp->vb_buf.b_lblkno = 0; 1393 nbp->vb_buf.b_blkno = nbn + btodb(off); 1394 nbp->vb_buf.b_rawblkno = nbp->vb_buf.b_blkno; 1395 nbp->vb_buf.b_iodone = sw_reg_biodone; 1396 nbp->vb_buf.b_vp = vp; 1397 nbp->vb_buf.b_objlock = vp->v_interlock; 1398 if (vp->v_type == VBLK) { 1399 nbp->vb_buf.b_dev = vp->v_rdev; 1400 } 1401 1402 nbp->vb_xfer = vnx; /* patch it back in to vnx */ 1403 1404 /* 1405 * Just sort by block number 1406 */ 1407 s = splbio(); 1408 if (vnx->vx_error != 0) { 1409 buf_destroy(&nbp->vb_buf); 1410 pool_put(&vndbuf_pool, nbp); 1411 goto out; 1412 } 1413 vnx->vx_pending++; 1414 1415 /* sort it in and start I/O if we are not over our limit */ 1416 /* XXXAD locking */ 1417 bufq_put(sdp->swd_tab, &nbp->vb_buf); 1418 sw_reg_start(sdp); 1419 splx(s); 1420 1421 /* 1422 * advance to the next I/O 1423 */ 1424 byteoff += sz; 1425 addr += sz; 1426 } 1427 1428 s = splbio(); 1429 1430 out: /* Arrive here at splbio */ 1431 vnx->vx_flags &= ~VX_BUSY; 1432 if (vnx->vx_pending == 0) { 1433 error = vnx->vx_error; 1434 pool_put(&vndxfer_pool, vnx); 1435 bp->b_error = error; 1436 biodone(bp); 1437 } 1438 splx(s); 1439 } 1440 1441 /* 1442 * sw_reg_start: start an I/O request on the requested swapdev 1443 * 1444 * => reqs are sorted by b_rawblkno (above) 1445 */ 1446 static void 1447 sw_reg_start(struct swapdev *sdp) 1448 { 1449 struct buf *bp; 1450 struct vnode *vp; 1451 UVMHIST_FUNC("sw_reg_start"); UVMHIST_CALLED(pdhist); 1452 1453 /* recursion control */ 1454 if ((sdp->swd_flags & SWF_BUSY) != 0) 1455 return; 1456 1457 sdp->swd_flags |= SWF_BUSY; 1458 1459 while (sdp->swd_active < sdp->swd_maxactive) { 1460 bp = bufq_get(sdp->swd_tab); 1461 if (bp == NULL) 1462 break; 1463 sdp->swd_active++; 1464 1465 UVMHIST_LOG(pdhist, 1466 "sw_reg_start: bp %p vp %p blkno %p cnt %lx", 1467 bp, bp->b_vp, bp->b_blkno, bp->b_bcount); 1468 vp = bp->b_vp; 1469 KASSERT(bp->b_objlock == vp->v_interlock); 1470 if ((bp->b_flags & B_READ) == 0) { 1471 mutex_enter(vp->v_interlock); 1472 vp->v_numoutput++; 1473 mutex_exit(vp->v_interlock); 1474 } 1475 VOP_STRATEGY(vp, bp); 1476 } 1477 sdp->swd_flags &= ~SWF_BUSY; 1478 } 1479 1480 /* 1481 * sw_reg_biodone: one of our i/o's has completed 1482 */ 1483 static void 1484 sw_reg_biodone(struct buf *bp) 1485 { 1486 workqueue_enqueue(sw_reg_workqueue, &bp->b_work, NULL); 1487 } 1488 1489 /* 1490 * sw_reg_iodone: one of our i/o's has completed and needs post-i/o cleanup 1491 * 1492 * => note that we can recover the vndbuf struct by casting the buf ptr 1493 */ 1494 static void 1495 sw_reg_iodone(struct work *wk, void *dummy) 1496 { 1497 struct vndbuf *vbp = (void *)wk; 1498 struct vndxfer *vnx = vbp->vb_xfer; 1499 struct buf *pbp = vnx->vx_bp; /* parent buffer */ 1500 struct swapdev *sdp = vnx->vx_sdp; 1501 int s, resid, error; 1502 KASSERT(&vbp->vb_buf.b_work == wk); 1503 UVMHIST_FUNC("sw_reg_iodone"); UVMHIST_CALLED(pdhist); 1504 1505 UVMHIST_LOG(pdhist, " vbp=%p vp=%p blkno=%x addr=%p", 1506 vbp, vbp->vb_buf.b_vp, vbp->vb_buf.b_blkno, vbp->vb_buf.b_data); 1507 UVMHIST_LOG(pdhist, " cnt=%lx resid=%lx", 1508 vbp->vb_buf.b_bcount, vbp->vb_buf.b_resid, 0, 0); 1509 1510 /* 1511 * protect vbp at splbio and update. 1512 */ 1513 1514 s = splbio(); 1515 resid = vbp->vb_buf.b_bcount - vbp->vb_buf.b_resid; 1516 pbp->b_resid -= resid; 1517 vnx->vx_pending--; 1518 1519 if (vbp->vb_buf.b_error != 0) { 1520 /* pass error upward */ 1521 error = vbp->vb_buf.b_error ? vbp->vb_buf.b_error : EIO; 1522 UVMHIST_LOG(pdhist, " got error=%d !", error, 0, 0, 0); 1523 vnx->vx_error = error; 1524 } 1525 1526 /* 1527 * kill vbp structure 1528 */ 1529 buf_destroy(&vbp->vb_buf); 1530 pool_put(&vndbuf_pool, vbp); 1531 1532 /* 1533 * wrap up this transaction if it has run to completion or, in 1534 * case of an error, when all auxiliary buffers have returned. 1535 */ 1536 if (vnx->vx_error != 0) { 1537 /* pass error upward */ 1538 error = vnx->vx_error; 1539 if ((vnx->vx_flags & VX_BUSY) == 0 && vnx->vx_pending == 0) { 1540 pbp->b_error = error; 1541 biodone(pbp); 1542 pool_put(&vndxfer_pool, vnx); 1543 } 1544 } else if (pbp->b_resid == 0) { 1545 KASSERT(vnx->vx_pending == 0); 1546 if ((vnx->vx_flags & VX_BUSY) == 0) { 1547 UVMHIST_LOG(pdhist, " iodone error=%d !", 1548 pbp, vnx->vx_error, 0, 0); 1549 biodone(pbp); 1550 pool_put(&vndxfer_pool, vnx); 1551 } 1552 } 1553 1554 /* 1555 * done! start next swapdev I/O if one is pending 1556 */ 1557 sdp->swd_active--; 1558 sw_reg_start(sdp); 1559 splx(s); 1560 } 1561 1562 1563 /* 1564 * uvm_swap_alloc: allocate space on swap 1565 * 1566 * => allocation is done "round robin" down the priority list, as we 1567 * allocate in a priority we "rotate" the circle queue. 1568 * => space can be freed with uvm_swap_free 1569 * => we return the page slot number in /dev/drum (0 == invalid slot) 1570 * => we lock uvm_swap_data_lock 1571 * => XXXMRG: "LESSOK" INTERFACE NEEDED TO EXTENT SYSTEM 1572 */ 1573 int 1574 uvm_swap_alloc(int *nslots /* IN/OUT */, bool lessok) 1575 { 1576 struct swapdev *sdp; 1577 struct swappri *spp; 1578 UVMHIST_FUNC("uvm_swap_alloc"); UVMHIST_CALLED(pdhist); 1579 1580 /* 1581 * no swap devices configured yet? definite failure. 1582 */ 1583 if (uvmexp.nswapdev < 1) 1584 return 0; 1585 1586 /* 1587 * XXXJAK: BEGIN HACK 1588 * 1589 * blist_alloc() in subr_blist.c will panic if we try to allocate 1590 * too many slots. 1591 */ 1592 if (*nslots > BLIST_MAX_ALLOC) { 1593 if (__predict_false(lessok == false)) 1594 return 0; 1595 *nslots = BLIST_MAX_ALLOC; 1596 } 1597 /* XXXJAK: END HACK */ 1598 1599 /* 1600 * lock data lock, convert slots into blocks, and enter loop 1601 */ 1602 mutex_enter(&uvm_swap_data_lock); 1603 1604 ReTry: /* XXXMRG */ 1605 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 1606 TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 1607 uint64_t result; 1608 1609 /* if it's not enabled, then we can't swap from it */ 1610 if ((sdp->swd_flags & SWF_ENABLE) == 0) 1611 continue; 1612 if (sdp->swd_npginuse + *nslots > sdp->swd_npages) 1613 continue; 1614 result = blist_alloc(sdp->swd_blist, *nslots); 1615 if (result == BLIST_NONE) { 1616 continue; 1617 } 1618 KASSERT(result < sdp->swd_drumsize); 1619 1620 /* 1621 * successful allocation! now rotate the tailq. 1622 */ 1623 TAILQ_REMOVE(&spp->spi_swapdev, sdp, swd_next); 1624 TAILQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next); 1625 sdp->swd_npginuse += *nslots; 1626 uvmexp.swpginuse += *nslots; 1627 mutex_exit(&uvm_swap_data_lock); 1628 /* done! return drum slot number */ 1629 UVMHIST_LOG(pdhist, 1630 "success! returning %d slots starting at %d", 1631 *nslots, result + sdp->swd_drumoffset, 0, 0); 1632 return (result + sdp->swd_drumoffset); 1633 } 1634 } 1635 1636 /* XXXMRG: BEGIN HACK */ 1637 if (*nslots > 1 && lessok) { 1638 *nslots = 1; 1639 /* XXXMRG: ugh! blist should support this for us */ 1640 goto ReTry; 1641 } 1642 /* XXXMRG: END HACK */ 1643 1644 mutex_exit(&uvm_swap_data_lock); 1645 return 0; 1646 } 1647 1648 /* 1649 * uvm_swapisfull: return true if most of available swap is allocated 1650 * and in use. we don't count some small portion as it may be inaccessible 1651 * to us at any given moment, for example if there is lock contention or if 1652 * pages are busy. 1653 */ 1654 bool 1655 uvm_swapisfull(void) 1656 { 1657 int swpgonly; 1658 bool rv; 1659 1660 mutex_enter(&uvm_swap_data_lock); 1661 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1662 swpgonly = (int)((uint64_t)uvmexp.swpgonly * 100 / 1663 uvm_swapisfull_factor); 1664 rv = (swpgonly >= uvmexp.swpgavail); 1665 mutex_exit(&uvm_swap_data_lock); 1666 1667 return (rv); 1668 } 1669 1670 /* 1671 * uvm_swap_markbad: keep track of swap ranges where we've had i/o errors 1672 * 1673 * => we lock uvm_swap_data_lock 1674 */ 1675 void 1676 uvm_swap_markbad(int startslot, int nslots) 1677 { 1678 struct swapdev *sdp; 1679 UVMHIST_FUNC("uvm_swap_markbad"); UVMHIST_CALLED(pdhist); 1680 1681 mutex_enter(&uvm_swap_data_lock); 1682 sdp = swapdrum_getsdp(startslot); 1683 KASSERT(sdp != NULL); 1684 1685 /* 1686 * we just keep track of how many pages have been marked bad 1687 * in this device, to make everything add up in swap_off(). 1688 * we assume here that the range of slots will all be within 1689 * one swap device. 1690 */ 1691 1692 KASSERT(uvmexp.swpgonly >= nslots); 1693 uvmexp.swpgonly -= nslots; 1694 sdp->swd_npgbad += nslots; 1695 UVMHIST_LOG(pdhist, "now %d bad", sdp->swd_npgbad, 0,0,0); 1696 mutex_exit(&uvm_swap_data_lock); 1697 } 1698 1699 /* 1700 * uvm_swap_free: free swap slots 1701 * 1702 * => this can be all or part of an allocation made by uvm_swap_alloc 1703 * => we lock uvm_swap_data_lock 1704 */ 1705 void 1706 uvm_swap_free(int startslot, int nslots) 1707 { 1708 struct swapdev *sdp; 1709 UVMHIST_FUNC("uvm_swap_free"); UVMHIST_CALLED(pdhist); 1710 1711 UVMHIST_LOG(pdhist, "freeing %d slots starting at %d", nslots, 1712 startslot, 0, 0); 1713 1714 /* 1715 * ignore attempts to free the "bad" slot. 1716 */ 1717 1718 if (startslot == SWSLOT_BAD) { 1719 return; 1720 } 1721 1722 /* 1723 * convert drum slot offset back to sdp, free the blocks 1724 * in the extent, and return. must hold pri lock to do 1725 * lookup and access the extent. 1726 */ 1727 1728 mutex_enter(&uvm_swap_data_lock); 1729 sdp = swapdrum_getsdp(startslot); 1730 KASSERT(uvmexp.nswapdev >= 1); 1731 KASSERT(sdp != NULL); 1732 KASSERT(sdp->swd_npginuse >= nslots); 1733 blist_free(sdp->swd_blist, startslot - sdp->swd_drumoffset, nslots); 1734 sdp->swd_npginuse -= nslots; 1735 uvmexp.swpginuse -= nslots; 1736 mutex_exit(&uvm_swap_data_lock); 1737 } 1738 1739 /* 1740 * uvm_swap_put: put any number of pages into a contig place on swap 1741 * 1742 * => can be sync or async 1743 */ 1744 1745 int 1746 uvm_swap_put(int swslot, struct vm_page **ppsp, int npages, int flags) 1747 { 1748 int error; 1749 1750 error = uvm_swap_io(ppsp, swslot, npages, B_WRITE | 1751 ((flags & PGO_SYNCIO) ? 0 : B_ASYNC)); 1752 return error; 1753 } 1754 1755 /* 1756 * uvm_swap_get: get a single page from swap 1757 * 1758 * => usually a sync op (from fault) 1759 */ 1760 1761 int 1762 uvm_swap_get(struct vm_page *page, int swslot, int flags) 1763 { 1764 int error; 1765 1766 uvmexp.nswget++; 1767 KASSERT(flags & PGO_SYNCIO); 1768 if (swslot == SWSLOT_BAD) { 1769 return EIO; 1770 } 1771 1772 error = uvm_swap_io(&page, swslot, 1, B_READ | 1773 ((flags & PGO_SYNCIO) ? 0 : B_ASYNC)); 1774 if (error == 0) { 1775 1776 /* 1777 * this page is no longer only in swap. 1778 */ 1779 1780 mutex_enter(&uvm_swap_data_lock); 1781 KASSERT(uvmexp.swpgonly > 0); 1782 uvmexp.swpgonly--; 1783 mutex_exit(&uvm_swap_data_lock); 1784 } 1785 return error; 1786 } 1787 1788 /* 1789 * uvm_swap_io: do an i/o operation to swap 1790 */ 1791 1792 static int 1793 uvm_swap_io(struct vm_page **pps, int startslot, int npages, int flags) 1794 { 1795 daddr_t startblk; 1796 struct buf *bp; 1797 vaddr_t kva; 1798 int error, mapinflags; 1799 bool write, async; 1800 UVMHIST_FUNC("uvm_swap_io"); UVMHIST_CALLED(pdhist); 1801 1802 UVMHIST_LOG(pdhist, "<- called, startslot=%d, npages=%d, flags=%d", 1803 startslot, npages, flags, 0); 1804 1805 write = (flags & B_READ) == 0; 1806 async = (flags & B_ASYNC) != 0; 1807 1808 /* 1809 * allocate a buf for the i/o. 1810 */ 1811 1812 KASSERT(curlwp != uvm.pagedaemon_lwp || (write && async)); 1813 bp = getiobuf(swapdev_vp, curlwp != uvm.pagedaemon_lwp); 1814 if (bp == NULL) { 1815 uvm_aio_aiodone_pages(pps, npages, true, ENOMEM); 1816 return ENOMEM; 1817 } 1818 1819 /* 1820 * convert starting drum slot to block number 1821 */ 1822 1823 startblk = btodb((uint64_t)startslot << PAGE_SHIFT); 1824 1825 /* 1826 * first, map the pages into the kernel. 1827 */ 1828 1829 mapinflags = !write ? 1830 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_READ : 1831 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_WRITE; 1832 kva = uvm_pagermapin(pps, npages, mapinflags); 1833 1834 /* 1835 * fill in the bp/sbp. we currently route our i/o through 1836 * /dev/drum's vnode [swapdev_vp]. 1837 */ 1838 1839 bp->b_cflags = BC_BUSY | BC_NOCACHE; 1840 bp->b_flags = (flags & (B_READ|B_ASYNC)); 1841 bp->b_proc = &proc0; /* XXX */ 1842 bp->b_vnbufs.le_next = NOLIST; 1843 bp->b_data = (void *)kva; 1844 bp->b_blkno = startblk; 1845 bp->b_bufsize = bp->b_bcount = npages << PAGE_SHIFT; 1846 1847 /* 1848 * bump v_numoutput (counter of number of active outputs). 1849 */ 1850 1851 if (write) { 1852 mutex_enter(swapdev_vp->v_interlock); 1853 swapdev_vp->v_numoutput++; 1854 mutex_exit(swapdev_vp->v_interlock); 1855 } 1856 1857 /* 1858 * for async ops we must set up the iodone handler. 1859 */ 1860 1861 if (async) { 1862 bp->b_iodone = uvm_aio_biodone; 1863 UVMHIST_LOG(pdhist, "doing async!", 0, 0, 0, 0); 1864 if (curlwp == uvm.pagedaemon_lwp) 1865 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); 1866 else 1867 BIO_SETPRIO(bp, BPRIO_TIMELIMITED); 1868 } else { 1869 bp->b_iodone = NULL; 1870 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); 1871 } 1872 UVMHIST_LOG(pdhist, 1873 "about to start io: data = %p blkno = 0x%x, bcount = %ld", 1874 bp->b_data, bp->b_blkno, bp->b_bcount, 0); 1875 1876 /* 1877 * now we start the I/O, and if async, return. 1878 */ 1879 1880 VOP_STRATEGY(swapdev_vp, bp); 1881 if (async) 1882 return 0; 1883 1884 /* 1885 * must be sync i/o. wait for it to finish 1886 */ 1887 1888 error = biowait(bp); 1889 1890 /* 1891 * kill the pager mapping 1892 */ 1893 1894 uvm_pagermapout(kva, npages); 1895 1896 /* 1897 * now dispose of the buf and we're done. 1898 */ 1899 1900 if (write) { 1901 mutex_enter(swapdev_vp->v_interlock); 1902 vwakeup(bp); 1903 mutex_exit(swapdev_vp->v_interlock); 1904 } 1905 putiobuf(bp); 1906 UVMHIST_LOG(pdhist, "<- done (sync) error=%d", error, 0, 0, 0); 1907 1908 return (error); 1909 } 1910