1 /* $NetBSD: uvm_swap.c,v 1.170 2014/06/28 15:52:45 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 1995, 1996, 1997, 2009 Matthew R. Green 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * from: NetBSD: vm_swap.c,v 1.52 1997/12/02 13:47:37 pk Exp 29 * from: Id: uvm_swap.c,v 1.1.2.42 1998/02/02 20:38:06 chuck Exp 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.170 2014/06/28 15:52:45 maxv Exp $"); 34 35 #include "opt_uvmhist.h" 36 #include "opt_compat_netbsd.h" 37 #include "opt_ddb.h" 38 39 #include <sys/param.h> 40 #include <sys/systm.h> 41 #include <sys/buf.h> 42 #include <sys/bufq.h> 43 #include <sys/conf.h> 44 #include <sys/proc.h> 45 #include <sys/namei.h> 46 #include <sys/disklabel.h> 47 #include <sys/errno.h> 48 #include <sys/kernel.h> 49 #include <sys/vnode.h> 50 #include <sys/file.h> 51 #include <sys/vmem.h> 52 #include <sys/blist.h> 53 #include <sys/mount.h> 54 #include <sys/pool.h> 55 #include <sys/kmem.h> 56 #include <sys/syscallargs.h> 57 #include <sys/swap.h> 58 #include <sys/kauth.h> 59 #include <sys/sysctl.h> 60 #include <sys/workqueue.h> 61 62 #include <uvm/uvm.h> 63 64 #include <miscfs/specfs/specdev.h> 65 66 /* 67 * uvm_swap.c: manage configuration and i/o to swap space. 68 */ 69 70 /* 71 * swap space is managed in the following way: 72 * 73 * each swap partition or file is described by a "swapdev" structure. 74 * each "swapdev" structure contains a "swapent" structure which contains 75 * information that is passed up to the user (via system calls). 76 * 77 * each swap partition is assigned a "priority" (int) which controls 78 * swap parition usage. 79 * 80 * the system maintains a global data structure describing all swap 81 * partitions/files. there is a sorted LIST of "swappri" structures 82 * which describe "swapdev"'s at that priority. this LIST is headed 83 * by the "swap_priority" global var. each "swappri" contains a 84 * TAILQ of "swapdev" structures at that priority. 85 * 86 * locking: 87 * - swap_syscall_lock (krwlock_t): this lock serializes the swapctl 88 * system call and prevents the swap priority list from changing 89 * while we are in the middle of a system call (e.g. SWAP_STATS). 90 * - uvm_swap_data_lock (kmutex_t): this lock protects all swap data 91 * structures including the priority list, the swapdev structures, 92 * and the swapmap arena. 93 * 94 * each swap device has the following info: 95 * - swap device in use (could be disabled, preventing future use) 96 * - swap enabled (allows new allocations on swap) 97 * - map info in /dev/drum 98 * - vnode pointer 99 * for swap files only: 100 * - block size 101 * - max byte count in buffer 102 * - buffer 103 * 104 * userland controls and configures swap with the swapctl(2) system call. 105 * the sys_swapctl performs the following operations: 106 * [1] SWAP_NSWAP: returns the number of swap devices currently configured 107 * [2] SWAP_STATS: given a pointer to an array of swapent structures 108 * (passed in via "arg") of a size passed in via "misc" ... we load 109 * the current swap config into the array. The actual work is done 110 * in the uvm_swap_stats() function. 111 * [3] SWAP_ON: given a pathname in arg (could be device or file) and a 112 * priority in "misc", start swapping on it. 113 * [4] SWAP_OFF: as SWAP_ON, but stops swapping to a device 114 * [5] SWAP_CTL: changes the priority of a swap device (new priority in 115 * "misc") 116 */ 117 118 /* 119 * swapdev: describes a single swap partition/file 120 * 121 * note the following should be true: 122 * swd_inuse <= swd_nblks [number of blocks in use is <= total blocks] 123 * swd_nblks <= swd_mapsize [because mapsize includes miniroot+disklabel] 124 */ 125 struct swapdev { 126 dev_t swd_dev; /* device id */ 127 int swd_flags; /* flags:inuse/enable/fake */ 128 int swd_priority; /* our priority */ 129 int swd_nblks; /* blocks in this device */ 130 char *swd_path; /* saved pathname of device */ 131 int swd_pathlen; /* length of pathname */ 132 int swd_npages; /* #pages we can use */ 133 int swd_npginuse; /* #pages in use */ 134 int swd_npgbad; /* #pages bad */ 135 int swd_drumoffset; /* page0 offset in drum */ 136 int swd_drumsize; /* #pages in drum */ 137 blist_t swd_blist; /* blist for this swapdev */ 138 struct vnode *swd_vp; /* backing vnode */ 139 TAILQ_ENTRY(swapdev) swd_next; /* priority tailq */ 140 141 int swd_bsize; /* blocksize (bytes) */ 142 int swd_maxactive; /* max active i/o reqs */ 143 struct bufq_state *swd_tab; /* buffer list */ 144 int swd_active; /* number of active buffers */ 145 }; 146 147 /* 148 * swap device priority entry; the list is kept sorted on `spi_priority'. 149 */ 150 struct swappri { 151 int spi_priority; /* priority */ 152 TAILQ_HEAD(spi_swapdev, swapdev) spi_swapdev; 153 /* tailq of swapdevs at this priority */ 154 LIST_ENTRY(swappri) spi_swappri; /* global list of pri's */ 155 }; 156 157 /* 158 * The following two structures are used to keep track of data transfers 159 * on swap devices associated with regular files. 160 * NOTE: this code is more or less a copy of vnd.c; we use the same 161 * structure names here to ease porting.. 162 */ 163 struct vndxfer { 164 struct buf *vx_bp; /* Pointer to parent buffer */ 165 struct swapdev *vx_sdp; 166 int vx_error; 167 int vx_pending; /* # of pending aux buffers */ 168 int vx_flags; 169 #define VX_BUSY 1 170 #define VX_DEAD 2 171 }; 172 173 struct vndbuf { 174 struct buf vb_buf; 175 struct vndxfer *vb_xfer; 176 }; 177 178 /* 179 * NetBSD 1.3 swapctl(SWAP_STATS, ...) swapent structure; uses 32 bit 180 * dev_t and has no se_path[] member. 181 */ 182 struct swapent13 { 183 int32_t se13_dev; /* device id */ 184 int se13_flags; /* flags */ 185 int se13_nblks; /* total blocks */ 186 int se13_inuse; /* blocks in use */ 187 int se13_priority; /* priority of this device */ 188 }; 189 190 /* 191 * NetBSD 5.0 swapctl(SWAP_STATS, ...) swapent structure; uses 32 bit 192 * dev_t. 193 */ 194 struct swapent50 { 195 int32_t se50_dev; /* device id */ 196 int se50_flags; /* flags */ 197 int se50_nblks; /* total blocks */ 198 int se50_inuse; /* blocks in use */ 199 int se50_priority; /* priority of this device */ 200 char se50_path[PATH_MAX+1]; /* path name */ 201 }; 202 203 /* 204 * We keep a of pool vndbuf's and vndxfer structures. 205 */ 206 static struct pool vndxfer_pool, vndbuf_pool; 207 208 /* 209 * local variables 210 */ 211 static vmem_t *swapmap; /* controls the mapping of /dev/drum */ 212 213 /* list of all active swap devices [by priority] */ 214 LIST_HEAD(swap_priority, swappri); 215 static struct swap_priority swap_priority; 216 217 /* locks */ 218 static krwlock_t swap_syscall_lock; 219 220 /* workqueue and use counter for swap to regular files */ 221 static int sw_reg_count = 0; 222 static struct workqueue *sw_reg_workqueue; 223 224 /* tuneables */ 225 u_int uvm_swapisfull_factor = 99; 226 227 /* 228 * prototypes 229 */ 230 static struct swapdev *swapdrum_getsdp(int); 231 232 static struct swapdev *swaplist_find(struct vnode *, bool); 233 static void swaplist_insert(struct swapdev *, 234 struct swappri *, int); 235 static void swaplist_trim(void); 236 237 static int swap_on(struct lwp *, struct swapdev *); 238 static int swap_off(struct lwp *, struct swapdev *); 239 240 static void sw_reg_strategy(struct swapdev *, struct buf *, int); 241 static void sw_reg_biodone(struct buf *); 242 static void sw_reg_iodone(struct work *wk, void *dummy); 243 static void sw_reg_start(struct swapdev *); 244 245 static int uvm_swap_io(struct vm_page **, int, int, int); 246 247 /* 248 * uvm_swap_init: init the swap system data structures and locks 249 * 250 * => called at boot time from init_main.c after the filesystems 251 * are brought up (which happens after uvm_init()) 252 */ 253 void 254 uvm_swap_init(void) 255 { 256 UVMHIST_FUNC("uvm_swap_init"); 257 258 UVMHIST_CALLED(pdhist); 259 /* 260 * first, init the swap list, its counter, and its lock. 261 * then get a handle on the vnode for /dev/drum by using 262 * the its dev_t number ("swapdev", from MD conf.c). 263 */ 264 265 LIST_INIT(&swap_priority); 266 uvmexp.nswapdev = 0; 267 rw_init(&swap_syscall_lock); 268 mutex_init(&uvm_swap_data_lock, MUTEX_DEFAULT, IPL_NONE); 269 270 if (bdevvp(swapdev, &swapdev_vp)) 271 panic("%s: can't get vnode for swap device", __func__); 272 if (vn_lock(swapdev_vp, LK_EXCLUSIVE | LK_RETRY)) 273 panic("%s: can't lock swap device", __func__); 274 if (VOP_OPEN(swapdev_vp, FREAD | FWRITE, NOCRED)) 275 panic("%s: can't open swap device", __func__); 276 VOP_UNLOCK(swapdev_vp); 277 278 /* 279 * create swap block resource map to map /dev/drum. the range 280 * from 1 to INT_MAX allows 2 gigablocks of swap space. note 281 * that block 0 is reserved (used to indicate an allocation 282 * failure, or no allocation). 283 */ 284 swapmap = vmem_create("swapmap", 1, INT_MAX - 1, 1, NULL, NULL, NULL, 0, 285 VM_NOSLEEP, IPL_NONE); 286 if (swapmap == 0) { 287 panic("%s: vmem_create failed", __func__); 288 } 289 290 pool_init(&vndxfer_pool, sizeof(struct vndxfer), 0, 0, 0, "swp vnx", 291 NULL, IPL_BIO); 292 pool_init(&vndbuf_pool, sizeof(struct vndbuf), 0, 0, 0, "swp vnd", 293 NULL, IPL_BIO); 294 295 UVMHIST_LOG(pdhist, "<- done", 0, 0, 0, 0); 296 } 297 298 /* 299 * swaplist functions: functions that operate on the list of swap 300 * devices on the system. 301 */ 302 303 /* 304 * swaplist_insert: insert swap device "sdp" into the global list 305 * 306 * => caller must hold both swap_syscall_lock and uvm_swap_data_lock 307 * => caller must provide a newly allocated swappri structure (we will 308 * FREE it if we don't need it... this it to prevent allocation 309 * blocking here while adding swap) 310 */ 311 static void 312 swaplist_insert(struct swapdev *sdp, struct swappri *newspp, int priority) 313 { 314 struct swappri *spp, *pspp; 315 UVMHIST_FUNC("swaplist_insert"); UVMHIST_CALLED(pdhist); 316 317 /* 318 * find entry at or after which to insert the new device. 319 */ 320 pspp = NULL; 321 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 322 if (priority <= spp->spi_priority) 323 break; 324 pspp = spp; 325 } 326 327 /* 328 * new priority? 329 */ 330 if (spp == NULL || spp->spi_priority != priority) { 331 spp = newspp; /* use newspp! */ 332 UVMHIST_LOG(pdhist, "created new swappri = %d", 333 priority, 0, 0, 0); 334 335 spp->spi_priority = priority; 336 TAILQ_INIT(&spp->spi_swapdev); 337 338 if (pspp) 339 LIST_INSERT_AFTER(pspp, spp, spi_swappri); 340 else 341 LIST_INSERT_HEAD(&swap_priority, spp, spi_swappri); 342 } else { 343 /* we don't need a new priority structure, free it */ 344 kmem_free(newspp, sizeof(*newspp)); 345 } 346 347 /* 348 * priority found (or created). now insert on the priority's 349 * tailq list and bump the total number of swapdevs. 350 */ 351 sdp->swd_priority = priority; 352 TAILQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next); 353 uvmexp.nswapdev++; 354 } 355 356 /* 357 * swaplist_find: find and optionally remove a swap device from the 358 * global list. 359 * 360 * => caller must hold both swap_syscall_lock and uvm_swap_data_lock 361 * => we return the swapdev we found (and removed) 362 */ 363 static struct swapdev * 364 swaplist_find(struct vnode *vp, bool remove) 365 { 366 struct swapdev *sdp; 367 struct swappri *spp; 368 369 /* 370 * search the lists for the requested vp 371 */ 372 373 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 374 TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 375 if (sdp->swd_vp == vp) { 376 if (remove) { 377 TAILQ_REMOVE(&spp->spi_swapdev, 378 sdp, swd_next); 379 uvmexp.nswapdev--; 380 } 381 return(sdp); 382 } 383 } 384 } 385 return (NULL); 386 } 387 388 /* 389 * swaplist_trim: scan priority list for empty priority entries and kill 390 * them. 391 * 392 * => caller must hold both swap_syscall_lock and uvm_swap_data_lock 393 */ 394 static void 395 swaplist_trim(void) 396 { 397 struct swappri *spp, *nextspp; 398 399 LIST_FOREACH_SAFE(spp, &swap_priority, spi_swappri, nextspp) { 400 if (!TAILQ_EMPTY(&spp->spi_swapdev)) 401 continue; 402 LIST_REMOVE(spp, spi_swappri); 403 kmem_free(spp, sizeof(*spp)); 404 } 405 } 406 407 /* 408 * swapdrum_getsdp: given a page offset in /dev/drum, convert it back 409 * to the "swapdev" that maps that section of the drum. 410 * 411 * => each swapdev takes one big contig chunk of the drum 412 * => caller must hold uvm_swap_data_lock 413 */ 414 static struct swapdev * 415 swapdrum_getsdp(int pgno) 416 { 417 struct swapdev *sdp; 418 struct swappri *spp; 419 420 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 421 TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 422 if (sdp->swd_flags & SWF_FAKE) 423 continue; 424 if (pgno >= sdp->swd_drumoffset && 425 pgno < (sdp->swd_drumoffset + sdp->swd_drumsize)) { 426 return sdp; 427 } 428 } 429 } 430 return NULL; 431 } 432 433 434 /* 435 * sys_swapctl: main entry point for swapctl(2) system call 436 * [with two helper functions: swap_on and swap_off] 437 */ 438 int 439 sys_swapctl(struct lwp *l, const struct sys_swapctl_args *uap, register_t *retval) 440 { 441 /* { 442 syscallarg(int) cmd; 443 syscallarg(void *) arg; 444 syscallarg(int) misc; 445 } */ 446 struct vnode *vp; 447 struct nameidata nd; 448 struct swappri *spp; 449 struct swapdev *sdp; 450 struct swapent *sep; 451 #define SWAP_PATH_MAX (PATH_MAX + 1) 452 char *userpath; 453 size_t len = 0; 454 int error, misc; 455 int priority; 456 UVMHIST_FUNC("sys_swapctl"); UVMHIST_CALLED(pdhist); 457 458 /* 459 * we handle the non-priv NSWAP and STATS request first. 460 * 461 * SWAP_NSWAP: return number of config'd swap devices 462 * [can also be obtained with uvmexp sysctl] 463 */ 464 if (SCARG(uap, cmd) == SWAP_NSWAP) { 465 const int nswapdev = uvmexp.nswapdev; 466 UVMHIST_LOG(pdhist, "<- done SWAP_NSWAP=%d", nswapdev, 0, 0, 0); 467 *retval = nswapdev; 468 return 0; 469 } 470 471 misc = SCARG(uap, misc); 472 userpath = kmem_alloc(SWAP_PATH_MAX, KM_SLEEP); 473 474 /* 475 * ensure serialized syscall access by grabbing the swap_syscall_lock 476 */ 477 rw_enter(&swap_syscall_lock, RW_WRITER); 478 479 /* 480 * SWAP_STATS: get stats on current # of configured swap devs 481 * 482 * note that the swap_priority list can't change as long 483 * as we are holding the swap_syscall_lock. we don't want 484 * to grab the uvm_swap_data_lock because we may fault&sleep during 485 * copyout() and we don't want to be holding that lock then! 486 */ 487 if (SCARG(uap, cmd) == SWAP_STATS 488 #if defined(COMPAT_50) 489 || SCARG(uap, cmd) == SWAP_STATS50 490 #endif 491 #if defined(COMPAT_13) 492 || SCARG(uap, cmd) == SWAP_STATS13 493 #endif 494 ) { 495 if (misc < 0) { 496 error = EINVAL; 497 goto out; 498 } 499 if (misc == 0 || uvmexp.nswapdev == 0) { 500 error = 0; 501 goto out; 502 } 503 /* Make sure userland cannot exhaust kernel memory */ 504 if ((size_t)misc > (size_t)uvmexp.nswapdev) 505 misc = uvmexp.nswapdev; 506 KASSERT(misc > 0); 507 #if defined(COMPAT_13) 508 if (SCARG(uap, cmd) == SWAP_STATS13) 509 len = sizeof(struct swapent13) * misc; 510 else 511 #endif 512 #if defined(COMPAT_50) 513 if (SCARG(uap, cmd) == SWAP_STATS50) 514 len = sizeof(struct swapent50) * misc; 515 else 516 #endif 517 len = sizeof(struct swapent) * misc; 518 sep = (struct swapent *)kmem_alloc(len, KM_SLEEP); 519 520 uvm_swap_stats(SCARG(uap, cmd), sep, misc, retval); 521 error = copyout(sep, SCARG(uap, arg), len); 522 523 kmem_free(sep, len); 524 UVMHIST_LOG(pdhist, "<- done SWAP_STATS", 0, 0, 0, 0); 525 goto out; 526 } 527 if (SCARG(uap, cmd) == SWAP_GETDUMPDEV) { 528 dev_t *devp = (dev_t *)SCARG(uap, arg); 529 530 error = copyout(&dumpdev, devp, sizeof(dumpdev)); 531 goto out; 532 } 533 534 /* 535 * all other requests require superuser privs. verify. 536 */ 537 if ((error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_SWAPCTL, 538 0, NULL, NULL, NULL))) 539 goto out; 540 541 if (SCARG(uap, cmd) == SWAP_DUMPOFF) { 542 /* drop the current dump device */ 543 dumpdev = NODEV; 544 dumpcdev = NODEV; 545 cpu_dumpconf(); 546 goto out; 547 } 548 549 /* 550 * at this point we expect a path name in arg. we will 551 * use namei() to gain a vnode reference (vref), and lock 552 * the vnode (VOP_LOCK). 553 * 554 * XXX: a NULL arg means use the root vnode pointer (e.g. for 555 * miniroot) 556 */ 557 if (SCARG(uap, arg) == NULL) { 558 vp = rootvp; /* miniroot */ 559 vref(vp); 560 if (vn_lock(vp, LK_EXCLUSIVE)) { 561 vrele(vp); 562 error = EBUSY; 563 goto out; 564 } 565 if (SCARG(uap, cmd) == SWAP_ON && 566 copystr("miniroot", userpath, SWAP_PATH_MAX, &len)) 567 panic("swapctl: miniroot copy failed"); 568 } else { 569 struct pathbuf *pb; 570 571 /* 572 * This used to allow copying in one extra byte 573 * (SWAP_PATH_MAX instead of PATH_MAX) for SWAP_ON. 574 * This was completely pointless because if anyone 575 * used that extra byte namei would fail with 576 * ENAMETOOLONG anyway, so I've removed the excess 577 * logic. - dholland 20100215 578 */ 579 580 error = pathbuf_copyin(SCARG(uap, arg), &pb); 581 if (error) { 582 goto out; 583 } 584 if (SCARG(uap, cmd) == SWAP_ON) { 585 /* get a copy of the string */ 586 pathbuf_copystring(pb, userpath, SWAP_PATH_MAX); 587 len = strlen(userpath) + 1; 588 } 589 NDINIT(&nd, LOOKUP, FOLLOW | LOCKLEAF | TRYEMULROOT, pb); 590 if ((error = namei(&nd))) { 591 pathbuf_destroy(pb); 592 goto out; 593 } 594 vp = nd.ni_vp; 595 pathbuf_destroy(pb); 596 } 597 /* note: "vp" is referenced and locked */ 598 599 error = 0; /* assume no error */ 600 switch(SCARG(uap, cmd)) { 601 602 case SWAP_DUMPDEV: 603 if (vp->v_type != VBLK) { 604 error = ENOTBLK; 605 break; 606 } 607 if (bdevsw_lookup(vp->v_rdev)) { 608 dumpdev = vp->v_rdev; 609 dumpcdev = devsw_blk2chr(dumpdev); 610 } else 611 dumpdev = NODEV; 612 cpu_dumpconf(); 613 break; 614 615 case SWAP_CTL: 616 /* 617 * get new priority, remove old entry (if any) and then 618 * reinsert it in the correct place. finally, prune out 619 * any empty priority structures. 620 */ 621 priority = SCARG(uap, misc); 622 spp = kmem_alloc(sizeof(*spp), KM_SLEEP); 623 mutex_enter(&uvm_swap_data_lock); 624 if ((sdp = swaplist_find(vp, true)) == NULL) { 625 error = ENOENT; 626 } else { 627 swaplist_insert(sdp, spp, priority); 628 swaplist_trim(); 629 } 630 mutex_exit(&uvm_swap_data_lock); 631 if (error) 632 kmem_free(spp, sizeof(*spp)); 633 break; 634 635 case SWAP_ON: 636 637 /* 638 * check for duplicates. if none found, then insert a 639 * dummy entry on the list to prevent someone else from 640 * trying to enable this device while we are working on 641 * it. 642 */ 643 644 priority = SCARG(uap, misc); 645 sdp = kmem_zalloc(sizeof(*sdp), KM_SLEEP); 646 spp = kmem_alloc(sizeof(*spp), KM_SLEEP); 647 sdp->swd_flags = SWF_FAKE; 648 sdp->swd_vp = vp; 649 sdp->swd_dev = (vp->v_type == VBLK) ? vp->v_rdev : NODEV; 650 bufq_alloc(&sdp->swd_tab, "disksort", BUFQ_SORT_RAWBLOCK); 651 mutex_enter(&uvm_swap_data_lock); 652 if (swaplist_find(vp, false) != NULL) { 653 error = EBUSY; 654 mutex_exit(&uvm_swap_data_lock); 655 bufq_free(sdp->swd_tab); 656 kmem_free(sdp, sizeof(*sdp)); 657 kmem_free(spp, sizeof(*spp)); 658 break; 659 } 660 swaplist_insert(sdp, spp, priority); 661 mutex_exit(&uvm_swap_data_lock); 662 663 KASSERT(len > 0); 664 sdp->swd_pathlen = len; 665 sdp->swd_path = kmem_alloc(len, KM_SLEEP); 666 if (copystr(userpath, sdp->swd_path, len, 0) != 0) 667 panic("swapctl: copystr"); 668 669 /* 670 * we've now got a FAKE placeholder in the swap list. 671 * now attempt to enable swap on it. if we fail, undo 672 * what we've done and kill the fake entry we just inserted. 673 * if swap_on is a success, it will clear the SWF_FAKE flag 674 */ 675 676 if ((error = swap_on(l, sdp)) != 0) { 677 mutex_enter(&uvm_swap_data_lock); 678 (void) swaplist_find(vp, true); /* kill fake entry */ 679 swaplist_trim(); 680 mutex_exit(&uvm_swap_data_lock); 681 bufq_free(sdp->swd_tab); 682 kmem_free(sdp->swd_path, sdp->swd_pathlen); 683 kmem_free(sdp, sizeof(*sdp)); 684 break; 685 } 686 break; 687 688 case SWAP_OFF: 689 mutex_enter(&uvm_swap_data_lock); 690 if ((sdp = swaplist_find(vp, false)) == NULL) { 691 mutex_exit(&uvm_swap_data_lock); 692 error = ENXIO; 693 break; 694 } 695 696 /* 697 * If a device isn't in use or enabled, we 698 * can't stop swapping from it (again). 699 */ 700 if ((sdp->swd_flags & (SWF_INUSE|SWF_ENABLE)) == 0) { 701 mutex_exit(&uvm_swap_data_lock); 702 error = EBUSY; 703 break; 704 } 705 706 /* 707 * do the real work. 708 */ 709 error = swap_off(l, sdp); 710 break; 711 712 default: 713 error = EINVAL; 714 } 715 716 /* 717 * done! release the ref gained by namei() and unlock. 718 */ 719 vput(vp); 720 out: 721 rw_exit(&swap_syscall_lock); 722 kmem_free(userpath, SWAP_PATH_MAX); 723 724 UVMHIST_LOG(pdhist, "<- done! error=%d", error, 0, 0, 0); 725 return (error); 726 } 727 728 /* 729 * uvm_swap_stats: implements swapctl(SWAP_STATS). The function is kept 730 * away from sys_swapctl() in order to allow COMPAT_* swapctl() 731 * emulation to use it directly without going through sys_swapctl(). 732 * The problem with using sys_swapctl() there is that it involves 733 * copying the swapent array to the stackgap, and this array's size 734 * is not known at build time. Hence it would not be possible to 735 * ensure it would fit in the stackgap in any case. 736 */ 737 void 738 uvm_swap_stats(int cmd, struct swapent *sep, int sec, register_t *retval) 739 { 740 struct swappri *spp; 741 struct swapdev *sdp; 742 int count = 0; 743 744 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 745 TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 746 int inuse; 747 748 if (sec-- <= 0) 749 break; 750 751 /* 752 * backwards compatibility for system call. 753 * For NetBSD 1.3 and 5.0, we have to use 754 * the 32 bit dev_t. For 5.0 and -current 755 * we have to add the path. 756 */ 757 inuse = btodb((uint64_t)sdp->swd_npginuse << 758 PAGE_SHIFT); 759 760 #if defined(COMPAT_13) || defined(COMPAT_50) 761 if (cmd == SWAP_STATS) { 762 #endif 763 sep->se_dev = sdp->swd_dev; 764 sep->se_flags = sdp->swd_flags; 765 sep->se_nblks = sdp->swd_nblks; 766 sep->se_inuse = inuse; 767 sep->se_priority = sdp->swd_priority; 768 KASSERT(sdp->swd_pathlen < 769 sizeof(sep->se_path)); 770 strcpy(sep->se_path, sdp->swd_path); 771 sep++; 772 #if defined(COMPAT_13) 773 } else if (cmd == SWAP_STATS13) { 774 struct swapent13 *sep13 = 775 (struct swapent13 *)sep; 776 777 sep13->se13_dev = sdp->swd_dev; 778 sep13->se13_flags = sdp->swd_flags; 779 sep13->se13_nblks = sdp->swd_nblks; 780 sep13->se13_inuse = inuse; 781 sep13->se13_priority = sdp->swd_priority; 782 sep = (struct swapent *)(sep13 + 1); 783 #endif 784 #if defined(COMPAT_50) 785 } else if (cmd == SWAP_STATS50) { 786 struct swapent50 *sep50 = 787 (struct swapent50 *)sep; 788 789 sep50->se50_dev = sdp->swd_dev; 790 sep50->se50_flags = sdp->swd_flags; 791 sep50->se50_nblks = sdp->swd_nblks; 792 sep50->se50_inuse = inuse; 793 sep50->se50_priority = sdp->swd_priority; 794 KASSERT(sdp->swd_pathlen < 795 sizeof(sep50->se50_path)); 796 strcpy(sep50->se50_path, sdp->swd_path); 797 sep = (struct swapent *)(sep50 + 1); 798 #endif 799 #if defined(COMPAT_13) || defined(COMPAT_50) 800 } 801 #endif 802 count++; 803 } 804 } 805 *retval = count; 806 } 807 808 /* 809 * swap_on: attempt to enable a swapdev for swapping. note that the 810 * swapdev is already on the global list, but disabled (marked 811 * SWF_FAKE). 812 * 813 * => we avoid the start of the disk (to protect disk labels) 814 * => we also avoid the miniroot, if we are swapping to root. 815 * => caller should leave uvm_swap_data_lock unlocked, we may lock it 816 * if needed. 817 */ 818 static int 819 swap_on(struct lwp *l, struct swapdev *sdp) 820 { 821 struct vnode *vp; 822 int error, npages, nblocks, size; 823 long addr; 824 vmem_addr_t result; 825 struct vattr va; 826 dev_t dev; 827 UVMHIST_FUNC("swap_on"); UVMHIST_CALLED(pdhist); 828 829 /* 830 * we want to enable swapping on sdp. the swd_vp contains 831 * the vnode we want (locked and ref'd), and the swd_dev 832 * contains the dev_t of the file, if it a block device. 833 */ 834 835 vp = sdp->swd_vp; 836 dev = sdp->swd_dev; 837 838 /* 839 * open the swap file (mostly useful for block device files to 840 * let device driver know what is up). 841 * 842 * we skip the open/close for root on swap because the root 843 * has already been opened when root was mounted (mountroot). 844 */ 845 if (vp != rootvp) { 846 if ((error = VOP_OPEN(vp, FREAD|FWRITE, l->l_cred))) 847 return (error); 848 } 849 850 /* XXX this only works for block devices */ 851 UVMHIST_LOG(pdhist, " dev=%d, major(dev)=%d", dev, major(dev), 0,0); 852 853 /* 854 * we now need to determine the size of the swap area. for 855 * block specials we can call the d_psize function. 856 * for normal files, we must stat [get attrs]. 857 * 858 * we put the result in nblks. 859 * for normal files, we also want the filesystem block size 860 * (which we get with statfs). 861 */ 862 switch (vp->v_type) { 863 case VBLK: 864 if ((nblocks = bdev_size(dev)) == -1) { 865 error = ENXIO; 866 goto bad; 867 } 868 break; 869 870 case VREG: 871 if ((error = VOP_GETATTR(vp, &va, l->l_cred))) 872 goto bad; 873 nblocks = (int)btodb(va.va_size); 874 sdp->swd_bsize = 1 << vp->v_mount->mnt_fs_bshift; 875 /* 876 * limit the max # of outstanding I/O requests we issue 877 * at any one time. take it easy on NFS servers. 878 */ 879 if (vp->v_tag == VT_NFS) 880 sdp->swd_maxactive = 2; /* XXX */ 881 else 882 sdp->swd_maxactive = 8; /* XXX */ 883 break; 884 885 default: 886 error = ENXIO; 887 goto bad; 888 } 889 890 /* 891 * save nblocks in a safe place and convert to pages. 892 */ 893 894 sdp->swd_nblks = nblocks; 895 npages = dbtob((uint64_t)nblocks) >> PAGE_SHIFT; 896 897 /* 898 * for block special files, we want to make sure that leave 899 * the disklabel and bootblocks alone, so we arrange to skip 900 * over them (arbitrarily choosing to skip PAGE_SIZE bytes). 901 * note that because of this the "size" can be less than the 902 * actual number of blocks on the device. 903 */ 904 if (vp->v_type == VBLK) { 905 /* we use pages 1 to (size - 1) [inclusive] */ 906 size = npages - 1; 907 addr = 1; 908 } else { 909 /* we use pages 0 to (size - 1) [inclusive] */ 910 size = npages; 911 addr = 0; 912 } 913 914 /* 915 * make sure we have enough blocks for a reasonable sized swap 916 * area. we want at least one page. 917 */ 918 919 if (size < 1) { 920 UVMHIST_LOG(pdhist, " size <= 1!!", 0, 0, 0, 0); 921 error = EINVAL; 922 goto bad; 923 } 924 925 UVMHIST_LOG(pdhist, " dev=%x: size=%d addr=%ld\n", dev, size, addr, 0); 926 927 /* 928 * now we need to allocate an extent to manage this swap device 929 */ 930 931 sdp->swd_blist = blist_create(npages); 932 /* mark all expect the `saved' region free. */ 933 blist_free(sdp->swd_blist, addr, size); 934 935 /* 936 * if the vnode we are swapping to is the root vnode 937 * (i.e. we are swapping to the miniroot) then we want 938 * to make sure we don't overwrite it. do a statfs to 939 * find its size and skip over it. 940 */ 941 if (vp == rootvp) { 942 struct mount *mp; 943 struct statvfs *sp; 944 int rootblocks, rootpages; 945 946 mp = rootvnode->v_mount; 947 sp = &mp->mnt_stat; 948 rootblocks = sp->f_blocks * btodb(sp->f_frsize); 949 /* 950 * XXX: sp->f_blocks isn't the total number of 951 * blocks in the filesystem, it's the number of 952 * data blocks. so, our rootblocks almost 953 * definitely underestimates the total size 954 * of the filesystem - how badly depends on the 955 * details of the filesystem type. there isn't 956 * an obvious way to deal with this cleanly 957 * and perfectly, so for now we just pad our 958 * rootblocks estimate with an extra 5 percent. 959 */ 960 rootblocks += (rootblocks >> 5) + 961 (rootblocks >> 6) + 962 (rootblocks >> 7); 963 rootpages = round_page(dbtob(rootblocks)) >> PAGE_SHIFT; 964 if (rootpages > size) 965 panic("swap_on: miniroot larger than swap?"); 966 967 if (rootpages != blist_fill(sdp->swd_blist, addr, rootpages)) { 968 panic("swap_on: unable to preserve miniroot"); 969 } 970 971 size -= rootpages; 972 printf("Preserved %d pages of miniroot ", rootpages); 973 printf("leaving %d pages of swap\n", size); 974 } 975 976 /* 977 * add a ref to vp to reflect usage as a swap device. 978 */ 979 vref(vp); 980 981 /* 982 * now add the new swapdev to the drum and enable. 983 */ 984 error = vmem_alloc(swapmap, npages, VM_BESTFIT | VM_SLEEP, &result); 985 if (error != 0) 986 panic("swapdrum_add"); 987 /* 988 * If this is the first regular swap create the workqueue. 989 * => Protected by swap_syscall_lock. 990 */ 991 if (vp->v_type != VBLK) { 992 if (sw_reg_count++ == 0) { 993 KASSERT(sw_reg_workqueue == NULL); 994 if (workqueue_create(&sw_reg_workqueue, "swapiod", 995 sw_reg_iodone, NULL, PRIBIO, IPL_BIO, 0) != 0) 996 panic("%s: workqueue_create failed", __func__); 997 } 998 } 999 1000 sdp->swd_drumoffset = (int)result; 1001 sdp->swd_drumsize = npages; 1002 sdp->swd_npages = size; 1003 mutex_enter(&uvm_swap_data_lock); 1004 sdp->swd_flags &= ~SWF_FAKE; /* going live */ 1005 sdp->swd_flags |= (SWF_INUSE|SWF_ENABLE); 1006 uvmexp.swpages += size; 1007 uvmexp.swpgavail += size; 1008 mutex_exit(&uvm_swap_data_lock); 1009 return (0); 1010 1011 /* 1012 * failure: clean up and return error. 1013 */ 1014 1015 bad: 1016 if (sdp->swd_blist) { 1017 blist_destroy(sdp->swd_blist); 1018 } 1019 if (vp != rootvp) { 1020 (void)VOP_CLOSE(vp, FREAD|FWRITE, l->l_cred); 1021 } 1022 return (error); 1023 } 1024 1025 /* 1026 * swap_off: stop swapping on swapdev 1027 * 1028 * => swap data should be locked, we will unlock. 1029 */ 1030 static int 1031 swap_off(struct lwp *l, struct swapdev *sdp) 1032 { 1033 int npages = sdp->swd_npages; 1034 int error = 0; 1035 1036 UVMHIST_FUNC("swap_off"); UVMHIST_CALLED(pdhist); 1037 UVMHIST_LOG(pdhist, " dev=%x, npages=%d", sdp->swd_dev,npages,0,0); 1038 1039 /* disable the swap area being removed */ 1040 sdp->swd_flags &= ~SWF_ENABLE; 1041 uvmexp.swpgavail -= npages; 1042 mutex_exit(&uvm_swap_data_lock); 1043 1044 /* 1045 * the idea is to find all the pages that are paged out to this 1046 * device, and page them all in. in uvm, swap-backed pageable 1047 * memory can take two forms: aobjs and anons. call the 1048 * swapoff hook for each subsystem to bring in pages. 1049 */ 1050 1051 if (uao_swap_off(sdp->swd_drumoffset, 1052 sdp->swd_drumoffset + sdp->swd_drumsize) || 1053 amap_swap_off(sdp->swd_drumoffset, 1054 sdp->swd_drumoffset + sdp->swd_drumsize)) { 1055 error = ENOMEM; 1056 } else if (sdp->swd_npginuse > sdp->swd_npgbad) { 1057 error = EBUSY; 1058 } 1059 1060 if (error) { 1061 mutex_enter(&uvm_swap_data_lock); 1062 sdp->swd_flags |= SWF_ENABLE; 1063 uvmexp.swpgavail += npages; 1064 mutex_exit(&uvm_swap_data_lock); 1065 1066 return error; 1067 } 1068 1069 /* 1070 * If this is the last regular swap destroy the workqueue. 1071 * => Protected by swap_syscall_lock. 1072 */ 1073 if (sdp->swd_vp->v_type != VBLK) { 1074 KASSERT(sw_reg_count > 0); 1075 KASSERT(sw_reg_workqueue != NULL); 1076 if (--sw_reg_count == 0) { 1077 workqueue_destroy(sw_reg_workqueue); 1078 sw_reg_workqueue = NULL; 1079 } 1080 } 1081 1082 /* 1083 * done with the vnode. 1084 * drop our ref on the vnode before calling VOP_CLOSE() 1085 * so that spec_close() can tell if this is the last close. 1086 */ 1087 vrele(sdp->swd_vp); 1088 if (sdp->swd_vp != rootvp) { 1089 (void) VOP_CLOSE(sdp->swd_vp, FREAD|FWRITE, l->l_cred); 1090 } 1091 1092 mutex_enter(&uvm_swap_data_lock); 1093 uvmexp.swpages -= npages; 1094 uvmexp.swpginuse -= sdp->swd_npgbad; 1095 1096 if (swaplist_find(sdp->swd_vp, true) == NULL) 1097 panic("%s: swapdev not in list", __func__); 1098 swaplist_trim(); 1099 mutex_exit(&uvm_swap_data_lock); 1100 1101 /* 1102 * free all resources! 1103 */ 1104 vmem_free(swapmap, sdp->swd_drumoffset, sdp->swd_drumsize); 1105 blist_destroy(sdp->swd_blist); 1106 bufq_free(sdp->swd_tab); 1107 kmem_free(sdp, sizeof(*sdp)); 1108 return (0); 1109 } 1110 1111 void 1112 uvm_swap_shutdown(struct lwp *l) 1113 { 1114 struct swapdev *sdp; 1115 struct swappri *spp; 1116 struct vnode *vp; 1117 int error; 1118 1119 printf("turning of swap..."); 1120 rw_enter(&swap_syscall_lock, RW_WRITER); 1121 mutex_enter(&uvm_swap_data_lock); 1122 again: 1123 LIST_FOREACH(spp, &swap_priority, spi_swappri) 1124 TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 1125 if (sdp->swd_flags & SWF_FAKE) 1126 continue; 1127 if ((sdp->swd_flags & (SWF_INUSE|SWF_ENABLE)) == 0) 1128 continue; 1129 #ifdef DEBUG 1130 printf("\nturning off swap on %s...", 1131 sdp->swd_path); 1132 #endif 1133 if (vn_lock(vp = sdp->swd_vp, LK_EXCLUSIVE)) { 1134 error = EBUSY; 1135 vp = NULL; 1136 } else 1137 error = 0; 1138 if (!error) { 1139 error = swap_off(l, sdp); 1140 mutex_enter(&uvm_swap_data_lock); 1141 } 1142 if (error) { 1143 printf("stopping swap on %s failed " 1144 "with error %d\n", sdp->swd_path, error); 1145 TAILQ_REMOVE(&spp->spi_swapdev, sdp, 1146 swd_next); 1147 uvmexp.nswapdev--; 1148 swaplist_trim(); 1149 if (vp) 1150 vput(vp); 1151 } 1152 goto again; 1153 } 1154 printf(" done\n"); 1155 mutex_exit(&uvm_swap_data_lock); 1156 rw_exit(&swap_syscall_lock); 1157 } 1158 1159 1160 /* 1161 * /dev/drum interface and i/o functions 1162 */ 1163 1164 /* 1165 * swstrategy: perform I/O on the drum 1166 * 1167 * => we must map the i/o request from the drum to the correct swapdev. 1168 */ 1169 static void 1170 swstrategy(struct buf *bp) 1171 { 1172 struct swapdev *sdp; 1173 struct vnode *vp; 1174 int pageno, bn; 1175 UVMHIST_FUNC("swstrategy"); UVMHIST_CALLED(pdhist); 1176 1177 /* 1178 * convert block number to swapdev. note that swapdev can't 1179 * be yanked out from under us because we are holding resources 1180 * in it (i.e. the blocks we are doing I/O on). 1181 */ 1182 pageno = dbtob((int64_t)bp->b_blkno) >> PAGE_SHIFT; 1183 mutex_enter(&uvm_swap_data_lock); 1184 sdp = swapdrum_getsdp(pageno); 1185 mutex_exit(&uvm_swap_data_lock); 1186 if (sdp == NULL) { 1187 bp->b_error = EINVAL; 1188 bp->b_resid = bp->b_bcount; 1189 biodone(bp); 1190 UVMHIST_LOG(pdhist, " failed to get swap device", 0, 0, 0, 0); 1191 return; 1192 } 1193 1194 /* 1195 * convert drum page number to block number on this swapdev. 1196 */ 1197 1198 pageno -= sdp->swd_drumoffset; /* page # on swapdev */ 1199 bn = btodb((uint64_t)pageno << PAGE_SHIFT); /* convert to diskblock */ 1200 1201 UVMHIST_LOG(pdhist, " %s: mapoff=%x bn=%x bcount=%ld", 1202 ((bp->b_flags & B_READ) == 0) ? "write" : "read", 1203 sdp->swd_drumoffset, bn, bp->b_bcount); 1204 1205 /* 1206 * for block devices we finish up here. 1207 * for regular files we have to do more work which we delegate 1208 * to sw_reg_strategy(). 1209 */ 1210 1211 vp = sdp->swd_vp; /* swapdev vnode pointer */ 1212 switch (vp->v_type) { 1213 default: 1214 panic("%s: vnode type 0x%x", __func__, vp->v_type); 1215 1216 case VBLK: 1217 1218 /* 1219 * must convert "bp" from an I/O on /dev/drum to an I/O 1220 * on the swapdev (sdp). 1221 */ 1222 bp->b_blkno = bn; /* swapdev block number */ 1223 bp->b_dev = sdp->swd_dev; /* swapdev dev_t */ 1224 1225 /* 1226 * if we are doing a write, we have to redirect the i/o on 1227 * drum's v_numoutput counter to the swapdevs. 1228 */ 1229 if ((bp->b_flags & B_READ) == 0) { 1230 mutex_enter(bp->b_objlock); 1231 vwakeup(bp); /* kills one 'v_numoutput' on drum */ 1232 mutex_exit(bp->b_objlock); 1233 mutex_enter(vp->v_interlock); 1234 vp->v_numoutput++; /* put it on swapdev */ 1235 mutex_exit(vp->v_interlock); 1236 } 1237 1238 /* 1239 * finally plug in swapdev vnode and start I/O 1240 */ 1241 bp->b_vp = vp; 1242 bp->b_objlock = vp->v_interlock; 1243 VOP_STRATEGY(vp, bp); 1244 return; 1245 1246 case VREG: 1247 /* 1248 * delegate to sw_reg_strategy function. 1249 */ 1250 sw_reg_strategy(sdp, bp, bn); 1251 return; 1252 } 1253 /* NOTREACHED */ 1254 } 1255 1256 /* 1257 * swread: the read function for the drum (just a call to physio) 1258 */ 1259 /*ARGSUSED*/ 1260 static int 1261 swread(dev_t dev, struct uio *uio, int ioflag) 1262 { 1263 UVMHIST_FUNC("swread"); UVMHIST_CALLED(pdhist); 1264 1265 UVMHIST_LOG(pdhist, " dev=%x offset=%qx", dev, uio->uio_offset, 0, 0); 1266 return (physio(swstrategy, NULL, dev, B_READ, minphys, uio)); 1267 } 1268 1269 /* 1270 * swwrite: the write function for the drum (just a call to physio) 1271 */ 1272 /*ARGSUSED*/ 1273 static int 1274 swwrite(dev_t dev, struct uio *uio, int ioflag) 1275 { 1276 UVMHIST_FUNC("swwrite"); UVMHIST_CALLED(pdhist); 1277 1278 UVMHIST_LOG(pdhist, " dev=%x offset=%qx", dev, uio->uio_offset, 0, 0); 1279 return (physio(swstrategy, NULL, dev, B_WRITE, minphys, uio)); 1280 } 1281 1282 const struct bdevsw swap_bdevsw = { 1283 .d_open = nullopen, 1284 .d_close = nullclose, 1285 .d_strategy = swstrategy, 1286 .d_ioctl = noioctl, 1287 .d_dump = nodump, 1288 .d_psize = nosize, 1289 .d_flag = D_OTHER 1290 }; 1291 1292 const struct cdevsw swap_cdevsw = { 1293 .d_open = nullopen, 1294 .d_close = nullclose, 1295 .d_read = swread, 1296 .d_write = swwrite, 1297 .d_ioctl = noioctl, 1298 .d_stop = nostop, 1299 .d_tty = notty, 1300 .d_poll = nopoll, 1301 .d_mmap = nommap, 1302 .d_kqfilter = nokqfilter, 1303 .d_flag = D_OTHER, 1304 }; 1305 1306 /* 1307 * sw_reg_strategy: handle swap i/o to regular files 1308 */ 1309 static void 1310 sw_reg_strategy(struct swapdev *sdp, struct buf *bp, int bn) 1311 { 1312 struct vnode *vp; 1313 struct vndxfer *vnx; 1314 daddr_t nbn; 1315 char *addr; 1316 off_t byteoff; 1317 int s, off, nra, error, sz, resid; 1318 UVMHIST_FUNC("sw_reg_strategy"); UVMHIST_CALLED(pdhist); 1319 1320 /* 1321 * allocate a vndxfer head for this transfer and point it to 1322 * our buffer. 1323 */ 1324 vnx = pool_get(&vndxfer_pool, PR_WAITOK); 1325 vnx->vx_flags = VX_BUSY; 1326 vnx->vx_error = 0; 1327 vnx->vx_pending = 0; 1328 vnx->vx_bp = bp; 1329 vnx->vx_sdp = sdp; 1330 1331 /* 1332 * setup for main loop where we read filesystem blocks into 1333 * our buffer. 1334 */ 1335 error = 0; 1336 bp->b_resid = bp->b_bcount; /* nothing transfered yet! */ 1337 addr = bp->b_data; /* current position in buffer */ 1338 byteoff = dbtob((uint64_t)bn); 1339 1340 for (resid = bp->b_resid; resid; resid -= sz) { 1341 struct vndbuf *nbp; 1342 1343 /* 1344 * translate byteoffset into block number. return values: 1345 * vp = vnode of underlying device 1346 * nbn = new block number (on underlying vnode dev) 1347 * nra = num blocks we can read-ahead (excludes requested 1348 * block) 1349 */ 1350 nra = 0; 1351 error = VOP_BMAP(sdp->swd_vp, byteoff / sdp->swd_bsize, 1352 &vp, &nbn, &nra); 1353 1354 if (error == 0 && nbn == (daddr_t)-1) { 1355 /* 1356 * this used to just set error, but that doesn't 1357 * do the right thing. Instead, it causes random 1358 * memory errors. The panic() should remain until 1359 * this condition doesn't destabilize the system. 1360 */ 1361 #if 1 1362 panic("%s: swap to sparse file", __func__); 1363 #else 1364 error = EIO; /* failure */ 1365 #endif 1366 } 1367 1368 /* 1369 * punt if there was an error or a hole in the file. 1370 * we must wait for any i/o ops we have already started 1371 * to finish before returning. 1372 * 1373 * XXX we could deal with holes here but it would be 1374 * a hassle (in the write case). 1375 */ 1376 if (error) { 1377 s = splbio(); 1378 vnx->vx_error = error; /* pass error up */ 1379 goto out; 1380 } 1381 1382 /* 1383 * compute the size ("sz") of this transfer (in bytes). 1384 */ 1385 off = byteoff % sdp->swd_bsize; 1386 sz = (1 + nra) * sdp->swd_bsize - off; 1387 if (sz > resid) 1388 sz = resid; 1389 1390 UVMHIST_LOG(pdhist, "sw_reg_strategy: " 1391 "vp %p/%p offset 0x%x/0x%x", 1392 sdp->swd_vp, vp, byteoff, nbn); 1393 1394 /* 1395 * now get a buf structure. note that the vb_buf is 1396 * at the front of the nbp structure so that you can 1397 * cast pointers between the two structure easily. 1398 */ 1399 nbp = pool_get(&vndbuf_pool, PR_WAITOK); 1400 buf_init(&nbp->vb_buf); 1401 nbp->vb_buf.b_flags = bp->b_flags; 1402 nbp->vb_buf.b_cflags = bp->b_cflags; 1403 nbp->vb_buf.b_oflags = bp->b_oflags; 1404 nbp->vb_buf.b_bcount = sz; 1405 nbp->vb_buf.b_bufsize = sz; 1406 nbp->vb_buf.b_error = 0; 1407 nbp->vb_buf.b_data = addr; 1408 nbp->vb_buf.b_lblkno = 0; 1409 nbp->vb_buf.b_blkno = nbn + btodb(off); 1410 nbp->vb_buf.b_rawblkno = nbp->vb_buf.b_blkno; 1411 nbp->vb_buf.b_iodone = sw_reg_biodone; 1412 nbp->vb_buf.b_vp = vp; 1413 nbp->vb_buf.b_objlock = vp->v_interlock; 1414 if (vp->v_type == VBLK) { 1415 nbp->vb_buf.b_dev = vp->v_rdev; 1416 } 1417 1418 nbp->vb_xfer = vnx; /* patch it back in to vnx */ 1419 1420 /* 1421 * Just sort by block number 1422 */ 1423 s = splbio(); 1424 if (vnx->vx_error != 0) { 1425 buf_destroy(&nbp->vb_buf); 1426 pool_put(&vndbuf_pool, nbp); 1427 goto out; 1428 } 1429 vnx->vx_pending++; 1430 1431 /* sort it in and start I/O if we are not over our limit */ 1432 /* XXXAD locking */ 1433 bufq_put(sdp->swd_tab, &nbp->vb_buf); 1434 sw_reg_start(sdp); 1435 splx(s); 1436 1437 /* 1438 * advance to the next I/O 1439 */ 1440 byteoff += sz; 1441 addr += sz; 1442 } 1443 1444 s = splbio(); 1445 1446 out: /* Arrive here at splbio */ 1447 vnx->vx_flags &= ~VX_BUSY; 1448 if (vnx->vx_pending == 0) { 1449 error = vnx->vx_error; 1450 pool_put(&vndxfer_pool, vnx); 1451 bp->b_error = error; 1452 biodone(bp); 1453 } 1454 splx(s); 1455 } 1456 1457 /* 1458 * sw_reg_start: start an I/O request on the requested swapdev 1459 * 1460 * => reqs are sorted by b_rawblkno (above) 1461 */ 1462 static void 1463 sw_reg_start(struct swapdev *sdp) 1464 { 1465 struct buf *bp; 1466 struct vnode *vp; 1467 UVMHIST_FUNC("sw_reg_start"); UVMHIST_CALLED(pdhist); 1468 1469 /* recursion control */ 1470 if ((sdp->swd_flags & SWF_BUSY) != 0) 1471 return; 1472 1473 sdp->swd_flags |= SWF_BUSY; 1474 1475 while (sdp->swd_active < sdp->swd_maxactive) { 1476 bp = bufq_get(sdp->swd_tab); 1477 if (bp == NULL) 1478 break; 1479 sdp->swd_active++; 1480 1481 UVMHIST_LOG(pdhist, 1482 "sw_reg_start: bp %p vp %p blkno %p cnt %lx", 1483 bp, bp->b_vp, bp->b_blkno, bp->b_bcount); 1484 vp = bp->b_vp; 1485 KASSERT(bp->b_objlock == vp->v_interlock); 1486 if ((bp->b_flags & B_READ) == 0) { 1487 mutex_enter(vp->v_interlock); 1488 vp->v_numoutput++; 1489 mutex_exit(vp->v_interlock); 1490 } 1491 VOP_STRATEGY(vp, bp); 1492 } 1493 sdp->swd_flags &= ~SWF_BUSY; 1494 } 1495 1496 /* 1497 * sw_reg_biodone: one of our i/o's has completed 1498 */ 1499 static void 1500 sw_reg_biodone(struct buf *bp) 1501 { 1502 workqueue_enqueue(sw_reg_workqueue, &bp->b_work, NULL); 1503 } 1504 1505 /* 1506 * sw_reg_iodone: one of our i/o's has completed and needs post-i/o cleanup 1507 * 1508 * => note that we can recover the vndbuf struct by casting the buf ptr 1509 */ 1510 static void 1511 sw_reg_iodone(struct work *wk, void *dummy) 1512 { 1513 struct vndbuf *vbp = (void *)wk; 1514 struct vndxfer *vnx = vbp->vb_xfer; 1515 struct buf *pbp = vnx->vx_bp; /* parent buffer */ 1516 struct swapdev *sdp = vnx->vx_sdp; 1517 int s, resid, error; 1518 KASSERT(&vbp->vb_buf.b_work == wk); 1519 UVMHIST_FUNC("sw_reg_iodone"); UVMHIST_CALLED(pdhist); 1520 1521 UVMHIST_LOG(pdhist, " vbp=%p vp=%p blkno=%x addr=%p", 1522 vbp, vbp->vb_buf.b_vp, vbp->vb_buf.b_blkno, vbp->vb_buf.b_data); 1523 UVMHIST_LOG(pdhist, " cnt=%lx resid=%lx", 1524 vbp->vb_buf.b_bcount, vbp->vb_buf.b_resid, 0, 0); 1525 1526 /* 1527 * protect vbp at splbio and update. 1528 */ 1529 1530 s = splbio(); 1531 resid = vbp->vb_buf.b_bcount - vbp->vb_buf.b_resid; 1532 pbp->b_resid -= resid; 1533 vnx->vx_pending--; 1534 1535 if (vbp->vb_buf.b_error != 0) { 1536 /* pass error upward */ 1537 error = vbp->vb_buf.b_error ? vbp->vb_buf.b_error : EIO; 1538 UVMHIST_LOG(pdhist, " got error=%d !", error, 0, 0, 0); 1539 vnx->vx_error = error; 1540 } 1541 1542 /* 1543 * kill vbp structure 1544 */ 1545 buf_destroy(&vbp->vb_buf); 1546 pool_put(&vndbuf_pool, vbp); 1547 1548 /* 1549 * wrap up this transaction if it has run to completion or, in 1550 * case of an error, when all auxiliary buffers have returned. 1551 */ 1552 if (vnx->vx_error != 0) { 1553 /* pass error upward */ 1554 error = vnx->vx_error; 1555 if ((vnx->vx_flags & VX_BUSY) == 0 && vnx->vx_pending == 0) { 1556 pbp->b_error = error; 1557 biodone(pbp); 1558 pool_put(&vndxfer_pool, vnx); 1559 } 1560 } else if (pbp->b_resid == 0) { 1561 KASSERT(vnx->vx_pending == 0); 1562 if ((vnx->vx_flags & VX_BUSY) == 0) { 1563 UVMHIST_LOG(pdhist, " iodone error=%d !", 1564 pbp, vnx->vx_error, 0, 0); 1565 biodone(pbp); 1566 pool_put(&vndxfer_pool, vnx); 1567 } 1568 } 1569 1570 /* 1571 * done! start next swapdev I/O if one is pending 1572 */ 1573 sdp->swd_active--; 1574 sw_reg_start(sdp); 1575 splx(s); 1576 } 1577 1578 1579 /* 1580 * uvm_swap_alloc: allocate space on swap 1581 * 1582 * => allocation is done "round robin" down the priority list, as we 1583 * allocate in a priority we "rotate" the circle queue. 1584 * => space can be freed with uvm_swap_free 1585 * => we return the page slot number in /dev/drum (0 == invalid slot) 1586 * => we lock uvm_swap_data_lock 1587 * => XXXMRG: "LESSOK" INTERFACE NEEDED TO EXTENT SYSTEM 1588 */ 1589 int 1590 uvm_swap_alloc(int *nslots /* IN/OUT */, bool lessok) 1591 { 1592 struct swapdev *sdp; 1593 struct swappri *spp; 1594 UVMHIST_FUNC("uvm_swap_alloc"); UVMHIST_CALLED(pdhist); 1595 1596 /* 1597 * no swap devices configured yet? definite failure. 1598 */ 1599 if (uvmexp.nswapdev < 1) 1600 return 0; 1601 1602 /* 1603 * XXXJAK: BEGIN HACK 1604 * 1605 * blist_alloc() in subr_blist.c will panic if we try to allocate 1606 * too many slots. 1607 */ 1608 if (*nslots > BLIST_MAX_ALLOC) { 1609 if (__predict_false(lessok == false)) 1610 return 0; 1611 *nslots = BLIST_MAX_ALLOC; 1612 } 1613 /* XXXJAK: END HACK */ 1614 1615 /* 1616 * lock data lock, convert slots into blocks, and enter loop 1617 */ 1618 mutex_enter(&uvm_swap_data_lock); 1619 1620 ReTry: /* XXXMRG */ 1621 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 1622 TAILQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 1623 uint64_t result; 1624 1625 /* if it's not enabled, then we can't swap from it */ 1626 if ((sdp->swd_flags & SWF_ENABLE) == 0) 1627 continue; 1628 if (sdp->swd_npginuse + *nslots > sdp->swd_npages) 1629 continue; 1630 result = blist_alloc(sdp->swd_blist, *nslots); 1631 if (result == BLIST_NONE) { 1632 continue; 1633 } 1634 KASSERT(result < sdp->swd_drumsize); 1635 1636 /* 1637 * successful allocation! now rotate the tailq. 1638 */ 1639 TAILQ_REMOVE(&spp->spi_swapdev, sdp, swd_next); 1640 TAILQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next); 1641 sdp->swd_npginuse += *nslots; 1642 uvmexp.swpginuse += *nslots; 1643 mutex_exit(&uvm_swap_data_lock); 1644 /* done! return drum slot number */ 1645 UVMHIST_LOG(pdhist, 1646 "success! returning %d slots starting at %d", 1647 *nslots, result + sdp->swd_drumoffset, 0, 0); 1648 return (result + sdp->swd_drumoffset); 1649 } 1650 } 1651 1652 /* XXXMRG: BEGIN HACK */ 1653 if (*nslots > 1 && lessok) { 1654 *nslots = 1; 1655 /* XXXMRG: ugh! blist should support this for us */ 1656 goto ReTry; 1657 } 1658 /* XXXMRG: END HACK */ 1659 1660 mutex_exit(&uvm_swap_data_lock); 1661 return 0; 1662 } 1663 1664 /* 1665 * uvm_swapisfull: return true if most of available swap is allocated 1666 * and in use. we don't count some small portion as it may be inaccessible 1667 * to us at any given moment, for example if there is lock contention or if 1668 * pages are busy. 1669 */ 1670 bool 1671 uvm_swapisfull(void) 1672 { 1673 int swpgonly; 1674 bool rv; 1675 1676 mutex_enter(&uvm_swap_data_lock); 1677 KASSERT(uvmexp.swpgonly <= uvmexp.swpages); 1678 swpgonly = (int)((uint64_t)uvmexp.swpgonly * 100 / 1679 uvm_swapisfull_factor); 1680 rv = (swpgonly >= uvmexp.swpgavail); 1681 mutex_exit(&uvm_swap_data_lock); 1682 1683 return (rv); 1684 } 1685 1686 /* 1687 * uvm_swap_markbad: keep track of swap ranges where we've had i/o errors 1688 * 1689 * => we lock uvm_swap_data_lock 1690 */ 1691 void 1692 uvm_swap_markbad(int startslot, int nslots) 1693 { 1694 struct swapdev *sdp; 1695 UVMHIST_FUNC("uvm_swap_markbad"); UVMHIST_CALLED(pdhist); 1696 1697 mutex_enter(&uvm_swap_data_lock); 1698 sdp = swapdrum_getsdp(startslot); 1699 KASSERT(sdp != NULL); 1700 1701 /* 1702 * we just keep track of how many pages have been marked bad 1703 * in this device, to make everything add up in swap_off(). 1704 * we assume here that the range of slots will all be within 1705 * one swap device. 1706 */ 1707 1708 KASSERT(uvmexp.swpgonly >= nslots); 1709 uvmexp.swpgonly -= nslots; 1710 sdp->swd_npgbad += nslots; 1711 UVMHIST_LOG(pdhist, "now %d bad", sdp->swd_npgbad, 0,0,0); 1712 mutex_exit(&uvm_swap_data_lock); 1713 } 1714 1715 /* 1716 * uvm_swap_free: free swap slots 1717 * 1718 * => this can be all or part of an allocation made by uvm_swap_alloc 1719 * => we lock uvm_swap_data_lock 1720 */ 1721 void 1722 uvm_swap_free(int startslot, int nslots) 1723 { 1724 struct swapdev *sdp; 1725 UVMHIST_FUNC("uvm_swap_free"); UVMHIST_CALLED(pdhist); 1726 1727 UVMHIST_LOG(pdhist, "freeing %d slots starting at %d", nslots, 1728 startslot, 0, 0); 1729 1730 /* 1731 * ignore attempts to free the "bad" slot. 1732 */ 1733 1734 if (startslot == SWSLOT_BAD) { 1735 return; 1736 } 1737 1738 /* 1739 * convert drum slot offset back to sdp, free the blocks 1740 * in the extent, and return. must hold pri lock to do 1741 * lookup and access the extent. 1742 */ 1743 1744 mutex_enter(&uvm_swap_data_lock); 1745 sdp = swapdrum_getsdp(startslot); 1746 KASSERT(uvmexp.nswapdev >= 1); 1747 KASSERT(sdp != NULL); 1748 KASSERT(sdp->swd_npginuse >= nslots); 1749 blist_free(sdp->swd_blist, startslot - sdp->swd_drumoffset, nslots); 1750 sdp->swd_npginuse -= nslots; 1751 uvmexp.swpginuse -= nslots; 1752 mutex_exit(&uvm_swap_data_lock); 1753 } 1754 1755 /* 1756 * uvm_swap_put: put any number of pages into a contig place on swap 1757 * 1758 * => can be sync or async 1759 */ 1760 1761 int 1762 uvm_swap_put(int swslot, struct vm_page **ppsp, int npages, int flags) 1763 { 1764 int error; 1765 1766 error = uvm_swap_io(ppsp, swslot, npages, B_WRITE | 1767 ((flags & PGO_SYNCIO) ? 0 : B_ASYNC)); 1768 return error; 1769 } 1770 1771 /* 1772 * uvm_swap_get: get a single page from swap 1773 * 1774 * => usually a sync op (from fault) 1775 */ 1776 1777 int 1778 uvm_swap_get(struct vm_page *page, int swslot, int flags) 1779 { 1780 int error; 1781 1782 uvmexp.nswget++; 1783 KASSERT(flags & PGO_SYNCIO); 1784 if (swslot == SWSLOT_BAD) { 1785 return EIO; 1786 } 1787 1788 error = uvm_swap_io(&page, swslot, 1, B_READ | 1789 ((flags & PGO_SYNCIO) ? 0 : B_ASYNC)); 1790 if (error == 0) { 1791 1792 /* 1793 * this page is no longer only in swap. 1794 */ 1795 1796 mutex_enter(&uvm_swap_data_lock); 1797 KASSERT(uvmexp.swpgonly > 0); 1798 uvmexp.swpgonly--; 1799 mutex_exit(&uvm_swap_data_lock); 1800 } 1801 return error; 1802 } 1803 1804 /* 1805 * uvm_swap_io: do an i/o operation to swap 1806 */ 1807 1808 static int 1809 uvm_swap_io(struct vm_page **pps, int startslot, int npages, int flags) 1810 { 1811 daddr_t startblk; 1812 struct buf *bp; 1813 vaddr_t kva; 1814 int error, mapinflags; 1815 bool write, async; 1816 UVMHIST_FUNC("uvm_swap_io"); UVMHIST_CALLED(pdhist); 1817 1818 UVMHIST_LOG(pdhist, "<- called, startslot=%d, npages=%d, flags=%d", 1819 startslot, npages, flags, 0); 1820 1821 write = (flags & B_READ) == 0; 1822 async = (flags & B_ASYNC) != 0; 1823 1824 /* 1825 * allocate a buf for the i/o. 1826 */ 1827 1828 KASSERT(curlwp != uvm.pagedaemon_lwp || (write && async)); 1829 bp = getiobuf(swapdev_vp, curlwp != uvm.pagedaemon_lwp); 1830 if (bp == NULL) { 1831 uvm_aio_aiodone_pages(pps, npages, true, ENOMEM); 1832 return ENOMEM; 1833 } 1834 1835 /* 1836 * convert starting drum slot to block number 1837 */ 1838 1839 startblk = btodb((uint64_t)startslot << PAGE_SHIFT); 1840 1841 /* 1842 * first, map the pages into the kernel. 1843 */ 1844 1845 mapinflags = !write ? 1846 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_READ : 1847 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_WRITE; 1848 kva = uvm_pagermapin(pps, npages, mapinflags); 1849 1850 /* 1851 * fill in the bp/sbp. we currently route our i/o through 1852 * /dev/drum's vnode [swapdev_vp]. 1853 */ 1854 1855 bp->b_cflags = BC_BUSY | BC_NOCACHE; 1856 bp->b_flags = (flags & (B_READ|B_ASYNC)); 1857 bp->b_proc = &proc0; /* XXX */ 1858 bp->b_vnbufs.le_next = NOLIST; 1859 bp->b_data = (void *)kva; 1860 bp->b_blkno = startblk; 1861 bp->b_bufsize = bp->b_bcount = npages << PAGE_SHIFT; 1862 1863 /* 1864 * bump v_numoutput (counter of number of active outputs). 1865 */ 1866 1867 if (write) { 1868 mutex_enter(swapdev_vp->v_interlock); 1869 swapdev_vp->v_numoutput++; 1870 mutex_exit(swapdev_vp->v_interlock); 1871 } 1872 1873 /* 1874 * for async ops we must set up the iodone handler. 1875 */ 1876 1877 if (async) { 1878 bp->b_iodone = uvm_aio_biodone; 1879 UVMHIST_LOG(pdhist, "doing async!", 0, 0, 0, 0); 1880 if (curlwp == uvm.pagedaemon_lwp) 1881 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); 1882 else 1883 BIO_SETPRIO(bp, BPRIO_TIMELIMITED); 1884 } else { 1885 bp->b_iodone = NULL; 1886 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); 1887 } 1888 UVMHIST_LOG(pdhist, 1889 "about to start io: data = %p blkno = 0x%x, bcount = %ld", 1890 bp->b_data, bp->b_blkno, bp->b_bcount, 0); 1891 1892 /* 1893 * now we start the I/O, and if async, return. 1894 */ 1895 1896 VOP_STRATEGY(swapdev_vp, bp); 1897 if (async) 1898 return 0; 1899 1900 /* 1901 * must be sync i/o. wait for it to finish 1902 */ 1903 1904 error = biowait(bp); 1905 1906 /* 1907 * kill the pager mapping 1908 */ 1909 1910 uvm_pagermapout(kva, npages); 1911 1912 /* 1913 * now dispose of the buf and we're done. 1914 */ 1915 1916 if (write) { 1917 mutex_enter(swapdev_vp->v_interlock); 1918 vwakeup(bp); 1919 mutex_exit(swapdev_vp->v_interlock); 1920 } 1921 putiobuf(bp); 1922 UVMHIST_LOG(pdhist, "<- done (sync) error=%d", error, 0, 0, 0); 1923 1924 return (error); 1925 } 1926