1 /* $NetBSD: uvm_swap.c,v 1.57 2001/11/10 07:37:01 lukem Exp $ */ 2 3 /* 4 * Copyright (c) 1995, 1996, 1997 Matthew R. Green 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of the author may not be used to endorse or promote products 16 * derived from this software without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 * from: NetBSD: vm_swap.c,v 1.52 1997/12/02 13:47:37 pk Exp 31 * from: Id: uvm_swap.c,v 1.1.2.42 1998/02/02 20:38:06 chuck Exp 32 */ 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.57 2001/11/10 07:37:01 lukem Exp $"); 36 37 #include "fs_nfs.h" 38 #include "opt_uvmhist.h" 39 #include "opt_compat_netbsd.h" 40 #include "opt_ddb.h" 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/buf.h> 45 #include <sys/conf.h> 46 #include <sys/proc.h> 47 #include <sys/namei.h> 48 #include <sys/disklabel.h> 49 #include <sys/errno.h> 50 #include <sys/kernel.h> 51 #include <sys/malloc.h> 52 #include <sys/vnode.h> 53 #include <sys/file.h> 54 #include <sys/extent.h> 55 #include <sys/mount.h> 56 #include <sys/pool.h> 57 #include <sys/syscallargs.h> 58 #include <sys/swap.h> 59 60 #include <uvm/uvm.h> 61 62 #include <miscfs/specfs/specdev.h> 63 64 /* 65 * uvm_swap.c: manage configuration and i/o to swap space. 66 */ 67 68 /* 69 * swap space is managed in the following way: 70 * 71 * each swap partition or file is described by a "swapdev" structure. 72 * each "swapdev" structure contains a "swapent" structure which contains 73 * information that is passed up to the user (via system calls). 74 * 75 * each swap partition is assigned a "priority" (int) which controls 76 * swap parition usage. 77 * 78 * the system maintains a global data structure describing all swap 79 * partitions/files. there is a sorted LIST of "swappri" structures 80 * which describe "swapdev"'s at that priority. this LIST is headed 81 * by the "swap_priority" global var. each "swappri" contains a 82 * CIRCLEQ of "swapdev" structures at that priority. 83 * 84 * locking: 85 * - swap_syscall_lock (sleep lock): this lock serializes the swapctl 86 * system call and prevents the swap priority list from changing 87 * while we are in the middle of a system call (e.g. SWAP_STATS). 88 * - uvm.swap_data_lock (simple_lock): this lock protects all swap data 89 * structures including the priority list, the swapdev structures, 90 * and the swapmap extent. 91 * 92 * each swap device has the following info: 93 * - swap device in use (could be disabled, preventing future use) 94 * - swap enabled (allows new allocations on swap) 95 * - map info in /dev/drum 96 * - vnode pointer 97 * for swap files only: 98 * - block size 99 * - max byte count in buffer 100 * - buffer 101 * - credentials to use when doing i/o to file 102 * 103 * userland controls and configures swap with the swapctl(2) system call. 104 * the sys_swapctl performs the following operations: 105 * [1] SWAP_NSWAP: returns the number of swap devices currently configured 106 * [2] SWAP_STATS: given a pointer to an array of swapent structures 107 * (passed in via "arg") of a size passed in via "misc" ... we load 108 * the current swap config into the array. 109 * [3] SWAP_ON: given a pathname in arg (could be device or file) and a 110 * priority in "misc", start swapping on it. 111 * [4] SWAP_OFF: as SWAP_ON, but stops swapping to a device 112 * [5] SWAP_CTL: changes the priority of a swap device (new priority in 113 * "misc") 114 */ 115 116 /* 117 * swapdev: describes a single swap partition/file 118 * 119 * note the following should be true: 120 * swd_inuse <= swd_nblks [number of blocks in use is <= total blocks] 121 * swd_nblks <= swd_mapsize [because mapsize includes miniroot+disklabel] 122 */ 123 struct swapdev { 124 struct oswapent swd_ose; 125 #define swd_dev swd_ose.ose_dev /* device id */ 126 #define swd_flags swd_ose.ose_flags /* flags:inuse/enable/fake */ 127 #define swd_priority swd_ose.ose_priority /* our priority */ 128 /* also: swd_ose.ose_nblks, swd_ose.ose_inuse */ 129 char *swd_path; /* saved pathname of device */ 130 int swd_pathlen; /* length of pathname */ 131 int swd_npages; /* #pages we can use */ 132 int swd_npginuse; /* #pages in use */ 133 int swd_npgbad; /* #pages bad */ 134 int swd_drumoffset; /* page0 offset in drum */ 135 int swd_drumsize; /* #pages in drum */ 136 struct extent *swd_ex; /* extent for this swapdev */ 137 char swd_exname[12]; /* name of extent above */ 138 struct vnode *swd_vp; /* backing vnode */ 139 CIRCLEQ_ENTRY(swapdev) swd_next; /* priority circleq */ 140 141 int swd_bsize; /* blocksize (bytes) */ 142 int swd_maxactive; /* max active i/o reqs */ 143 struct buf_queue swd_tab; /* buffer list */ 144 int swd_active; /* number of active buffers */ 145 struct ucred *swd_cred; /* cred for file access */ 146 }; 147 148 /* 149 * swap device priority entry; the list is kept sorted on `spi_priority'. 150 */ 151 struct swappri { 152 int spi_priority; /* priority */ 153 CIRCLEQ_HEAD(spi_swapdev, swapdev) spi_swapdev; 154 /* circleq of swapdevs at this priority */ 155 LIST_ENTRY(swappri) spi_swappri; /* global list of pri's */ 156 }; 157 158 /* 159 * The following two structures are used to keep track of data transfers 160 * on swap devices associated with regular files. 161 * NOTE: this code is more or less a copy of vnd.c; we use the same 162 * structure names here to ease porting.. 163 */ 164 struct vndxfer { 165 struct buf *vx_bp; /* Pointer to parent buffer */ 166 struct swapdev *vx_sdp; 167 int vx_error; 168 int vx_pending; /* # of pending aux buffers */ 169 int vx_flags; 170 #define VX_BUSY 1 171 #define VX_DEAD 2 172 }; 173 174 struct vndbuf { 175 struct buf vb_buf; 176 struct vndxfer *vb_xfer; 177 }; 178 179 180 /* 181 * We keep a of pool vndbuf's and vndxfer structures. 182 */ 183 static struct pool vndxfer_pool; 184 static struct pool vndbuf_pool; 185 186 #define getvndxfer(vnx) do { \ 187 int s = splbio(); \ 188 vnx = pool_get(&vndxfer_pool, PR_MALLOCOK|PR_WAITOK); \ 189 splx(s); \ 190 } while (0) 191 192 #define putvndxfer(vnx) { \ 193 pool_put(&vndxfer_pool, (void *)(vnx)); \ 194 } 195 196 #define getvndbuf(vbp) do { \ 197 int s = splbio(); \ 198 vbp = pool_get(&vndbuf_pool, PR_MALLOCOK|PR_WAITOK); \ 199 splx(s); \ 200 } while (0) 201 202 #define putvndbuf(vbp) { \ 203 pool_put(&vndbuf_pool, (void *)(vbp)); \ 204 } 205 206 /* /dev/drum */ 207 bdev_decl(sw); 208 cdev_decl(sw); 209 210 /* 211 * local variables 212 */ 213 static struct extent *swapmap; /* controls the mapping of /dev/drum */ 214 215 /* list of all active swap devices [by priority] */ 216 LIST_HEAD(swap_priority, swappri); 217 static struct swap_priority swap_priority; 218 219 /* locks */ 220 struct lock swap_syscall_lock; 221 222 /* 223 * prototypes 224 */ 225 static struct swapdev *swapdrum_getsdp __P((int)); 226 227 static struct swapdev *swaplist_find __P((struct vnode *, int)); 228 static void swaplist_insert __P((struct swapdev *, 229 struct swappri *, int)); 230 static void swaplist_trim __P((void)); 231 232 static int swap_on __P((struct proc *, struct swapdev *)); 233 static int swap_off __P((struct proc *, struct swapdev *)); 234 235 static void sw_reg_strategy __P((struct swapdev *, struct buf *, int)); 236 static void sw_reg_iodone __P((struct buf *)); 237 static void sw_reg_start __P((struct swapdev *)); 238 239 static int uvm_swap_io __P((struct vm_page **, int, int, int)); 240 241 /* 242 * uvm_swap_init: init the swap system data structures and locks 243 * 244 * => called at boot time from init_main.c after the filesystems 245 * are brought up (which happens after uvm_init()) 246 */ 247 void 248 uvm_swap_init() 249 { 250 UVMHIST_FUNC("uvm_swap_init"); 251 252 UVMHIST_CALLED(pdhist); 253 /* 254 * first, init the swap list, its counter, and its lock. 255 * then get a handle on the vnode for /dev/drum by using 256 * the its dev_t number ("swapdev", from MD conf.c). 257 */ 258 259 LIST_INIT(&swap_priority); 260 uvmexp.nswapdev = 0; 261 lockinit(&swap_syscall_lock, PVM, "swapsys", 0, 0); 262 simple_lock_init(&uvm.swap_data_lock); 263 264 if (bdevvp(swapdev, &swapdev_vp)) 265 panic("uvm_swap_init: can't get vnode for swap device"); 266 267 /* 268 * create swap block resource map to map /dev/drum. the range 269 * from 1 to INT_MAX allows 2 gigablocks of swap space. note 270 * that block 0 is reserved (used to indicate an allocation 271 * failure, or no allocation). 272 */ 273 swapmap = extent_create("swapmap", 1, INT_MAX, 274 M_VMSWAP, 0, 0, EX_NOWAIT); 275 if (swapmap == 0) 276 panic("uvm_swap_init: extent_create failed"); 277 278 /* 279 * allocate pools for structures used for swapping to files. 280 */ 281 282 pool_init(&vndxfer_pool, sizeof(struct vndxfer), 0, 0, 0, 283 "swp vnx", 0, NULL, NULL, 0); 284 285 pool_init(&vndbuf_pool, sizeof(struct vndbuf), 0, 0, 0, 286 "swp vnd", 0, NULL, NULL, 0); 287 288 /* 289 * done! 290 */ 291 UVMHIST_LOG(pdhist, "<- done", 0, 0, 0, 0); 292 } 293 294 /* 295 * swaplist functions: functions that operate on the list of swap 296 * devices on the system. 297 */ 298 299 /* 300 * swaplist_insert: insert swap device "sdp" into the global list 301 * 302 * => caller must hold both swap_syscall_lock and uvm.swap_data_lock 303 * => caller must provide a newly malloc'd swappri structure (we will 304 * FREE it if we don't need it... this it to prevent malloc blocking 305 * here while adding swap) 306 */ 307 static void 308 swaplist_insert(sdp, newspp, priority) 309 struct swapdev *sdp; 310 struct swappri *newspp; 311 int priority; 312 { 313 struct swappri *spp, *pspp; 314 UVMHIST_FUNC("swaplist_insert"); UVMHIST_CALLED(pdhist); 315 316 /* 317 * find entry at or after which to insert the new device. 318 */ 319 pspp = NULL; 320 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 321 if (priority <= spp->spi_priority) 322 break; 323 pspp = spp; 324 } 325 326 /* 327 * new priority? 328 */ 329 if (spp == NULL || spp->spi_priority != priority) { 330 spp = newspp; /* use newspp! */ 331 UVMHIST_LOG(pdhist, "created new swappri = %d", 332 priority, 0, 0, 0); 333 334 spp->spi_priority = priority; 335 CIRCLEQ_INIT(&spp->spi_swapdev); 336 337 if (pspp) 338 LIST_INSERT_AFTER(pspp, spp, spi_swappri); 339 else 340 LIST_INSERT_HEAD(&swap_priority, spp, spi_swappri); 341 } else { 342 /* we don't need a new priority structure, free it */ 343 FREE(newspp, M_VMSWAP); 344 } 345 346 /* 347 * priority found (or created). now insert on the priority's 348 * circleq list and bump the total number of swapdevs. 349 */ 350 sdp->swd_priority = priority; 351 CIRCLEQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next); 352 uvmexp.nswapdev++; 353 } 354 355 /* 356 * swaplist_find: find and optionally remove a swap device from the 357 * global list. 358 * 359 * => caller must hold both swap_syscall_lock and uvm.swap_data_lock 360 * => we return the swapdev we found (and removed) 361 */ 362 static struct swapdev * 363 swaplist_find(vp, remove) 364 struct vnode *vp; 365 boolean_t remove; 366 { 367 struct swapdev *sdp; 368 struct swappri *spp; 369 370 /* 371 * search the lists for the requested vp 372 */ 373 374 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 375 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 376 if (sdp->swd_vp == vp) { 377 if (remove) { 378 CIRCLEQ_REMOVE(&spp->spi_swapdev, 379 sdp, swd_next); 380 uvmexp.nswapdev--; 381 } 382 return(sdp); 383 } 384 } 385 } 386 return (NULL); 387 } 388 389 390 /* 391 * swaplist_trim: scan priority list for empty priority entries and kill 392 * them. 393 * 394 * => caller must hold both swap_syscall_lock and uvm.swap_data_lock 395 */ 396 static void 397 swaplist_trim() 398 { 399 struct swappri *spp, *nextspp; 400 401 for (spp = LIST_FIRST(&swap_priority); spp != NULL; spp = nextspp) { 402 nextspp = LIST_NEXT(spp, spi_swappri); 403 if (CIRCLEQ_FIRST(&spp->spi_swapdev) != 404 (void *)&spp->spi_swapdev) 405 continue; 406 LIST_REMOVE(spp, spi_swappri); 407 free(spp, M_VMSWAP); 408 } 409 } 410 411 /* 412 * swapdrum_getsdp: given a page offset in /dev/drum, convert it back 413 * to the "swapdev" that maps that section of the drum. 414 * 415 * => each swapdev takes one big contig chunk of the drum 416 * => caller must hold uvm.swap_data_lock 417 */ 418 static struct swapdev * 419 swapdrum_getsdp(pgno) 420 int pgno; 421 { 422 struct swapdev *sdp; 423 struct swappri *spp; 424 425 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 426 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 427 if (sdp->swd_flags & SWF_FAKE) 428 continue; 429 if (pgno >= sdp->swd_drumoffset && 430 pgno < (sdp->swd_drumoffset + sdp->swd_drumsize)) { 431 return sdp; 432 } 433 } 434 } 435 return NULL; 436 } 437 438 439 /* 440 * sys_swapctl: main entry point for swapctl(2) system call 441 * [with two helper functions: swap_on and swap_off] 442 */ 443 int 444 sys_swapctl(p, v, retval) 445 struct proc *p; 446 void *v; 447 register_t *retval; 448 { 449 struct sys_swapctl_args /* { 450 syscallarg(int) cmd; 451 syscallarg(void *) arg; 452 syscallarg(int) misc; 453 } */ *uap = (struct sys_swapctl_args *)v; 454 struct vnode *vp; 455 struct nameidata nd; 456 struct swappri *spp; 457 struct swapdev *sdp; 458 struct swapent *sep; 459 char userpath[PATH_MAX + 1]; 460 size_t len; 461 int count, error, misc; 462 int priority; 463 UVMHIST_FUNC("sys_swapctl"); UVMHIST_CALLED(pdhist); 464 465 misc = SCARG(uap, misc); 466 467 /* 468 * ensure serialized syscall access by grabbing the swap_syscall_lock 469 */ 470 lockmgr(&swap_syscall_lock, LK_EXCLUSIVE, NULL); 471 472 /* 473 * we handle the non-priv NSWAP and STATS request first. 474 * 475 * SWAP_NSWAP: return number of config'd swap devices 476 * [can also be obtained with uvmexp sysctl] 477 */ 478 if (SCARG(uap, cmd) == SWAP_NSWAP) { 479 UVMHIST_LOG(pdhist, "<- done SWAP_NSWAP=%d", uvmexp.nswapdev, 480 0, 0, 0); 481 *retval = uvmexp.nswapdev; 482 error = 0; 483 goto out; 484 } 485 486 /* 487 * SWAP_STATS: get stats on current # of configured swap devs 488 * 489 * note that the swap_priority list can't change as long 490 * as we are holding the swap_syscall_lock. we don't want 491 * to grab the uvm.swap_data_lock because we may fault&sleep during 492 * copyout() and we don't want to be holding that lock then! 493 */ 494 if (SCARG(uap, cmd) == SWAP_STATS 495 #if defined(COMPAT_13) 496 || SCARG(uap, cmd) == SWAP_OSTATS 497 #endif 498 ) { 499 sep = (struct swapent *)SCARG(uap, arg); 500 count = 0; 501 502 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 503 for (sdp = CIRCLEQ_FIRST(&spp->spi_swapdev); 504 sdp != (void *)&spp->spi_swapdev && misc-- > 0; 505 sdp = CIRCLEQ_NEXT(sdp, swd_next)) { 506 /* 507 * backwards compatibility for system call. 508 * note that we use 'struct oswapent' as an 509 * overlay into both 'struct swapdev' and 510 * the userland 'struct swapent', as we 511 * want to retain backwards compatibility 512 * with NetBSD 1.3. 513 */ 514 sdp->swd_ose.ose_inuse = 515 btodb((u_int64_t)sdp->swd_npginuse << 516 PAGE_SHIFT); 517 error = copyout(&sdp->swd_ose, sep, 518 sizeof(struct oswapent)); 519 520 /* now copy out the path if necessary */ 521 #if defined(COMPAT_13) 522 if (error == 0 && SCARG(uap, cmd) == SWAP_STATS) 523 #else 524 if (error == 0) 525 #endif 526 error = copyout(sdp->swd_path, 527 &sep->se_path, sdp->swd_pathlen); 528 529 if (error) 530 goto out; 531 count++; 532 #if defined(COMPAT_13) 533 if (SCARG(uap, cmd) == SWAP_OSTATS) 534 sep = (struct swapent *) 535 ((struct oswapent *)sep + 1); 536 else 537 #endif 538 sep++; 539 } 540 } 541 542 UVMHIST_LOG(pdhist, "<- done SWAP_STATS", 0, 0, 0, 0); 543 544 *retval = count; 545 error = 0; 546 goto out; 547 } 548 if (SCARG(uap, cmd) == SWAP_GETDUMPDEV) { 549 dev_t *devp = (dev_t *)SCARG(uap, arg); 550 551 error = copyout(&dumpdev, devp, sizeof(dumpdev)); 552 goto out; 553 } 554 555 /* 556 * all other requests require superuser privs. verify. 557 */ 558 if ((error = suser(p->p_ucred, &p->p_acflag))) 559 goto out; 560 561 /* 562 * at this point we expect a path name in arg. we will 563 * use namei() to gain a vnode reference (vref), and lock 564 * the vnode (VOP_LOCK). 565 * 566 * XXX: a NULL arg means use the root vnode pointer (e.g. for 567 * miniroot) 568 */ 569 if (SCARG(uap, arg) == NULL) { 570 vp = rootvp; /* miniroot */ 571 if (vget(vp, LK_EXCLUSIVE)) { 572 error = EBUSY; 573 goto out; 574 } 575 if (SCARG(uap, cmd) == SWAP_ON && 576 copystr("miniroot", userpath, sizeof userpath, &len)) 577 panic("swapctl: miniroot copy failed"); 578 } else { 579 int space; 580 char *where; 581 582 if (SCARG(uap, cmd) == SWAP_ON) { 583 if ((error = copyinstr(SCARG(uap, arg), userpath, 584 sizeof userpath, &len))) 585 goto out; 586 space = UIO_SYSSPACE; 587 where = userpath; 588 } else { 589 space = UIO_USERSPACE; 590 where = (char *)SCARG(uap, arg); 591 } 592 NDINIT(&nd, LOOKUP, FOLLOW|LOCKLEAF, space, where, p); 593 if ((error = namei(&nd))) 594 goto out; 595 vp = nd.ni_vp; 596 } 597 /* note: "vp" is referenced and locked */ 598 599 error = 0; /* assume no error */ 600 switch(SCARG(uap, cmd)) { 601 602 case SWAP_DUMPDEV: 603 if (vp->v_type != VBLK) { 604 error = ENOTBLK; 605 break; 606 } 607 dumpdev = vp->v_rdev; 608 break; 609 610 case SWAP_CTL: 611 /* 612 * get new priority, remove old entry (if any) and then 613 * reinsert it in the correct place. finally, prune out 614 * any empty priority structures. 615 */ 616 priority = SCARG(uap, misc); 617 spp = malloc(sizeof *spp, M_VMSWAP, M_WAITOK); 618 simple_lock(&uvm.swap_data_lock); 619 if ((sdp = swaplist_find(vp, 1)) == NULL) { 620 error = ENOENT; 621 } else { 622 swaplist_insert(sdp, spp, priority); 623 swaplist_trim(); 624 } 625 simple_unlock(&uvm.swap_data_lock); 626 if (error) 627 free(spp, M_VMSWAP); 628 break; 629 630 case SWAP_ON: 631 632 /* 633 * check for duplicates. if none found, then insert a 634 * dummy entry on the list to prevent someone else from 635 * trying to enable this device while we are working on 636 * it. 637 */ 638 639 priority = SCARG(uap, misc); 640 sdp = malloc(sizeof *sdp, M_VMSWAP, M_WAITOK); 641 spp = malloc(sizeof *spp, M_VMSWAP, M_WAITOK); 642 simple_lock(&uvm.swap_data_lock); 643 if (swaplist_find(vp, 0) != NULL) { 644 error = EBUSY; 645 simple_unlock(&uvm.swap_data_lock); 646 free(sdp, M_VMSWAP); 647 free(spp, M_VMSWAP); 648 break; 649 } 650 memset(sdp, 0, sizeof(*sdp)); 651 sdp->swd_flags = SWF_FAKE; /* placeholder only */ 652 sdp->swd_vp = vp; 653 sdp->swd_dev = (vp->v_type == VBLK) ? vp->v_rdev : NODEV; 654 BUFQ_INIT(&sdp->swd_tab); 655 656 /* 657 * XXX Is NFS elaboration necessary? 658 */ 659 if (vp->v_type == VREG) { 660 sdp->swd_cred = crdup(p->p_ucred); 661 } 662 663 swaplist_insert(sdp, spp, priority); 664 simple_unlock(&uvm.swap_data_lock); 665 666 sdp->swd_pathlen = len; 667 sdp->swd_path = malloc(sdp->swd_pathlen, M_VMSWAP, M_WAITOK); 668 if (copystr(userpath, sdp->swd_path, sdp->swd_pathlen, 0) != 0) 669 panic("swapctl: copystr"); 670 671 /* 672 * we've now got a FAKE placeholder in the swap list. 673 * now attempt to enable swap on it. if we fail, undo 674 * what we've done and kill the fake entry we just inserted. 675 * if swap_on is a success, it will clear the SWF_FAKE flag 676 */ 677 678 if ((error = swap_on(p, sdp)) != 0) { 679 simple_lock(&uvm.swap_data_lock); 680 (void) swaplist_find(vp, 1); /* kill fake entry */ 681 swaplist_trim(); 682 simple_unlock(&uvm.swap_data_lock); 683 if (vp->v_type == VREG) { 684 crfree(sdp->swd_cred); 685 } 686 free(sdp->swd_path, M_VMSWAP); 687 free(sdp, M_VMSWAP); 688 break; 689 } 690 break; 691 692 case SWAP_OFF: 693 simple_lock(&uvm.swap_data_lock); 694 if ((sdp = swaplist_find(vp, 0)) == NULL) { 695 simple_unlock(&uvm.swap_data_lock); 696 error = ENXIO; 697 break; 698 } 699 700 /* 701 * If a device isn't in use or enabled, we 702 * can't stop swapping from it (again). 703 */ 704 if ((sdp->swd_flags & (SWF_INUSE|SWF_ENABLE)) == 0) { 705 simple_unlock(&uvm.swap_data_lock); 706 error = EBUSY; 707 break; 708 } 709 710 /* 711 * do the real work. 712 */ 713 error = swap_off(p, sdp); 714 break; 715 716 default: 717 error = EINVAL; 718 } 719 720 /* 721 * done! release the ref gained by namei() and unlock. 722 */ 723 vput(vp); 724 725 out: 726 lockmgr(&swap_syscall_lock, LK_RELEASE, NULL); 727 728 UVMHIST_LOG(pdhist, "<- done! error=%d", error, 0, 0, 0); 729 return (error); 730 } 731 732 /* 733 * swap_on: attempt to enable a swapdev for swapping. note that the 734 * swapdev is already on the global list, but disabled (marked 735 * SWF_FAKE). 736 * 737 * => we avoid the start of the disk (to protect disk labels) 738 * => we also avoid the miniroot, if we are swapping to root. 739 * => caller should leave uvm.swap_data_lock unlocked, we may lock it 740 * if needed. 741 */ 742 static int 743 swap_on(p, sdp) 744 struct proc *p; 745 struct swapdev *sdp; 746 { 747 static int count = 0; /* static */ 748 struct vnode *vp; 749 int error, npages, nblocks, size; 750 long addr; 751 u_long result; 752 struct vattr va; 753 #ifdef NFS 754 extern int (**nfsv2_vnodeop_p) __P((void *)); 755 #endif /* NFS */ 756 dev_t dev; 757 UVMHIST_FUNC("swap_on"); UVMHIST_CALLED(pdhist); 758 759 /* 760 * we want to enable swapping on sdp. the swd_vp contains 761 * the vnode we want (locked and ref'd), and the swd_dev 762 * contains the dev_t of the file, if it a block device. 763 */ 764 765 vp = sdp->swd_vp; 766 dev = sdp->swd_dev; 767 768 /* 769 * open the swap file (mostly useful for block device files to 770 * let device driver know what is up). 771 * 772 * we skip the open/close for root on swap because the root 773 * has already been opened when root was mounted (mountroot). 774 */ 775 if (vp != rootvp) { 776 if ((error = VOP_OPEN(vp, FREAD|FWRITE, p->p_ucred, p))) 777 return (error); 778 } 779 780 /* XXX this only works for block devices */ 781 UVMHIST_LOG(pdhist, " dev=%d, major(dev)=%d", dev, major(dev), 0,0); 782 783 /* 784 * we now need to determine the size of the swap area. for 785 * block specials we can call the d_psize function. 786 * for normal files, we must stat [get attrs]. 787 * 788 * we put the result in nblks. 789 * for normal files, we also want the filesystem block size 790 * (which we get with statfs). 791 */ 792 switch (vp->v_type) { 793 case VBLK: 794 if (bdevsw[major(dev)].d_psize == 0 || 795 (nblocks = (*bdevsw[major(dev)].d_psize)(dev)) == -1) { 796 error = ENXIO; 797 goto bad; 798 } 799 break; 800 801 case VREG: 802 if ((error = VOP_GETATTR(vp, &va, p->p_ucred, p))) 803 goto bad; 804 nblocks = (int)btodb(va.va_size); 805 if ((error = 806 VFS_STATFS(vp->v_mount, &vp->v_mount->mnt_stat, p)) != 0) 807 goto bad; 808 809 sdp->swd_bsize = vp->v_mount->mnt_stat.f_iosize; 810 /* 811 * limit the max # of outstanding I/O requests we issue 812 * at any one time. take it easy on NFS servers. 813 */ 814 #ifdef NFS 815 if (vp->v_op == nfsv2_vnodeop_p) 816 sdp->swd_maxactive = 2; /* XXX */ 817 else 818 #endif /* NFS */ 819 sdp->swd_maxactive = 8; /* XXX */ 820 break; 821 822 default: 823 error = ENXIO; 824 goto bad; 825 } 826 827 /* 828 * save nblocks in a safe place and convert to pages. 829 */ 830 831 sdp->swd_ose.ose_nblks = nblocks; 832 npages = dbtob((u_int64_t)nblocks) >> PAGE_SHIFT; 833 834 /* 835 * for block special files, we want to make sure that leave 836 * the disklabel and bootblocks alone, so we arrange to skip 837 * over them (arbitrarily choosing to skip PAGE_SIZE bytes). 838 * note that because of this the "size" can be less than the 839 * actual number of blocks on the device. 840 */ 841 if (vp->v_type == VBLK) { 842 /* we use pages 1 to (size - 1) [inclusive] */ 843 size = npages - 1; 844 addr = 1; 845 } else { 846 /* we use pages 0 to (size - 1) [inclusive] */ 847 size = npages; 848 addr = 0; 849 } 850 851 /* 852 * make sure we have enough blocks for a reasonable sized swap 853 * area. we want at least one page. 854 */ 855 856 if (size < 1) { 857 UVMHIST_LOG(pdhist, " size <= 1!!", 0, 0, 0, 0); 858 error = EINVAL; 859 goto bad; 860 } 861 862 UVMHIST_LOG(pdhist, " dev=%x: size=%d addr=%ld\n", dev, size, addr, 0); 863 864 /* 865 * now we need to allocate an extent to manage this swap device 866 */ 867 snprintf(sdp->swd_exname, sizeof(sdp->swd_exname), "swap0x%04x", 868 count++); 869 870 /* note that extent_create's 3rd arg is inclusive, thus "- 1" */ 871 sdp->swd_ex = extent_create(sdp->swd_exname, 0, npages - 1, M_VMSWAP, 872 0, 0, EX_WAITOK); 873 /* allocate the `saved' region from the extent so it won't be used */ 874 if (addr) { 875 if (extent_alloc_region(sdp->swd_ex, 0, addr, EX_WAITOK)) 876 panic("disklabel region"); 877 } 878 879 /* 880 * if the vnode we are swapping to is the root vnode 881 * (i.e. we are swapping to the miniroot) then we want 882 * to make sure we don't overwrite it. do a statfs to 883 * find its size and skip over it. 884 */ 885 if (vp == rootvp) { 886 struct mount *mp; 887 struct statfs *sp; 888 int rootblocks, rootpages; 889 890 mp = rootvnode->v_mount; 891 sp = &mp->mnt_stat; 892 rootblocks = sp->f_blocks * btodb(sp->f_bsize); 893 rootpages = round_page(dbtob(rootblocks)) >> PAGE_SHIFT; 894 if (rootpages > size) 895 panic("swap_on: miniroot larger than swap?"); 896 897 if (extent_alloc_region(sdp->swd_ex, addr, 898 rootpages, EX_WAITOK)) 899 panic("swap_on: unable to preserve miniroot"); 900 901 size -= rootpages; 902 printf("Preserved %d pages of miniroot ", rootpages); 903 printf("leaving %d pages of swap\n", size); 904 } 905 906 /* 907 * try to add anons to reflect the new swap space. 908 */ 909 910 error = uvm_anon_add(size); 911 if (error) { 912 goto bad; 913 } 914 915 /* 916 * add a ref to vp to reflect usage as a swap device. 917 */ 918 vref(vp); 919 920 /* 921 * now add the new swapdev to the drum and enable. 922 */ 923 if (extent_alloc(swapmap, npages, EX_NOALIGN, EX_NOBOUNDARY, 924 EX_WAITOK, &result)) 925 panic("swapdrum_add"); 926 927 sdp->swd_drumoffset = (int)result; 928 sdp->swd_drumsize = npages; 929 sdp->swd_npages = size; 930 simple_lock(&uvm.swap_data_lock); 931 sdp->swd_flags &= ~SWF_FAKE; /* going live */ 932 sdp->swd_flags |= (SWF_INUSE|SWF_ENABLE); 933 uvmexp.swpages += size; 934 simple_unlock(&uvm.swap_data_lock); 935 return (0); 936 937 /* 938 * failure: clean up and return error. 939 */ 940 941 bad: 942 if (sdp->swd_ex) { 943 extent_destroy(sdp->swd_ex); 944 } 945 if (vp != rootvp) { 946 (void)VOP_CLOSE(vp, FREAD|FWRITE, p->p_ucred, p); 947 } 948 return (error); 949 } 950 951 /* 952 * swap_off: stop swapping on swapdev 953 * 954 * => swap data should be locked, we will unlock. 955 */ 956 static int 957 swap_off(p, sdp) 958 struct proc *p; 959 struct swapdev *sdp; 960 { 961 UVMHIST_FUNC("swap_off"); UVMHIST_CALLED(pdhist); 962 UVMHIST_LOG(pdhist, " dev=%x", sdp->swd_dev,0,0,0); 963 964 /* disable the swap area being removed */ 965 sdp->swd_flags &= ~SWF_ENABLE; 966 simple_unlock(&uvm.swap_data_lock); 967 968 /* 969 * the idea is to find all the pages that are paged out to this 970 * device, and page them all in. in uvm, swap-backed pageable 971 * memory can take two forms: aobjs and anons. call the 972 * swapoff hook for each subsystem to bring in pages. 973 */ 974 975 if (uao_swap_off(sdp->swd_drumoffset, 976 sdp->swd_drumoffset + sdp->swd_drumsize) || 977 anon_swap_off(sdp->swd_drumoffset, 978 sdp->swd_drumoffset + sdp->swd_drumsize)) { 979 980 simple_lock(&uvm.swap_data_lock); 981 sdp->swd_flags |= SWF_ENABLE; 982 simple_unlock(&uvm.swap_data_lock); 983 return ENOMEM; 984 } 985 KASSERT(sdp->swd_npginuse == sdp->swd_npgbad); 986 987 /* 988 * done with the vnode and saved creds. 989 * drop our ref on the vnode before calling VOP_CLOSE() 990 * so that spec_close() can tell if this is the last close. 991 */ 992 if (sdp->swd_vp->v_type == VREG) { 993 crfree(sdp->swd_cred); 994 } 995 vrele(sdp->swd_vp); 996 if (sdp->swd_vp != rootvp) { 997 (void) VOP_CLOSE(sdp->swd_vp, FREAD|FWRITE, p->p_ucred, p); 998 } 999 1000 /* remove anons from the system */ 1001 uvm_anon_remove(sdp->swd_npages); 1002 1003 simple_lock(&uvm.swap_data_lock); 1004 uvmexp.swpages -= sdp->swd_npages; 1005 1006 if (swaplist_find(sdp->swd_vp, 1) == NULL) 1007 panic("swap_off: swapdev not in list\n"); 1008 swaplist_trim(); 1009 simple_unlock(&uvm.swap_data_lock); 1010 1011 /* 1012 * free all resources! 1013 */ 1014 extent_free(swapmap, sdp->swd_drumoffset, sdp->swd_drumsize, 1015 EX_WAITOK); 1016 extent_destroy(sdp->swd_ex); 1017 free(sdp, M_VMSWAP); 1018 return (0); 1019 } 1020 1021 /* 1022 * /dev/drum interface and i/o functions 1023 */ 1024 1025 /* 1026 * swread: the read function for the drum (just a call to physio) 1027 */ 1028 /*ARGSUSED*/ 1029 int 1030 swread(dev, uio, ioflag) 1031 dev_t dev; 1032 struct uio *uio; 1033 int ioflag; 1034 { 1035 UVMHIST_FUNC("swread"); UVMHIST_CALLED(pdhist); 1036 1037 UVMHIST_LOG(pdhist, " dev=%x offset=%qx", dev, uio->uio_offset, 0, 0); 1038 return (physio(swstrategy, NULL, dev, B_READ, minphys, uio)); 1039 } 1040 1041 /* 1042 * swwrite: the write function for the drum (just a call to physio) 1043 */ 1044 /*ARGSUSED*/ 1045 int 1046 swwrite(dev, uio, ioflag) 1047 dev_t dev; 1048 struct uio *uio; 1049 int ioflag; 1050 { 1051 UVMHIST_FUNC("swwrite"); UVMHIST_CALLED(pdhist); 1052 1053 UVMHIST_LOG(pdhist, " dev=%x offset=%qx", dev, uio->uio_offset, 0, 0); 1054 return (physio(swstrategy, NULL, dev, B_WRITE, minphys, uio)); 1055 } 1056 1057 /* 1058 * swstrategy: perform I/O on the drum 1059 * 1060 * => we must map the i/o request from the drum to the correct swapdev. 1061 */ 1062 void 1063 swstrategy(bp) 1064 struct buf *bp; 1065 { 1066 struct swapdev *sdp; 1067 struct vnode *vp; 1068 int s, pageno, bn; 1069 UVMHIST_FUNC("swstrategy"); UVMHIST_CALLED(pdhist); 1070 1071 /* 1072 * convert block number to swapdev. note that swapdev can't 1073 * be yanked out from under us because we are holding resources 1074 * in it (i.e. the blocks we are doing I/O on). 1075 */ 1076 pageno = dbtob((int64_t)bp->b_blkno) >> PAGE_SHIFT; 1077 simple_lock(&uvm.swap_data_lock); 1078 sdp = swapdrum_getsdp(pageno); 1079 simple_unlock(&uvm.swap_data_lock); 1080 if (sdp == NULL) { 1081 bp->b_error = EINVAL; 1082 bp->b_flags |= B_ERROR; 1083 biodone(bp); 1084 UVMHIST_LOG(pdhist, " failed to get swap device", 0, 0, 0, 0); 1085 return; 1086 } 1087 1088 /* 1089 * convert drum page number to block number on this swapdev. 1090 */ 1091 1092 pageno -= sdp->swd_drumoffset; /* page # on swapdev */ 1093 bn = btodb((u_int64_t)pageno << PAGE_SHIFT); /* convert to diskblock */ 1094 1095 UVMHIST_LOG(pdhist, " %s: mapoff=%x bn=%x bcount=%ld", 1096 ((bp->b_flags & B_READ) == 0) ? "write" : "read", 1097 sdp->swd_drumoffset, bn, bp->b_bcount); 1098 1099 /* 1100 * for block devices we finish up here. 1101 * for regular files we have to do more work which we delegate 1102 * to sw_reg_strategy(). 1103 */ 1104 1105 switch (sdp->swd_vp->v_type) { 1106 default: 1107 panic("swstrategy: vnode type 0x%x", sdp->swd_vp->v_type); 1108 1109 case VBLK: 1110 1111 /* 1112 * must convert "bp" from an I/O on /dev/drum to an I/O 1113 * on the swapdev (sdp). 1114 */ 1115 s = splbio(); 1116 bp->b_blkno = bn; /* swapdev block number */ 1117 vp = sdp->swd_vp; /* swapdev vnode pointer */ 1118 bp->b_dev = sdp->swd_dev; /* swapdev dev_t */ 1119 1120 /* 1121 * if we are doing a write, we have to redirect the i/o on 1122 * drum's v_numoutput counter to the swapdevs. 1123 */ 1124 if ((bp->b_flags & B_READ) == 0) { 1125 vwakeup(bp); /* kills one 'v_numoutput' on drum */ 1126 vp->v_numoutput++; /* put it on swapdev */ 1127 } 1128 1129 /* 1130 * finally plug in swapdev vnode and start I/O 1131 */ 1132 bp->b_vp = vp; 1133 splx(s); 1134 VOP_STRATEGY(bp); 1135 return; 1136 1137 case VREG: 1138 /* 1139 * delegate to sw_reg_strategy function. 1140 */ 1141 sw_reg_strategy(sdp, bp, bn); 1142 return; 1143 } 1144 /* NOTREACHED */ 1145 } 1146 1147 /* 1148 * sw_reg_strategy: handle swap i/o to regular files 1149 */ 1150 static void 1151 sw_reg_strategy(sdp, bp, bn) 1152 struct swapdev *sdp; 1153 struct buf *bp; 1154 int bn; 1155 { 1156 struct vnode *vp; 1157 struct vndxfer *vnx; 1158 daddr_t nbn; 1159 caddr_t addr; 1160 off_t byteoff; 1161 int s, off, nra, error, sz, resid; 1162 UVMHIST_FUNC("sw_reg_strategy"); UVMHIST_CALLED(pdhist); 1163 1164 /* 1165 * allocate a vndxfer head for this transfer and point it to 1166 * our buffer. 1167 */ 1168 getvndxfer(vnx); 1169 vnx->vx_flags = VX_BUSY; 1170 vnx->vx_error = 0; 1171 vnx->vx_pending = 0; 1172 vnx->vx_bp = bp; 1173 vnx->vx_sdp = sdp; 1174 1175 /* 1176 * setup for main loop where we read filesystem blocks into 1177 * our buffer. 1178 */ 1179 error = 0; 1180 bp->b_resid = bp->b_bcount; /* nothing transfered yet! */ 1181 addr = bp->b_data; /* current position in buffer */ 1182 byteoff = dbtob((u_int64_t)bn); 1183 1184 for (resid = bp->b_resid; resid; resid -= sz) { 1185 struct vndbuf *nbp; 1186 1187 /* 1188 * translate byteoffset into block number. return values: 1189 * vp = vnode of underlying device 1190 * nbn = new block number (on underlying vnode dev) 1191 * nra = num blocks we can read-ahead (excludes requested 1192 * block) 1193 */ 1194 nra = 0; 1195 error = VOP_BMAP(sdp->swd_vp, byteoff / sdp->swd_bsize, 1196 &vp, &nbn, &nra); 1197 1198 if (error == 0 && nbn == (daddr_t)-1) { 1199 /* 1200 * this used to just set error, but that doesn't 1201 * do the right thing. Instead, it causes random 1202 * memory errors. The panic() should remain until 1203 * this condition doesn't destabilize the system. 1204 */ 1205 #if 1 1206 panic("sw_reg_strategy: swap to sparse file"); 1207 #else 1208 error = EIO; /* failure */ 1209 #endif 1210 } 1211 1212 /* 1213 * punt if there was an error or a hole in the file. 1214 * we must wait for any i/o ops we have already started 1215 * to finish before returning. 1216 * 1217 * XXX we could deal with holes here but it would be 1218 * a hassle (in the write case). 1219 */ 1220 if (error) { 1221 s = splbio(); 1222 vnx->vx_error = error; /* pass error up */ 1223 goto out; 1224 } 1225 1226 /* 1227 * compute the size ("sz") of this transfer (in bytes). 1228 */ 1229 off = byteoff % sdp->swd_bsize; 1230 sz = (1 + nra) * sdp->swd_bsize - off; 1231 if (sz > resid) 1232 sz = resid; 1233 1234 UVMHIST_LOG(pdhist, "sw_reg_strategy: " 1235 "vp %p/%p offset 0x%x/0x%x", 1236 sdp->swd_vp, vp, byteoff, nbn); 1237 1238 /* 1239 * now get a buf structure. note that the vb_buf is 1240 * at the front of the nbp structure so that you can 1241 * cast pointers between the two structure easily. 1242 */ 1243 getvndbuf(nbp); 1244 nbp->vb_buf.b_flags = bp->b_flags | B_CALL; 1245 nbp->vb_buf.b_bcount = sz; 1246 nbp->vb_buf.b_bufsize = sz; 1247 nbp->vb_buf.b_error = 0; 1248 nbp->vb_buf.b_data = addr; 1249 nbp->vb_buf.b_lblkno = 0; 1250 nbp->vb_buf.b_blkno = nbn + btodb(off); 1251 nbp->vb_buf.b_rawblkno = nbp->vb_buf.b_blkno; 1252 nbp->vb_buf.b_iodone = sw_reg_iodone; 1253 nbp->vb_buf.b_vp = vp; 1254 if (vp->v_type == VBLK) { 1255 nbp->vb_buf.b_dev = vp->v_rdev; 1256 } 1257 LIST_INIT(&nbp->vb_buf.b_dep); 1258 1259 nbp->vb_xfer = vnx; /* patch it back in to vnx */ 1260 1261 /* 1262 * Just sort by block number 1263 */ 1264 s = splbio(); 1265 if (vnx->vx_error != 0) { 1266 putvndbuf(nbp); 1267 goto out; 1268 } 1269 vnx->vx_pending++; 1270 1271 /* sort it in and start I/O if we are not over our limit */ 1272 disksort_blkno(&sdp->swd_tab, &nbp->vb_buf); 1273 sw_reg_start(sdp); 1274 splx(s); 1275 1276 /* 1277 * advance to the next I/O 1278 */ 1279 byteoff += sz; 1280 addr += sz; 1281 } 1282 1283 s = splbio(); 1284 1285 out: /* Arrive here at splbio */ 1286 vnx->vx_flags &= ~VX_BUSY; 1287 if (vnx->vx_pending == 0) { 1288 if (vnx->vx_error != 0) { 1289 bp->b_error = vnx->vx_error; 1290 bp->b_flags |= B_ERROR; 1291 } 1292 putvndxfer(vnx); 1293 biodone(bp); 1294 } 1295 splx(s); 1296 } 1297 1298 /* 1299 * sw_reg_start: start an I/O request on the requested swapdev 1300 * 1301 * => reqs are sorted by disksort (above) 1302 */ 1303 static void 1304 sw_reg_start(sdp) 1305 struct swapdev *sdp; 1306 { 1307 struct buf *bp; 1308 UVMHIST_FUNC("sw_reg_start"); UVMHIST_CALLED(pdhist); 1309 1310 /* recursion control */ 1311 if ((sdp->swd_flags & SWF_BUSY) != 0) 1312 return; 1313 1314 sdp->swd_flags |= SWF_BUSY; 1315 1316 while (sdp->swd_active < sdp->swd_maxactive) { 1317 bp = BUFQ_FIRST(&sdp->swd_tab); 1318 if (bp == NULL) 1319 break; 1320 BUFQ_REMOVE(&sdp->swd_tab, bp); 1321 sdp->swd_active++; 1322 1323 UVMHIST_LOG(pdhist, 1324 "sw_reg_start: bp %p vp %p blkno %p cnt %lx", 1325 bp, bp->b_vp, bp->b_blkno, bp->b_bcount); 1326 if ((bp->b_flags & B_READ) == 0) 1327 bp->b_vp->v_numoutput++; 1328 1329 VOP_STRATEGY(bp); 1330 } 1331 sdp->swd_flags &= ~SWF_BUSY; 1332 } 1333 1334 /* 1335 * sw_reg_iodone: one of our i/o's has completed and needs post-i/o cleanup 1336 * 1337 * => note that we can recover the vndbuf struct by casting the buf ptr 1338 */ 1339 static void 1340 sw_reg_iodone(bp) 1341 struct buf *bp; 1342 { 1343 struct vndbuf *vbp = (struct vndbuf *) bp; 1344 struct vndxfer *vnx = vbp->vb_xfer; 1345 struct buf *pbp = vnx->vx_bp; /* parent buffer */ 1346 struct swapdev *sdp = vnx->vx_sdp; 1347 int s, resid; 1348 UVMHIST_FUNC("sw_reg_iodone"); UVMHIST_CALLED(pdhist); 1349 1350 UVMHIST_LOG(pdhist, " vbp=%p vp=%p blkno=%x addr=%p", 1351 vbp, vbp->vb_buf.b_vp, vbp->vb_buf.b_blkno, vbp->vb_buf.b_data); 1352 UVMHIST_LOG(pdhist, " cnt=%lx resid=%lx", 1353 vbp->vb_buf.b_bcount, vbp->vb_buf.b_resid, 0, 0); 1354 1355 /* 1356 * protect vbp at splbio and update. 1357 */ 1358 1359 s = splbio(); 1360 resid = vbp->vb_buf.b_bcount - vbp->vb_buf.b_resid; 1361 pbp->b_resid -= resid; 1362 vnx->vx_pending--; 1363 1364 if (vbp->vb_buf.b_error) { 1365 UVMHIST_LOG(pdhist, " got error=%d !", 1366 vbp->vb_buf.b_error, 0, 0, 0); 1367 1368 /* pass error upward */ 1369 vnx->vx_error = vbp->vb_buf.b_error; 1370 } 1371 1372 /* 1373 * kill vbp structure 1374 */ 1375 putvndbuf(vbp); 1376 1377 /* 1378 * wrap up this transaction if it has run to completion or, in 1379 * case of an error, when all auxiliary buffers have returned. 1380 */ 1381 if (vnx->vx_error != 0) { 1382 /* pass error upward */ 1383 pbp->b_flags |= B_ERROR; 1384 pbp->b_error = vnx->vx_error; 1385 if ((vnx->vx_flags & VX_BUSY) == 0 && vnx->vx_pending == 0) { 1386 putvndxfer(vnx); 1387 biodone(pbp); 1388 } 1389 } else if (pbp->b_resid == 0) { 1390 KASSERT(vnx->vx_pending == 0); 1391 if ((vnx->vx_flags & VX_BUSY) == 0) { 1392 UVMHIST_LOG(pdhist, " iodone error=%d !", 1393 pbp, vnx->vx_error, 0, 0); 1394 putvndxfer(vnx); 1395 biodone(pbp); 1396 } 1397 } 1398 1399 /* 1400 * done! start next swapdev I/O if one is pending 1401 */ 1402 sdp->swd_active--; 1403 sw_reg_start(sdp); 1404 splx(s); 1405 } 1406 1407 1408 /* 1409 * uvm_swap_alloc: allocate space on swap 1410 * 1411 * => allocation is done "round robin" down the priority list, as we 1412 * allocate in a priority we "rotate" the circle queue. 1413 * => space can be freed with uvm_swap_free 1414 * => we return the page slot number in /dev/drum (0 == invalid slot) 1415 * => we lock uvm.swap_data_lock 1416 * => XXXMRG: "LESSOK" INTERFACE NEEDED TO EXTENT SYSTEM 1417 */ 1418 int 1419 uvm_swap_alloc(nslots, lessok) 1420 int *nslots; /* IN/OUT */ 1421 boolean_t lessok; 1422 { 1423 struct swapdev *sdp; 1424 struct swappri *spp; 1425 u_long result; 1426 UVMHIST_FUNC("uvm_swap_alloc"); UVMHIST_CALLED(pdhist); 1427 1428 /* 1429 * no swap devices configured yet? definite failure. 1430 */ 1431 if (uvmexp.nswapdev < 1) 1432 return 0; 1433 1434 /* 1435 * lock data lock, convert slots into blocks, and enter loop 1436 */ 1437 simple_lock(&uvm.swap_data_lock); 1438 1439 ReTry: /* XXXMRG */ 1440 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 1441 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 1442 /* if it's not enabled, then we can't swap from it */ 1443 if ((sdp->swd_flags & SWF_ENABLE) == 0) 1444 continue; 1445 if (sdp->swd_npginuse + *nslots > sdp->swd_npages) 1446 continue; 1447 if (extent_alloc(sdp->swd_ex, *nslots, EX_NOALIGN, 1448 EX_NOBOUNDARY, EX_MALLOCOK|EX_NOWAIT, 1449 &result) != 0) { 1450 continue; 1451 } 1452 1453 /* 1454 * successful allocation! now rotate the circleq. 1455 */ 1456 CIRCLEQ_REMOVE(&spp->spi_swapdev, sdp, swd_next); 1457 CIRCLEQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next); 1458 sdp->swd_npginuse += *nslots; 1459 uvmexp.swpginuse += *nslots; 1460 simple_unlock(&uvm.swap_data_lock); 1461 /* done! return drum slot number */ 1462 UVMHIST_LOG(pdhist, 1463 "success! returning %d slots starting at %d", 1464 *nslots, result + sdp->swd_drumoffset, 0, 0); 1465 return (result + sdp->swd_drumoffset); 1466 } 1467 } 1468 1469 /* XXXMRG: BEGIN HACK */ 1470 if (*nslots > 1 && lessok) { 1471 *nslots = 1; 1472 goto ReTry; /* XXXMRG: ugh! extent should support this for us */ 1473 } 1474 /* XXXMRG: END HACK */ 1475 1476 simple_unlock(&uvm.swap_data_lock); 1477 return 0; 1478 } 1479 1480 /* 1481 * uvm_swap_markbad: keep track of swap ranges where we've had i/o errors 1482 * 1483 * => we lock uvm.swap_data_lock 1484 */ 1485 void 1486 uvm_swap_markbad(startslot, nslots) 1487 int startslot; 1488 int nslots; 1489 { 1490 struct swapdev *sdp; 1491 UVMHIST_FUNC("uvm_swap_markbad"); UVMHIST_CALLED(pdhist); 1492 1493 simple_lock(&uvm.swap_data_lock); 1494 sdp = swapdrum_getsdp(startslot); 1495 1496 /* 1497 * we just keep track of how many pages have been marked bad 1498 * in this device, to make everything add up in swap_off(). 1499 * we assume here that the range of slots will all be within 1500 * one swap device. 1501 */ 1502 1503 sdp->swd_npgbad += nslots; 1504 UVMHIST_LOG(pdhist, "now %d bad", sdp->swd_npgbad, 0,0,0); 1505 simple_unlock(&uvm.swap_data_lock); 1506 } 1507 1508 /* 1509 * uvm_swap_free: free swap slots 1510 * 1511 * => this can be all or part of an allocation made by uvm_swap_alloc 1512 * => we lock uvm.swap_data_lock 1513 */ 1514 void 1515 uvm_swap_free(startslot, nslots) 1516 int startslot; 1517 int nslots; 1518 { 1519 struct swapdev *sdp; 1520 UVMHIST_FUNC("uvm_swap_free"); UVMHIST_CALLED(pdhist); 1521 1522 UVMHIST_LOG(pdhist, "freeing %d slots starting at %d", nslots, 1523 startslot, 0, 0); 1524 1525 /* 1526 * ignore attempts to free the "bad" slot. 1527 */ 1528 1529 if (startslot == SWSLOT_BAD) { 1530 return; 1531 } 1532 1533 /* 1534 * convert drum slot offset back to sdp, free the blocks 1535 * in the extent, and return. must hold pri lock to do 1536 * lookup and access the extent. 1537 */ 1538 1539 simple_lock(&uvm.swap_data_lock); 1540 sdp = swapdrum_getsdp(startslot); 1541 KASSERT(uvmexp.nswapdev >= 1); 1542 KASSERT(sdp != NULL); 1543 KASSERT(sdp->swd_npginuse >= nslots); 1544 if (extent_free(sdp->swd_ex, startslot - sdp->swd_drumoffset, nslots, 1545 EX_MALLOCOK|EX_NOWAIT) != 0) { 1546 printf("warning: resource shortage: %d pages of swap lost\n", 1547 nslots); 1548 } 1549 sdp->swd_npginuse -= nslots; 1550 uvmexp.swpginuse -= nslots; 1551 simple_unlock(&uvm.swap_data_lock); 1552 } 1553 1554 /* 1555 * uvm_swap_put: put any number of pages into a contig place on swap 1556 * 1557 * => can be sync or async 1558 */ 1559 1560 int 1561 uvm_swap_put(swslot, ppsp, npages, flags) 1562 int swslot; 1563 struct vm_page **ppsp; 1564 int npages; 1565 int flags; 1566 { 1567 int error; 1568 1569 error = uvm_swap_io(ppsp, swslot, npages, B_WRITE | 1570 ((flags & PGO_SYNCIO) ? 0 : B_ASYNC)); 1571 return error; 1572 } 1573 1574 /* 1575 * uvm_swap_get: get a single page from swap 1576 * 1577 * => usually a sync op (from fault) 1578 */ 1579 1580 int 1581 uvm_swap_get(page, swslot, flags) 1582 struct vm_page *page; 1583 int swslot, flags; 1584 { 1585 int error; 1586 1587 uvmexp.nswget++; 1588 KASSERT(flags & PGO_SYNCIO); 1589 if (swslot == SWSLOT_BAD) { 1590 return EIO; 1591 } 1592 error = uvm_swap_io(&page, swslot, 1, B_READ | 1593 ((flags & PGO_SYNCIO) ? 0 : B_ASYNC)); 1594 if (error == 0) { 1595 1596 /* 1597 * this page is no longer only in swap. 1598 */ 1599 1600 simple_lock(&uvm.swap_data_lock); 1601 KASSERT(uvmexp.swpgonly > 0); 1602 uvmexp.swpgonly--; 1603 simple_unlock(&uvm.swap_data_lock); 1604 } 1605 return error; 1606 } 1607 1608 /* 1609 * uvm_swap_io: do an i/o operation to swap 1610 */ 1611 1612 static int 1613 uvm_swap_io(pps, startslot, npages, flags) 1614 struct vm_page **pps; 1615 int startslot, npages, flags; 1616 { 1617 daddr_t startblk; 1618 struct buf *bp; 1619 vaddr_t kva; 1620 int error, s, mapinflags; 1621 boolean_t write, async; 1622 UVMHIST_FUNC("uvm_swap_io"); UVMHIST_CALLED(pdhist); 1623 1624 UVMHIST_LOG(pdhist, "<- called, startslot=%d, npages=%d, flags=%d", 1625 startslot, npages, flags, 0); 1626 1627 write = (flags & B_READ) == 0; 1628 async = (flags & B_ASYNC) != 0; 1629 1630 /* 1631 * convert starting drum slot to block number 1632 */ 1633 1634 startblk = btodb((u_int64_t)startslot << PAGE_SHIFT); 1635 1636 /* 1637 * first, map the pages into the kernel. 1638 */ 1639 1640 mapinflags = !write ? 1641 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_READ : 1642 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_WRITE; 1643 kva = uvm_pagermapin(pps, npages, mapinflags); 1644 1645 /* 1646 * now allocate a buf for the i/o. 1647 */ 1648 1649 s = splbio(); 1650 bp = pool_get(&bufpool, PR_WAITOK); 1651 splx(s); 1652 1653 /* 1654 * fill in the bp/sbp. we currently route our i/o through 1655 * /dev/drum's vnode [swapdev_vp]. 1656 */ 1657 1658 bp->b_flags = B_BUSY | B_NOCACHE | (flags & (B_READ|B_ASYNC)); 1659 bp->b_proc = &proc0; /* XXX */ 1660 bp->b_vnbufs.le_next = NOLIST; 1661 bp->b_data = (caddr_t)kva; 1662 bp->b_blkno = startblk; 1663 bp->b_vp = swapdev_vp; 1664 bp->b_dev = swapdev_vp->v_rdev; 1665 bp->b_bufsize = bp->b_bcount = npages << PAGE_SHIFT; 1666 LIST_INIT(&bp->b_dep); 1667 1668 /* 1669 * bump v_numoutput (counter of number of active outputs). 1670 */ 1671 1672 if (write) { 1673 s = splbio(); 1674 swapdev_vp->v_numoutput++; 1675 splx(s); 1676 } 1677 1678 /* 1679 * for async ops we must set up the iodone handler. 1680 */ 1681 1682 if (async) { 1683 bp->b_flags |= B_CALL; 1684 bp->b_iodone = uvm_aio_biodone; 1685 UVMHIST_LOG(pdhist, "doing async!", 0, 0, 0, 0); 1686 } 1687 UVMHIST_LOG(pdhist, 1688 "about to start io: data = %p blkno = 0x%x, bcount = %ld", 1689 bp->b_data, bp->b_blkno, bp->b_bcount, 0); 1690 1691 /* 1692 * now we start the I/O, and if async, return. 1693 */ 1694 1695 VOP_STRATEGY(bp); 1696 if (async) 1697 return 0; 1698 1699 /* 1700 * must be sync i/o. wait for it to finish 1701 */ 1702 1703 error = biowait(bp); 1704 1705 /* 1706 * kill the pager mapping 1707 */ 1708 1709 uvm_pagermapout(kva, npages); 1710 1711 /* 1712 * now dispose of the buf and we're done. 1713 */ 1714 1715 s = splbio(); 1716 if (write) 1717 vwakeup(bp); 1718 pool_put(&bufpool, bp); 1719 splx(s); 1720 UVMHIST_LOG(pdhist, "<- done (sync) error=%d", error, 0, 0, 0); 1721 return (error); 1722 } 1723