1 /* $NetBSD: uvm_swap.c,v 1.58 2001/12/16 04:51:34 enami Exp $ */ 2 3 /* 4 * Copyright (c) 1995, 1996, 1997 Matthew R. Green 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of the author may not be used to endorse or promote products 16 * derived from this software without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 * from: NetBSD: vm_swap.c,v 1.52 1997/12/02 13:47:37 pk Exp 31 * from: Id: uvm_swap.c,v 1.1.2.42 1998/02/02 20:38:06 chuck Exp 32 */ 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: uvm_swap.c,v 1.58 2001/12/16 04:51:34 enami Exp $"); 36 37 #include "fs_nfs.h" 38 #include "opt_uvmhist.h" 39 #include "opt_compat_netbsd.h" 40 #include "opt_ddb.h" 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/buf.h> 45 #include <sys/conf.h> 46 #include <sys/proc.h> 47 #include <sys/namei.h> 48 #include <sys/disklabel.h> 49 #include <sys/errno.h> 50 #include <sys/kernel.h> 51 #include <sys/malloc.h> 52 #include <sys/vnode.h> 53 #include <sys/file.h> 54 #include <sys/extent.h> 55 #include <sys/mount.h> 56 #include <sys/pool.h> 57 #include <sys/syscallargs.h> 58 #include <sys/swap.h> 59 60 #include <uvm/uvm.h> 61 62 #include <miscfs/specfs/specdev.h> 63 64 /* 65 * uvm_swap.c: manage configuration and i/o to swap space. 66 */ 67 68 /* 69 * swap space is managed in the following way: 70 * 71 * each swap partition or file is described by a "swapdev" structure. 72 * each "swapdev" structure contains a "swapent" structure which contains 73 * information that is passed up to the user (via system calls). 74 * 75 * each swap partition is assigned a "priority" (int) which controls 76 * swap parition usage. 77 * 78 * the system maintains a global data structure describing all swap 79 * partitions/files. there is a sorted LIST of "swappri" structures 80 * which describe "swapdev"'s at that priority. this LIST is headed 81 * by the "swap_priority" global var. each "swappri" contains a 82 * CIRCLEQ of "swapdev" structures at that priority. 83 * 84 * locking: 85 * - swap_syscall_lock (sleep lock): this lock serializes the swapctl 86 * system call and prevents the swap priority list from changing 87 * while we are in the middle of a system call (e.g. SWAP_STATS). 88 * - uvm.swap_data_lock (simple_lock): this lock protects all swap data 89 * structures including the priority list, the swapdev structures, 90 * and the swapmap extent. 91 * 92 * each swap device has the following info: 93 * - swap device in use (could be disabled, preventing future use) 94 * - swap enabled (allows new allocations on swap) 95 * - map info in /dev/drum 96 * - vnode pointer 97 * for swap files only: 98 * - block size 99 * - max byte count in buffer 100 * - buffer 101 * 102 * userland controls and configures swap with the swapctl(2) system call. 103 * the sys_swapctl performs the following operations: 104 * [1] SWAP_NSWAP: returns the number of swap devices currently configured 105 * [2] SWAP_STATS: given a pointer to an array of swapent structures 106 * (passed in via "arg") of a size passed in via "misc" ... we load 107 * the current swap config into the array. 108 * [3] SWAP_ON: given a pathname in arg (could be device or file) and a 109 * priority in "misc", start swapping on it. 110 * [4] SWAP_OFF: as SWAP_ON, but stops swapping to a device 111 * [5] SWAP_CTL: changes the priority of a swap device (new priority in 112 * "misc") 113 */ 114 115 /* 116 * swapdev: describes a single swap partition/file 117 * 118 * note the following should be true: 119 * swd_inuse <= swd_nblks [number of blocks in use is <= total blocks] 120 * swd_nblks <= swd_mapsize [because mapsize includes miniroot+disklabel] 121 */ 122 struct swapdev { 123 struct oswapent swd_ose; 124 #define swd_dev swd_ose.ose_dev /* device id */ 125 #define swd_flags swd_ose.ose_flags /* flags:inuse/enable/fake */ 126 #define swd_priority swd_ose.ose_priority /* our priority */ 127 /* also: swd_ose.ose_nblks, swd_ose.ose_inuse */ 128 char *swd_path; /* saved pathname of device */ 129 int swd_pathlen; /* length of pathname */ 130 int swd_npages; /* #pages we can use */ 131 int swd_npginuse; /* #pages in use */ 132 int swd_npgbad; /* #pages bad */ 133 int swd_drumoffset; /* page0 offset in drum */ 134 int swd_drumsize; /* #pages in drum */ 135 struct extent *swd_ex; /* extent for this swapdev */ 136 char swd_exname[12]; /* name of extent above */ 137 struct vnode *swd_vp; /* backing vnode */ 138 CIRCLEQ_ENTRY(swapdev) swd_next; /* priority circleq */ 139 140 int swd_bsize; /* blocksize (bytes) */ 141 int swd_maxactive; /* max active i/o reqs */ 142 struct buf_queue swd_tab; /* buffer list */ 143 int swd_active; /* number of active buffers */ 144 }; 145 146 /* 147 * swap device priority entry; the list is kept sorted on `spi_priority'. 148 */ 149 struct swappri { 150 int spi_priority; /* priority */ 151 CIRCLEQ_HEAD(spi_swapdev, swapdev) spi_swapdev; 152 /* circleq of swapdevs at this priority */ 153 LIST_ENTRY(swappri) spi_swappri; /* global list of pri's */ 154 }; 155 156 /* 157 * The following two structures are used to keep track of data transfers 158 * on swap devices associated with regular files. 159 * NOTE: this code is more or less a copy of vnd.c; we use the same 160 * structure names here to ease porting.. 161 */ 162 struct vndxfer { 163 struct buf *vx_bp; /* Pointer to parent buffer */ 164 struct swapdev *vx_sdp; 165 int vx_error; 166 int vx_pending; /* # of pending aux buffers */ 167 int vx_flags; 168 #define VX_BUSY 1 169 #define VX_DEAD 2 170 }; 171 172 struct vndbuf { 173 struct buf vb_buf; 174 struct vndxfer *vb_xfer; 175 }; 176 177 178 /* 179 * We keep a of pool vndbuf's and vndxfer structures. 180 */ 181 static struct pool vndxfer_pool; 182 static struct pool vndbuf_pool; 183 184 #define getvndxfer(vnx) do { \ 185 int s = splbio(); \ 186 vnx = pool_get(&vndxfer_pool, PR_MALLOCOK|PR_WAITOK); \ 187 splx(s); \ 188 } while (0) 189 190 #define putvndxfer(vnx) { \ 191 pool_put(&vndxfer_pool, (void *)(vnx)); \ 192 } 193 194 #define getvndbuf(vbp) do { \ 195 int s = splbio(); \ 196 vbp = pool_get(&vndbuf_pool, PR_MALLOCOK|PR_WAITOK); \ 197 splx(s); \ 198 } while (0) 199 200 #define putvndbuf(vbp) { \ 201 pool_put(&vndbuf_pool, (void *)(vbp)); \ 202 } 203 204 /* /dev/drum */ 205 bdev_decl(sw); 206 cdev_decl(sw); 207 208 /* 209 * local variables 210 */ 211 static struct extent *swapmap; /* controls the mapping of /dev/drum */ 212 213 /* list of all active swap devices [by priority] */ 214 LIST_HEAD(swap_priority, swappri); 215 static struct swap_priority swap_priority; 216 217 /* locks */ 218 struct lock swap_syscall_lock; 219 220 /* 221 * prototypes 222 */ 223 static struct swapdev *swapdrum_getsdp __P((int)); 224 225 static struct swapdev *swaplist_find __P((struct vnode *, int)); 226 static void swaplist_insert __P((struct swapdev *, 227 struct swappri *, int)); 228 static void swaplist_trim __P((void)); 229 230 static int swap_on __P((struct proc *, struct swapdev *)); 231 static int swap_off __P((struct proc *, struct swapdev *)); 232 233 static void sw_reg_strategy __P((struct swapdev *, struct buf *, int)); 234 static void sw_reg_iodone __P((struct buf *)); 235 static void sw_reg_start __P((struct swapdev *)); 236 237 static int uvm_swap_io __P((struct vm_page **, int, int, int)); 238 239 /* 240 * uvm_swap_init: init the swap system data structures and locks 241 * 242 * => called at boot time from init_main.c after the filesystems 243 * are brought up (which happens after uvm_init()) 244 */ 245 void 246 uvm_swap_init() 247 { 248 UVMHIST_FUNC("uvm_swap_init"); 249 250 UVMHIST_CALLED(pdhist); 251 /* 252 * first, init the swap list, its counter, and its lock. 253 * then get a handle on the vnode for /dev/drum by using 254 * the its dev_t number ("swapdev", from MD conf.c). 255 */ 256 257 LIST_INIT(&swap_priority); 258 uvmexp.nswapdev = 0; 259 lockinit(&swap_syscall_lock, PVM, "swapsys", 0, 0); 260 simple_lock_init(&uvm.swap_data_lock); 261 262 if (bdevvp(swapdev, &swapdev_vp)) 263 panic("uvm_swap_init: can't get vnode for swap device"); 264 265 /* 266 * create swap block resource map to map /dev/drum. the range 267 * from 1 to INT_MAX allows 2 gigablocks of swap space. note 268 * that block 0 is reserved (used to indicate an allocation 269 * failure, or no allocation). 270 */ 271 swapmap = extent_create("swapmap", 1, INT_MAX, 272 M_VMSWAP, 0, 0, EX_NOWAIT); 273 if (swapmap == 0) 274 panic("uvm_swap_init: extent_create failed"); 275 276 /* 277 * allocate pools for structures used for swapping to files. 278 */ 279 280 pool_init(&vndxfer_pool, sizeof(struct vndxfer), 0, 0, 0, 281 "swp vnx", 0, NULL, NULL, 0); 282 283 pool_init(&vndbuf_pool, sizeof(struct vndbuf), 0, 0, 0, 284 "swp vnd", 0, NULL, NULL, 0); 285 286 /* 287 * done! 288 */ 289 UVMHIST_LOG(pdhist, "<- done", 0, 0, 0, 0); 290 } 291 292 /* 293 * swaplist functions: functions that operate on the list of swap 294 * devices on the system. 295 */ 296 297 /* 298 * swaplist_insert: insert swap device "sdp" into the global list 299 * 300 * => caller must hold both swap_syscall_lock and uvm.swap_data_lock 301 * => caller must provide a newly malloc'd swappri structure (we will 302 * FREE it if we don't need it... this it to prevent malloc blocking 303 * here while adding swap) 304 */ 305 static void 306 swaplist_insert(sdp, newspp, priority) 307 struct swapdev *sdp; 308 struct swappri *newspp; 309 int priority; 310 { 311 struct swappri *spp, *pspp; 312 UVMHIST_FUNC("swaplist_insert"); UVMHIST_CALLED(pdhist); 313 314 /* 315 * find entry at or after which to insert the new device. 316 */ 317 pspp = NULL; 318 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 319 if (priority <= spp->spi_priority) 320 break; 321 pspp = spp; 322 } 323 324 /* 325 * new priority? 326 */ 327 if (spp == NULL || spp->spi_priority != priority) { 328 spp = newspp; /* use newspp! */ 329 UVMHIST_LOG(pdhist, "created new swappri = %d", 330 priority, 0, 0, 0); 331 332 spp->spi_priority = priority; 333 CIRCLEQ_INIT(&spp->spi_swapdev); 334 335 if (pspp) 336 LIST_INSERT_AFTER(pspp, spp, spi_swappri); 337 else 338 LIST_INSERT_HEAD(&swap_priority, spp, spi_swappri); 339 } else { 340 /* we don't need a new priority structure, free it */ 341 FREE(newspp, M_VMSWAP); 342 } 343 344 /* 345 * priority found (or created). now insert on the priority's 346 * circleq list and bump the total number of swapdevs. 347 */ 348 sdp->swd_priority = priority; 349 CIRCLEQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next); 350 uvmexp.nswapdev++; 351 } 352 353 /* 354 * swaplist_find: find and optionally remove a swap device from the 355 * global list. 356 * 357 * => caller must hold both swap_syscall_lock and uvm.swap_data_lock 358 * => we return the swapdev we found (and removed) 359 */ 360 static struct swapdev * 361 swaplist_find(vp, remove) 362 struct vnode *vp; 363 boolean_t remove; 364 { 365 struct swapdev *sdp; 366 struct swappri *spp; 367 368 /* 369 * search the lists for the requested vp 370 */ 371 372 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 373 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 374 if (sdp->swd_vp == vp) { 375 if (remove) { 376 CIRCLEQ_REMOVE(&spp->spi_swapdev, 377 sdp, swd_next); 378 uvmexp.nswapdev--; 379 } 380 return(sdp); 381 } 382 } 383 } 384 return (NULL); 385 } 386 387 388 /* 389 * swaplist_trim: scan priority list for empty priority entries and kill 390 * them. 391 * 392 * => caller must hold both swap_syscall_lock and uvm.swap_data_lock 393 */ 394 static void 395 swaplist_trim() 396 { 397 struct swappri *spp, *nextspp; 398 399 for (spp = LIST_FIRST(&swap_priority); spp != NULL; spp = nextspp) { 400 nextspp = LIST_NEXT(spp, spi_swappri); 401 if (CIRCLEQ_FIRST(&spp->spi_swapdev) != 402 (void *)&spp->spi_swapdev) 403 continue; 404 LIST_REMOVE(spp, spi_swappri); 405 free(spp, M_VMSWAP); 406 } 407 } 408 409 /* 410 * swapdrum_getsdp: given a page offset in /dev/drum, convert it back 411 * to the "swapdev" that maps that section of the drum. 412 * 413 * => each swapdev takes one big contig chunk of the drum 414 * => caller must hold uvm.swap_data_lock 415 */ 416 static struct swapdev * 417 swapdrum_getsdp(pgno) 418 int pgno; 419 { 420 struct swapdev *sdp; 421 struct swappri *spp; 422 423 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 424 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 425 if (sdp->swd_flags & SWF_FAKE) 426 continue; 427 if (pgno >= sdp->swd_drumoffset && 428 pgno < (sdp->swd_drumoffset + sdp->swd_drumsize)) { 429 return sdp; 430 } 431 } 432 } 433 return NULL; 434 } 435 436 437 /* 438 * sys_swapctl: main entry point for swapctl(2) system call 439 * [with two helper functions: swap_on and swap_off] 440 */ 441 int 442 sys_swapctl(p, v, retval) 443 struct proc *p; 444 void *v; 445 register_t *retval; 446 { 447 struct sys_swapctl_args /* { 448 syscallarg(int) cmd; 449 syscallarg(void *) arg; 450 syscallarg(int) misc; 451 } */ *uap = (struct sys_swapctl_args *)v; 452 struct vnode *vp; 453 struct nameidata nd; 454 struct swappri *spp; 455 struct swapdev *sdp; 456 struct swapent *sep; 457 char userpath[PATH_MAX + 1]; 458 size_t len; 459 int count, error, misc; 460 int priority; 461 UVMHIST_FUNC("sys_swapctl"); UVMHIST_CALLED(pdhist); 462 463 misc = SCARG(uap, misc); 464 465 /* 466 * ensure serialized syscall access by grabbing the swap_syscall_lock 467 */ 468 lockmgr(&swap_syscall_lock, LK_EXCLUSIVE, NULL); 469 470 /* 471 * we handle the non-priv NSWAP and STATS request first. 472 * 473 * SWAP_NSWAP: return number of config'd swap devices 474 * [can also be obtained with uvmexp sysctl] 475 */ 476 if (SCARG(uap, cmd) == SWAP_NSWAP) { 477 UVMHIST_LOG(pdhist, "<- done SWAP_NSWAP=%d", uvmexp.nswapdev, 478 0, 0, 0); 479 *retval = uvmexp.nswapdev; 480 error = 0; 481 goto out; 482 } 483 484 /* 485 * SWAP_STATS: get stats on current # of configured swap devs 486 * 487 * note that the swap_priority list can't change as long 488 * as we are holding the swap_syscall_lock. we don't want 489 * to grab the uvm.swap_data_lock because we may fault&sleep during 490 * copyout() and we don't want to be holding that lock then! 491 */ 492 if (SCARG(uap, cmd) == SWAP_STATS 493 #if defined(COMPAT_13) 494 || SCARG(uap, cmd) == SWAP_OSTATS 495 #endif 496 ) { 497 sep = (struct swapent *)SCARG(uap, arg); 498 count = 0; 499 500 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 501 for (sdp = CIRCLEQ_FIRST(&spp->spi_swapdev); 502 sdp != (void *)&spp->spi_swapdev && misc-- > 0; 503 sdp = CIRCLEQ_NEXT(sdp, swd_next)) { 504 /* 505 * backwards compatibility for system call. 506 * note that we use 'struct oswapent' as an 507 * overlay into both 'struct swapdev' and 508 * the userland 'struct swapent', as we 509 * want to retain backwards compatibility 510 * with NetBSD 1.3. 511 */ 512 sdp->swd_ose.ose_inuse = 513 btodb((u_int64_t)sdp->swd_npginuse << 514 PAGE_SHIFT); 515 error = copyout(&sdp->swd_ose, sep, 516 sizeof(struct oswapent)); 517 518 /* now copy out the path if necessary */ 519 #if defined(COMPAT_13) 520 if (error == 0 && SCARG(uap, cmd) == SWAP_STATS) 521 #else 522 if (error == 0) 523 #endif 524 error = copyout(sdp->swd_path, 525 &sep->se_path, sdp->swd_pathlen); 526 527 if (error) 528 goto out; 529 count++; 530 #if defined(COMPAT_13) 531 if (SCARG(uap, cmd) == SWAP_OSTATS) 532 sep = (struct swapent *) 533 ((struct oswapent *)sep + 1); 534 else 535 #endif 536 sep++; 537 } 538 } 539 540 UVMHIST_LOG(pdhist, "<- done SWAP_STATS", 0, 0, 0, 0); 541 542 *retval = count; 543 error = 0; 544 goto out; 545 } 546 if (SCARG(uap, cmd) == SWAP_GETDUMPDEV) { 547 dev_t *devp = (dev_t *)SCARG(uap, arg); 548 549 error = copyout(&dumpdev, devp, sizeof(dumpdev)); 550 goto out; 551 } 552 553 /* 554 * all other requests require superuser privs. verify. 555 */ 556 if ((error = suser(p->p_ucred, &p->p_acflag))) 557 goto out; 558 559 /* 560 * at this point we expect a path name in arg. we will 561 * use namei() to gain a vnode reference (vref), and lock 562 * the vnode (VOP_LOCK). 563 * 564 * XXX: a NULL arg means use the root vnode pointer (e.g. for 565 * miniroot) 566 */ 567 if (SCARG(uap, arg) == NULL) { 568 vp = rootvp; /* miniroot */ 569 if (vget(vp, LK_EXCLUSIVE)) { 570 error = EBUSY; 571 goto out; 572 } 573 if (SCARG(uap, cmd) == SWAP_ON && 574 copystr("miniroot", userpath, sizeof userpath, &len)) 575 panic("swapctl: miniroot copy failed"); 576 } else { 577 int space; 578 char *where; 579 580 if (SCARG(uap, cmd) == SWAP_ON) { 581 if ((error = copyinstr(SCARG(uap, arg), userpath, 582 sizeof userpath, &len))) 583 goto out; 584 space = UIO_SYSSPACE; 585 where = userpath; 586 } else { 587 space = UIO_USERSPACE; 588 where = (char *)SCARG(uap, arg); 589 } 590 NDINIT(&nd, LOOKUP, FOLLOW|LOCKLEAF, space, where, p); 591 if ((error = namei(&nd))) 592 goto out; 593 vp = nd.ni_vp; 594 } 595 /* note: "vp" is referenced and locked */ 596 597 error = 0; /* assume no error */ 598 switch(SCARG(uap, cmd)) { 599 600 case SWAP_DUMPDEV: 601 if (vp->v_type != VBLK) { 602 error = ENOTBLK; 603 break; 604 } 605 dumpdev = vp->v_rdev; 606 break; 607 608 case SWAP_CTL: 609 /* 610 * get new priority, remove old entry (if any) and then 611 * reinsert it in the correct place. finally, prune out 612 * any empty priority structures. 613 */ 614 priority = SCARG(uap, misc); 615 spp = malloc(sizeof *spp, M_VMSWAP, M_WAITOK); 616 simple_lock(&uvm.swap_data_lock); 617 if ((sdp = swaplist_find(vp, 1)) == NULL) { 618 error = ENOENT; 619 } else { 620 swaplist_insert(sdp, spp, priority); 621 swaplist_trim(); 622 } 623 simple_unlock(&uvm.swap_data_lock); 624 if (error) 625 free(spp, M_VMSWAP); 626 break; 627 628 case SWAP_ON: 629 630 /* 631 * check for duplicates. if none found, then insert a 632 * dummy entry on the list to prevent someone else from 633 * trying to enable this device while we are working on 634 * it. 635 */ 636 637 priority = SCARG(uap, misc); 638 sdp = malloc(sizeof *sdp, M_VMSWAP, M_WAITOK); 639 spp = malloc(sizeof *spp, M_VMSWAP, M_WAITOK); 640 simple_lock(&uvm.swap_data_lock); 641 if (swaplist_find(vp, 0) != NULL) { 642 error = EBUSY; 643 simple_unlock(&uvm.swap_data_lock); 644 free(sdp, M_VMSWAP); 645 free(spp, M_VMSWAP); 646 break; 647 } 648 memset(sdp, 0, sizeof(*sdp)); 649 sdp->swd_flags = SWF_FAKE; /* placeholder only */ 650 sdp->swd_vp = vp; 651 sdp->swd_dev = (vp->v_type == VBLK) ? vp->v_rdev : NODEV; 652 BUFQ_INIT(&sdp->swd_tab); 653 654 swaplist_insert(sdp, spp, priority); 655 simple_unlock(&uvm.swap_data_lock); 656 657 sdp->swd_pathlen = len; 658 sdp->swd_path = malloc(sdp->swd_pathlen, M_VMSWAP, M_WAITOK); 659 if (copystr(userpath, sdp->swd_path, sdp->swd_pathlen, 0) != 0) 660 panic("swapctl: copystr"); 661 662 /* 663 * we've now got a FAKE placeholder in the swap list. 664 * now attempt to enable swap on it. if we fail, undo 665 * what we've done and kill the fake entry we just inserted. 666 * if swap_on is a success, it will clear the SWF_FAKE flag 667 */ 668 669 if ((error = swap_on(p, sdp)) != 0) { 670 simple_lock(&uvm.swap_data_lock); 671 (void) swaplist_find(vp, 1); /* kill fake entry */ 672 swaplist_trim(); 673 simple_unlock(&uvm.swap_data_lock); 674 free(sdp->swd_path, M_VMSWAP); 675 free(sdp, M_VMSWAP); 676 break; 677 } 678 break; 679 680 case SWAP_OFF: 681 simple_lock(&uvm.swap_data_lock); 682 if ((sdp = swaplist_find(vp, 0)) == NULL) { 683 simple_unlock(&uvm.swap_data_lock); 684 error = ENXIO; 685 break; 686 } 687 688 /* 689 * If a device isn't in use or enabled, we 690 * can't stop swapping from it (again). 691 */ 692 if ((sdp->swd_flags & (SWF_INUSE|SWF_ENABLE)) == 0) { 693 simple_unlock(&uvm.swap_data_lock); 694 error = EBUSY; 695 break; 696 } 697 698 /* 699 * do the real work. 700 */ 701 error = swap_off(p, sdp); 702 break; 703 704 default: 705 error = EINVAL; 706 } 707 708 /* 709 * done! release the ref gained by namei() and unlock. 710 */ 711 vput(vp); 712 713 out: 714 lockmgr(&swap_syscall_lock, LK_RELEASE, NULL); 715 716 UVMHIST_LOG(pdhist, "<- done! error=%d", error, 0, 0, 0); 717 return (error); 718 } 719 720 /* 721 * swap_on: attempt to enable a swapdev for swapping. note that the 722 * swapdev is already on the global list, but disabled (marked 723 * SWF_FAKE). 724 * 725 * => we avoid the start of the disk (to protect disk labels) 726 * => we also avoid the miniroot, if we are swapping to root. 727 * => caller should leave uvm.swap_data_lock unlocked, we may lock it 728 * if needed. 729 */ 730 static int 731 swap_on(p, sdp) 732 struct proc *p; 733 struct swapdev *sdp; 734 { 735 static int count = 0; /* static */ 736 struct vnode *vp; 737 int error, npages, nblocks, size; 738 long addr; 739 u_long result; 740 struct vattr va; 741 #ifdef NFS 742 extern int (**nfsv2_vnodeop_p) __P((void *)); 743 #endif /* NFS */ 744 dev_t dev; 745 UVMHIST_FUNC("swap_on"); UVMHIST_CALLED(pdhist); 746 747 /* 748 * we want to enable swapping on sdp. the swd_vp contains 749 * the vnode we want (locked and ref'd), and the swd_dev 750 * contains the dev_t of the file, if it a block device. 751 */ 752 753 vp = sdp->swd_vp; 754 dev = sdp->swd_dev; 755 756 /* 757 * open the swap file (mostly useful for block device files to 758 * let device driver know what is up). 759 * 760 * we skip the open/close for root on swap because the root 761 * has already been opened when root was mounted (mountroot). 762 */ 763 if (vp != rootvp) { 764 if ((error = VOP_OPEN(vp, FREAD|FWRITE, p->p_ucred, p))) 765 return (error); 766 } 767 768 /* XXX this only works for block devices */ 769 UVMHIST_LOG(pdhist, " dev=%d, major(dev)=%d", dev, major(dev), 0,0); 770 771 /* 772 * we now need to determine the size of the swap area. for 773 * block specials we can call the d_psize function. 774 * for normal files, we must stat [get attrs]. 775 * 776 * we put the result in nblks. 777 * for normal files, we also want the filesystem block size 778 * (which we get with statfs). 779 */ 780 switch (vp->v_type) { 781 case VBLK: 782 if (bdevsw[major(dev)].d_psize == 0 || 783 (nblocks = (*bdevsw[major(dev)].d_psize)(dev)) == -1) { 784 error = ENXIO; 785 goto bad; 786 } 787 break; 788 789 case VREG: 790 if ((error = VOP_GETATTR(vp, &va, p->p_ucred, p))) 791 goto bad; 792 nblocks = (int)btodb(va.va_size); 793 if ((error = 794 VFS_STATFS(vp->v_mount, &vp->v_mount->mnt_stat, p)) != 0) 795 goto bad; 796 797 sdp->swd_bsize = vp->v_mount->mnt_stat.f_iosize; 798 /* 799 * limit the max # of outstanding I/O requests we issue 800 * at any one time. take it easy on NFS servers. 801 */ 802 #ifdef NFS 803 if (vp->v_op == nfsv2_vnodeop_p) 804 sdp->swd_maxactive = 2; /* XXX */ 805 else 806 #endif /* NFS */ 807 sdp->swd_maxactive = 8; /* XXX */ 808 break; 809 810 default: 811 error = ENXIO; 812 goto bad; 813 } 814 815 /* 816 * save nblocks in a safe place and convert to pages. 817 */ 818 819 sdp->swd_ose.ose_nblks = nblocks; 820 npages = dbtob((u_int64_t)nblocks) >> PAGE_SHIFT; 821 822 /* 823 * for block special files, we want to make sure that leave 824 * the disklabel and bootblocks alone, so we arrange to skip 825 * over them (arbitrarily choosing to skip PAGE_SIZE bytes). 826 * note that because of this the "size" can be less than the 827 * actual number of blocks on the device. 828 */ 829 if (vp->v_type == VBLK) { 830 /* we use pages 1 to (size - 1) [inclusive] */ 831 size = npages - 1; 832 addr = 1; 833 } else { 834 /* we use pages 0 to (size - 1) [inclusive] */ 835 size = npages; 836 addr = 0; 837 } 838 839 /* 840 * make sure we have enough blocks for a reasonable sized swap 841 * area. we want at least one page. 842 */ 843 844 if (size < 1) { 845 UVMHIST_LOG(pdhist, " size <= 1!!", 0, 0, 0, 0); 846 error = EINVAL; 847 goto bad; 848 } 849 850 UVMHIST_LOG(pdhist, " dev=%x: size=%d addr=%ld\n", dev, size, addr, 0); 851 852 /* 853 * now we need to allocate an extent to manage this swap device 854 */ 855 snprintf(sdp->swd_exname, sizeof(sdp->swd_exname), "swap0x%04x", 856 count++); 857 858 /* note that extent_create's 3rd arg is inclusive, thus "- 1" */ 859 sdp->swd_ex = extent_create(sdp->swd_exname, 0, npages - 1, M_VMSWAP, 860 0, 0, EX_WAITOK); 861 /* allocate the `saved' region from the extent so it won't be used */ 862 if (addr) { 863 if (extent_alloc_region(sdp->swd_ex, 0, addr, EX_WAITOK)) 864 panic("disklabel region"); 865 } 866 867 /* 868 * if the vnode we are swapping to is the root vnode 869 * (i.e. we are swapping to the miniroot) then we want 870 * to make sure we don't overwrite it. do a statfs to 871 * find its size and skip over it. 872 */ 873 if (vp == rootvp) { 874 struct mount *mp; 875 struct statfs *sp; 876 int rootblocks, rootpages; 877 878 mp = rootvnode->v_mount; 879 sp = &mp->mnt_stat; 880 rootblocks = sp->f_blocks * btodb(sp->f_bsize); 881 rootpages = round_page(dbtob(rootblocks)) >> PAGE_SHIFT; 882 if (rootpages > size) 883 panic("swap_on: miniroot larger than swap?"); 884 885 if (extent_alloc_region(sdp->swd_ex, addr, 886 rootpages, EX_WAITOK)) 887 panic("swap_on: unable to preserve miniroot"); 888 889 size -= rootpages; 890 printf("Preserved %d pages of miniroot ", rootpages); 891 printf("leaving %d pages of swap\n", size); 892 } 893 894 /* 895 * try to add anons to reflect the new swap space. 896 */ 897 898 error = uvm_anon_add(size); 899 if (error) { 900 goto bad; 901 } 902 903 /* 904 * add a ref to vp to reflect usage as a swap device. 905 */ 906 vref(vp); 907 908 /* 909 * now add the new swapdev to the drum and enable. 910 */ 911 if (extent_alloc(swapmap, npages, EX_NOALIGN, EX_NOBOUNDARY, 912 EX_WAITOK, &result)) 913 panic("swapdrum_add"); 914 915 sdp->swd_drumoffset = (int)result; 916 sdp->swd_drumsize = npages; 917 sdp->swd_npages = size; 918 simple_lock(&uvm.swap_data_lock); 919 sdp->swd_flags &= ~SWF_FAKE; /* going live */ 920 sdp->swd_flags |= (SWF_INUSE|SWF_ENABLE); 921 uvmexp.swpages += size; 922 simple_unlock(&uvm.swap_data_lock); 923 return (0); 924 925 /* 926 * failure: clean up and return error. 927 */ 928 929 bad: 930 if (sdp->swd_ex) { 931 extent_destroy(sdp->swd_ex); 932 } 933 if (vp != rootvp) { 934 (void)VOP_CLOSE(vp, FREAD|FWRITE, p->p_ucred, p); 935 } 936 return (error); 937 } 938 939 /* 940 * swap_off: stop swapping on swapdev 941 * 942 * => swap data should be locked, we will unlock. 943 */ 944 static int 945 swap_off(p, sdp) 946 struct proc *p; 947 struct swapdev *sdp; 948 { 949 UVMHIST_FUNC("swap_off"); UVMHIST_CALLED(pdhist); 950 UVMHIST_LOG(pdhist, " dev=%x", sdp->swd_dev,0,0,0); 951 952 /* disable the swap area being removed */ 953 sdp->swd_flags &= ~SWF_ENABLE; 954 simple_unlock(&uvm.swap_data_lock); 955 956 /* 957 * the idea is to find all the pages that are paged out to this 958 * device, and page them all in. in uvm, swap-backed pageable 959 * memory can take two forms: aobjs and anons. call the 960 * swapoff hook for each subsystem to bring in pages. 961 */ 962 963 if (uao_swap_off(sdp->swd_drumoffset, 964 sdp->swd_drumoffset + sdp->swd_drumsize) || 965 anon_swap_off(sdp->swd_drumoffset, 966 sdp->swd_drumoffset + sdp->swd_drumsize)) { 967 968 simple_lock(&uvm.swap_data_lock); 969 sdp->swd_flags |= SWF_ENABLE; 970 simple_unlock(&uvm.swap_data_lock); 971 return ENOMEM; 972 } 973 KASSERT(sdp->swd_npginuse == sdp->swd_npgbad); 974 975 /* 976 * done with the vnode. 977 * drop our ref on the vnode before calling VOP_CLOSE() 978 * so that spec_close() can tell if this is the last close. 979 */ 980 vrele(sdp->swd_vp); 981 if (sdp->swd_vp != rootvp) { 982 (void) VOP_CLOSE(sdp->swd_vp, FREAD|FWRITE, p->p_ucred, p); 983 } 984 985 /* remove anons from the system */ 986 uvm_anon_remove(sdp->swd_npages); 987 988 simple_lock(&uvm.swap_data_lock); 989 uvmexp.swpages -= sdp->swd_npages; 990 991 if (swaplist_find(sdp->swd_vp, 1) == NULL) 992 panic("swap_off: swapdev not in list\n"); 993 swaplist_trim(); 994 simple_unlock(&uvm.swap_data_lock); 995 996 /* 997 * free all resources! 998 */ 999 extent_free(swapmap, sdp->swd_drumoffset, sdp->swd_drumsize, 1000 EX_WAITOK); 1001 extent_destroy(sdp->swd_ex); 1002 free(sdp, M_VMSWAP); 1003 return (0); 1004 } 1005 1006 /* 1007 * /dev/drum interface and i/o functions 1008 */ 1009 1010 /* 1011 * swread: the read function for the drum (just a call to physio) 1012 */ 1013 /*ARGSUSED*/ 1014 int 1015 swread(dev, uio, ioflag) 1016 dev_t dev; 1017 struct uio *uio; 1018 int ioflag; 1019 { 1020 UVMHIST_FUNC("swread"); UVMHIST_CALLED(pdhist); 1021 1022 UVMHIST_LOG(pdhist, " dev=%x offset=%qx", dev, uio->uio_offset, 0, 0); 1023 return (physio(swstrategy, NULL, dev, B_READ, minphys, uio)); 1024 } 1025 1026 /* 1027 * swwrite: the write function for the drum (just a call to physio) 1028 */ 1029 /*ARGSUSED*/ 1030 int 1031 swwrite(dev, uio, ioflag) 1032 dev_t dev; 1033 struct uio *uio; 1034 int ioflag; 1035 { 1036 UVMHIST_FUNC("swwrite"); UVMHIST_CALLED(pdhist); 1037 1038 UVMHIST_LOG(pdhist, " dev=%x offset=%qx", dev, uio->uio_offset, 0, 0); 1039 return (physio(swstrategy, NULL, dev, B_WRITE, minphys, uio)); 1040 } 1041 1042 /* 1043 * swstrategy: perform I/O on the drum 1044 * 1045 * => we must map the i/o request from the drum to the correct swapdev. 1046 */ 1047 void 1048 swstrategy(bp) 1049 struct buf *bp; 1050 { 1051 struct swapdev *sdp; 1052 struct vnode *vp; 1053 int s, pageno, bn; 1054 UVMHIST_FUNC("swstrategy"); UVMHIST_CALLED(pdhist); 1055 1056 /* 1057 * convert block number to swapdev. note that swapdev can't 1058 * be yanked out from under us because we are holding resources 1059 * in it (i.e. the blocks we are doing I/O on). 1060 */ 1061 pageno = dbtob((int64_t)bp->b_blkno) >> PAGE_SHIFT; 1062 simple_lock(&uvm.swap_data_lock); 1063 sdp = swapdrum_getsdp(pageno); 1064 simple_unlock(&uvm.swap_data_lock); 1065 if (sdp == NULL) { 1066 bp->b_error = EINVAL; 1067 bp->b_flags |= B_ERROR; 1068 biodone(bp); 1069 UVMHIST_LOG(pdhist, " failed to get swap device", 0, 0, 0, 0); 1070 return; 1071 } 1072 1073 /* 1074 * convert drum page number to block number on this swapdev. 1075 */ 1076 1077 pageno -= sdp->swd_drumoffset; /* page # on swapdev */ 1078 bn = btodb((u_int64_t)pageno << PAGE_SHIFT); /* convert to diskblock */ 1079 1080 UVMHIST_LOG(pdhist, " %s: mapoff=%x bn=%x bcount=%ld", 1081 ((bp->b_flags & B_READ) == 0) ? "write" : "read", 1082 sdp->swd_drumoffset, bn, bp->b_bcount); 1083 1084 /* 1085 * for block devices we finish up here. 1086 * for regular files we have to do more work which we delegate 1087 * to sw_reg_strategy(). 1088 */ 1089 1090 switch (sdp->swd_vp->v_type) { 1091 default: 1092 panic("swstrategy: vnode type 0x%x", sdp->swd_vp->v_type); 1093 1094 case VBLK: 1095 1096 /* 1097 * must convert "bp" from an I/O on /dev/drum to an I/O 1098 * on the swapdev (sdp). 1099 */ 1100 s = splbio(); 1101 bp->b_blkno = bn; /* swapdev block number */ 1102 vp = sdp->swd_vp; /* swapdev vnode pointer */ 1103 bp->b_dev = sdp->swd_dev; /* swapdev dev_t */ 1104 1105 /* 1106 * if we are doing a write, we have to redirect the i/o on 1107 * drum's v_numoutput counter to the swapdevs. 1108 */ 1109 if ((bp->b_flags & B_READ) == 0) { 1110 vwakeup(bp); /* kills one 'v_numoutput' on drum */ 1111 vp->v_numoutput++; /* put it on swapdev */ 1112 } 1113 1114 /* 1115 * finally plug in swapdev vnode and start I/O 1116 */ 1117 bp->b_vp = vp; 1118 splx(s); 1119 VOP_STRATEGY(bp); 1120 return; 1121 1122 case VREG: 1123 /* 1124 * delegate to sw_reg_strategy function. 1125 */ 1126 sw_reg_strategy(sdp, bp, bn); 1127 return; 1128 } 1129 /* NOTREACHED */ 1130 } 1131 1132 /* 1133 * sw_reg_strategy: handle swap i/o to regular files 1134 */ 1135 static void 1136 sw_reg_strategy(sdp, bp, bn) 1137 struct swapdev *sdp; 1138 struct buf *bp; 1139 int bn; 1140 { 1141 struct vnode *vp; 1142 struct vndxfer *vnx; 1143 daddr_t nbn; 1144 caddr_t addr; 1145 off_t byteoff; 1146 int s, off, nra, error, sz, resid; 1147 UVMHIST_FUNC("sw_reg_strategy"); UVMHIST_CALLED(pdhist); 1148 1149 /* 1150 * allocate a vndxfer head for this transfer and point it to 1151 * our buffer. 1152 */ 1153 getvndxfer(vnx); 1154 vnx->vx_flags = VX_BUSY; 1155 vnx->vx_error = 0; 1156 vnx->vx_pending = 0; 1157 vnx->vx_bp = bp; 1158 vnx->vx_sdp = sdp; 1159 1160 /* 1161 * setup for main loop where we read filesystem blocks into 1162 * our buffer. 1163 */ 1164 error = 0; 1165 bp->b_resid = bp->b_bcount; /* nothing transfered yet! */ 1166 addr = bp->b_data; /* current position in buffer */ 1167 byteoff = dbtob((u_int64_t)bn); 1168 1169 for (resid = bp->b_resid; resid; resid -= sz) { 1170 struct vndbuf *nbp; 1171 1172 /* 1173 * translate byteoffset into block number. return values: 1174 * vp = vnode of underlying device 1175 * nbn = new block number (on underlying vnode dev) 1176 * nra = num blocks we can read-ahead (excludes requested 1177 * block) 1178 */ 1179 nra = 0; 1180 error = VOP_BMAP(sdp->swd_vp, byteoff / sdp->swd_bsize, 1181 &vp, &nbn, &nra); 1182 1183 if (error == 0 && nbn == (daddr_t)-1) { 1184 /* 1185 * this used to just set error, but that doesn't 1186 * do the right thing. Instead, it causes random 1187 * memory errors. The panic() should remain until 1188 * this condition doesn't destabilize the system. 1189 */ 1190 #if 1 1191 panic("sw_reg_strategy: swap to sparse file"); 1192 #else 1193 error = EIO; /* failure */ 1194 #endif 1195 } 1196 1197 /* 1198 * punt if there was an error or a hole in the file. 1199 * we must wait for any i/o ops we have already started 1200 * to finish before returning. 1201 * 1202 * XXX we could deal with holes here but it would be 1203 * a hassle (in the write case). 1204 */ 1205 if (error) { 1206 s = splbio(); 1207 vnx->vx_error = error; /* pass error up */ 1208 goto out; 1209 } 1210 1211 /* 1212 * compute the size ("sz") of this transfer (in bytes). 1213 */ 1214 off = byteoff % sdp->swd_bsize; 1215 sz = (1 + nra) * sdp->swd_bsize - off; 1216 if (sz > resid) 1217 sz = resid; 1218 1219 UVMHIST_LOG(pdhist, "sw_reg_strategy: " 1220 "vp %p/%p offset 0x%x/0x%x", 1221 sdp->swd_vp, vp, byteoff, nbn); 1222 1223 /* 1224 * now get a buf structure. note that the vb_buf is 1225 * at the front of the nbp structure so that you can 1226 * cast pointers between the two structure easily. 1227 */ 1228 getvndbuf(nbp); 1229 nbp->vb_buf.b_flags = bp->b_flags | B_CALL; 1230 nbp->vb_buf.b_bcount = sz; 1231 nbp->vb_buf.b_bufsize = sz; 1232 nbp->vb_buf.b_error = 0; 1233 nbp->vb_buf.b_data = addr; 1234 nbp->vb_buf.b_lblkno = 0; 1235 nbp->vb_buf.b_blkno = nbn + btodb(off); 1236 nbp->vb_buf.b_rawblkno = nbp->vb_buf.b_blkno; 1237 nbp->vb_buf.b_iodone = sw_reg_iodone; 1238 nbp->vb_buf.b_vp = vp; 1239 if (vp->v_type == VBLK) { 1240 nbp->vb_buf.b_dev = vp->v_rdev; 1241 } 1242 LIST_INIT(&nbp->vb_buf.b_dep); 1243 1244 nbp->vb_xfer = vnx; /* patch it back in to vnx */ 1245 1246 /* 1247 * Just sort by block number 1248 */ 1249 s = splbio(); 1250 if (vnx->vx_error != 0) { 1251 putvndbuf(nbp); 1252 goto out; 1253 } 1254 vnx->vx_pending++; 1255 1256 /* sort it in and start I/O if we are not over our limit */ 1257 disksort_blkno(&sdp->swd_tab, &nbp->vb_buf); 1258 sw_reg_start(sdp); 1259 splx(s); 1260 1261 /* 1262 * advance to the next I/O 1263 */ 1264 byteoff += sz; 1265 addr += sz; 1266 } 1267 1268 s = splbio(); 1269 1270 out: /* Arrive here at splbio */ 1271 vnx->vx_flags &= ~VX_BUSY; 1272 if (vnx->vx_pending == 0) { 1273 if (vnx->vx_error != 0) { 1274 bp->b_error = vnx->vx_error; 1275 bp->b_flags |= B_ERROR; 1276 } 1277 putvndxfer(vnx); 1278 biodone(bp); 1279 } 1280 splx(s); 1281 } 1282 1283 /* 1284 * sw_reg_start: start an I/O request on the requested swapdev 1285 * 1286 * => reqs are sorted by disksort (above) 1287 */ 1288 static void 1289 sw_reg_start(sdp) 1290 struct swapdev *sdp; 1291 { 1292 struct buf *bp; 1293 UVMHIST_FUNC("sw_reg_start"); UVMHIST_CALLED(pdhist); 1294 1295 /* recursion control */ 1296 if ((sdp->swd_flags & SWF_BUSY) != 0) 1297 return; 1298 1299 sdp->swd_flags |= SWF_BUSY; 1300 1301 while (sdp->swd_active < sdp->swd_maxactive) { 1302 bp = BUFQ_FIRST(&sdp->swd_tab); 1303 if (bp == NULL) 1304 break; 1305 BUFQ_REMOVE(&sdp->swd_tab, bp); 1306 sdp->swd_active++; 1307 1308 UVMHIST_LOG(pdhist, 1309 "sw_reg_start: bp %p vp %p blkno %p cnt %lx", 1310 bp, bp->b_vp, bp->b_blkno, bp->b_bcount); 1311 if ((bp->b_flags & B_READ) == 0) 1312 bp->b_vp->v_numoutput++; 1313 1314 VOP_STRATEGY(bp); 1315 } 1316 sdp->swd_flags &= ~SWF_BUSY; 1317 } 1318 1319 /* 1320 * sw_reg_iodone: one of our i/o's has completed and needs post-i/o cleanup 1321 * 1322 * => note that we can recover the vndbuf struct by casting the buf ptr 1323 */ 1324 static void 1325 sw_reg_iodone(bp) 1326 struct buf *bp; 1327 { 1328 struct vndbuf *vbp = (struct vndbuf *) bp; 1329 struct vndxfer *vnx = vbp->vb_xfer; 1330 struct buf *pbp = vnx->vx_bp; /* parent buffer */ 1331 struct swapdev *sdp = vnx->vx_sdp; 1332 int s, resid; 1333 UVMHIST_FUNC("sw_reg_iodone"); UVMHIST_CALLED(pdhist); 1334 1335 UVMHIST_LOG(pdhist, " vbp=%p vp=%p blkno=%x addr=%p", 1336 vbp, vbp->vb_buf.b_vp, vbp->vb_buf.b_blkno, vbp->vb_buf.b_data); 1337 UVMHIST_LOG(pdhist, " cnt=%lx resid=%lx", 1338 vbp->vb_buf.b_bcount, vbp->vb_buf.b_resid, 0, 0); 1339 1340 /* 1341 * protect vbp at splbio and update. 1342 */ 1343 1344 s = splbio(); 1345 resid = vbp->vb_buf.b_bcount - vbp->vb_buf.b_resid; 1346 pbp->b_resid -= resid; 1347 vnx->vx_pending--; 1348 1349 if (vbp->vb_buf.b_error) { 1350 UVMHIST_LOG(pdhist, " got error=%d !", 1351 vbp->vb_buf.b_error, 0, 0, 0); 1352 1353 /* pass error upward */ 1354 vnx->vx_error = vbp->vb_buf.b_error; 1355 } 1356 1357 /* 1358 * kill vbp structure 1359 */ 1360 putvndbuf(vbp); 1361 1362 /* 1363 * wrap up this transaction if it has run to completion or, in 1364 * case of an error, when all auxiliary buffers have returned. 1365 */ 1366 if (vnx->vx_error != 0) { 1367 /* pass error upward */ 1368 pbp->b_flags |= B_ERROR; 1369 pbp->b_error = vnx->vx_error; 1370 if ((vnx->vx_flags & VX_BUSY) == 0 && vnx->vx_pending == 0) { 1371 putvndxfer(vnx); 1372 biodone(pbp); 1373 } 1374 } else if (pbp->b_resid == 0) { 1375 KASSERT(vnx->vx_pending == 0); 1376 if ((vnx->vx_flags & VX_BUSY) == 0) { 1377 UVMHIST_LOG(pdhist, " iodone error=%d !", 1378 pbp, vnx->vx_error, 0, 0); 1379 putvndxfer(vnx); 1380 biodone(pbp); 1381 } 1382 } 1383 1384 /* 1385 * done! start next swapdev I/O if one is pending 1386 */ 1387 sdp->swd_active--; 1388 sw_reg_start(sdp); 1389 splx(s); 1390 } 1391 1392 1393 /* 1394 * uvm_swap_alloc: allocate space on swap 1395 * 1396 * => allocation is done "round robin" down the priority list, as we 1397 * allocate in a priority we "rotate" the circle queue. 1398 * => space can be freed with uvm_swap_free 1399 * => we return the page slot number in /dev/drum (0 == invalid slot) 1400 * => we lock uvm.swap_data_lock 1401 * => XXXMRG: "LESSOK" INTERFACE NEEDED TO EXTENT SYSTEM 1402 */ 1403 int 1404 uvm_swap_alloc(nslots, lessok) 1405 int *nslots; /* IN/OUT */ 1406 boolean_t lessok; 1407 { 1408 struct swapdev *sdp; 1409 struct swappri *spp; 1410 u_long result; 1411 UVMHIST_FUNC("uvm_swap_alloc"); UVMHIST_CALLED(pdhist); 1412 1413 /* 1414 * no swap devices configured yet? definite failure. 1415 */ 1416 if (uvmexp.nswapdev < 1) 1417 return 0; 1418 1419 /* 1420 * lock data lock, convert slots into blocks, and enter loop 1421 */ 1422 simple_lock(&uvm.swap_data_lock); 1423 1424 ReTry: /* XXXMRG */ 1425 LIST_FOREACH(spp, &swap_priority, spi_swappri) { 1426 CIRCLEQ_FOREACH(sdp, &spp->spi_swapdev, swd_next) { 1427 /* if it's not enabled, then we can't swap from it */ 1428 if ((sdp->swd_flags & SWF_ENABLE) == 0) 1429 continue; 1430 if (sdp->swd_npginuse + *nslots > sdp->swd_npages) 1431 continue; 1432 if (extent_alloc(sdp->swd_ex, *nslots, EX_NOALIGN, 1433 EX_NOBOUNDARY, EX_MALLOCOK|EX_NOWAIT, 1434 &result) != 0) { 1435 continue; 1436 } 1437 1438 /* 1439 * successful allocation! now rotate the circleq. 1440 */ 1441 CIRCLEQ_REMOVE(&spp->spi_swapdev, sdp, swd_next); 1442 CIRCLEQ_INSERT_TAIL(&spp->spi_swapdev, sdp, swd_next); 1443 sdp->swd_npginuse += *nslots; 1444 uvmexp.swpginuse += *nslots; 1445 simple_unlock(&uvm.swap_data_lock); 1446 /* done! return drum slot number */ 1447 UVMHIST_LOG(pdhist, 1448 "success! returning %d slots starting at %d", 1449 *nslots, result + sdp->swd_drumoffset, 0, 0); 1450 return (result + sdp->swd_drumoffset); 1451 } 1452 } 1453 1454 /* XXXMRG: BEGIN HACK */ 1455 if (*nslots > 1 && lessok) { 1456 *nslots = 1; 1457 goto ReTry; /* XXXMRG: ugh! extent should support this for us */ 1458 } 1459 /* XXXMRG: END HACK */ 1460 1461 simple_unlock(&uvm.swap_data_lock); 1462 return 0; 1463 } 1464 1465 /* 1466 * uvm_swap_markbad: keep track of swap ranges where we've had i/o errors 1467 * 1468 * => we lock uvm.swap_data_lock 1469 */ 1470 void 1471 uvm_swap_markbad(startslot, nslots) 1472 int startslot; 1473 int nslots; 1474 { 1475 struct swapdev *sdp; 1476 UVMHIST_FUNC("uvm_swap_markbad"); UVMHIST_CALLED(pdhist); 1477 1478 simple_lock(&uvm.swap_data_lock); 1479 sdp = swapdrum_getsdp(startslot); 1480 1481 /* 1482 * we just keep track of how many pages have been marked bad 1483 * in this device, to make everything add up in swap_off(). 1484 * we assume here that the range of slots will all be within 1485 * one swap device. 1486 */ 1487 1488 sdp->swd_npgbad += nslots; 1489 UVMHIST_LOG(pdhist, "now %d bad", sdp->swd_npgbad, 0,0,0); 1490 simple_unlock(&uvm.swap_data_lock); 1491 } 1492 1493 /* 1494 * uvm_swap_free: free swap slots 1495 * 1496 * => this can be all or part of an allocation made by uvm_swap_alloc 1497 * => we lock uvm.swap_data_lock 1498 */ 1499 void 1500 uvm_swap_free(startslot, nslots) 1501 int startslot; 1502 int nslots; 1503 { 1504 struct swapdev *sdp; 1505 UVMHIST_FUNC("uvm_swap_free"); UVMHIST_CALLED(pdhist); 1506 1507 UVMHIST_LOG(pdhist, "freeing %d slots starting at %d", nslots, 1508 startslot, 0, 0); 1509 1510 /* 1511 * ignore attempts to free the "bad" slot. 1512 */ 1513 1514 if (startslot == SWSLOT_BAD) { 1515 return; 1516 } 1517 1518 /* 1519 * convert drum slot offset back to sdp, free the blocks 1520 * in the extent, and return. must hold pri lock to do 1521 * lookup and access the extent. 1522 */ 1523 1524 simple_lock(&uvm.swap_data_lock); 1525 sdp = swapdrum_getsdp(startslot); 1526 KASSERT(uvmexp.nswapdev >= 1); 1527 KASSERT(sdp != NULL); 1528 KASSERT(sdp->swd_npginuse >= nslots); 1529 if (extent_free(sdp->swd_ex, startslot - sdp->swd_drumoffset, nslots, 1530 EX_MALLOCOK|EX_NOWAIT) != 0) { 1531 printf("warning: resource shortage: %d pages of swap lost\n", 1532 nslots); 1533 } 1534 sdp->swd_npginuse -= nslots; 1535 uvmexp.swpginuse -= nslots; 1536 simple_unlock(&uvm.swap_data_lock); 1537 } 1538 1539 /* 1540 * uvm_swap_put: put any number of pages into a contig place on swap 1541 * 1542 * => can be sync or async 1543 */ 1544 1545 int 1546 uvm_swap_put(swslot, ppsp, npages, flags) 1547 int swslot; 1548 struct vm_page **ppsp; 1549 int npages; 1550 int flags; 1551 { 1552 int error; 1553 1554 error = uvm_swap_io(ppsp, swslot, npages, B_WRITE | 1555 ((flags & PGO_SYNCIO) ? 0 : B_ASYNC)); 1556 return error; 1557 } 1558 1559 /* 1560 * uvm_swap_get: get a single page from swap 1561 * 1562 * => usually a sync op (from fault) 1563 */ 1564 1565 int 1566 uvm_swap_get(page, swslot, flags) 1567 struct vm_page *page; 1568 int swslot, flags; 1569 { 1570 int error; 1571 1572 uvmexp.nswget++; 1573 KASSERT(flags & PGO_SYNCIO); 1574 if (swslot == SWSLOT_BAD) { 1575 return EIO; 1576 } 1577 error = uvm_swap_io(&page, swslot, 1, B_READ | 1578 ((flags & PGO_SYNCIO) ? 0 : B_ASYNC)); 1579 if (error == 0) { 1580 1581 /* 1582 * this page is no longer only in swap. 1583 */ 1584 1585 simple_lock(&uvm.swap_data_lock); 1586 KASSERT(uvmexp.swpgonly > 0); 1587 uvmexp.swpgonly--; 1588 simple_unlock(&uvm.swap_data_lock); 1589 } 1590 return error; 1591 } 1592 1593 /* 1594 * uvm_swap_io: do an i/o operation to swap 1595 */ 1596 1597 static int 1598 uvm_swap_io(pps, startslot, npages, flags) 1599 struct vm_page **pps; 1600 int startslot, npages, flags; 1601 { 1602 daddr_t startblk; 1603 struct buf *bp; 1604 vaddr_t kva; 1605 int error, s, mapinflags; 1606 boolean_t write, async; 1607 UVMHIST_FUNC("uvm_swap_io"); UVMHIST_CALLED(pdhist); 1608 1609 UVMHIST_LOG(pdhist, "<- called, startslot=%d, npages=%d, flags=%d", 1610 startslot, npages, flags, 0); 1611 1612 write = (flags & B_READ) == 0; 1613 async = (flags & B_ASYNC) != 0; 1614 1615 /* 1616 * convert starting drum slot to block number 1617 */ 1618 1619 startblk = btodb((u_int64_t)startslot << PAGE_SHIFT); 1620 1621 /* 1622 * first, map the pages into the kernel. 1623 */ 1624 1625 mapinflags = !write ? 1626 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_READ : 1627 UVMPAGER_MAPIN_WAITOK|UVMPAGER_MAPIN_WRITE; 1628 kva = uvm_pagermapin(pps, npages, mapinflags); 1629 1630 /* 1631 * now allocate a buf for the i/o. 1632 */ 1633 1634 s = splbio(); 1635 bp = pool_get(&bufpool, PR_WAITOK); 1636 splx(s); 1637 1638 /* 1639 * fill in the bp/sbp. we currently route our i/o through 1640 * /dev/drum's vnode [swapdev_vp]. 1641 */ 1642 1643 bp->b_flags = B_BUSY | B_NOCACHE | (flags & (B_READ|B_ASYNC)); 1644 bp->b_proc = &proc0; /* XXX */ 1645 bp->b_vnbufs.le_next = NOLIST; 1646 bp->b_data = (caddr_t)kva; 1647 bp->b_blkno = startblk; 1648 bp->b_vp = swapdev_vp; 1649 bp->b_dev = swapdev_vp->v_rdev; 1650 bp->b_bufsize = bp->b_bcount = npages << PAGE_SHIFT; 1651 LIST_INIT(&bp->b_dep); 1652 1653 /* 1654 * bump v_numoutput (counter of number of active outputs). 1655 */ 1656 1657 if (write) { 1658 s = splbio(); 1659 swapdev_vp->v_numoutput++; 1660 splx(s); 1661 } 1662 1663 /* 1664 * for async ops we must set up the iodone handler. 1665 */ 1666 1667 if (async) { 1668 bp->b_flags |= B_CALL; 1669 bp->b_iodone = uvm_aio_biodone; 1670 UVMHIST_LOG(pdhist, "doing async!", 0, 0, 0, 0); 1671 } 1672 UVMHIST_LOG(pdhist, 1673 "about to start io: data = %p blkno = 0x%x, bcount = %ld", 1674 bp->b_data, bp->b_blkno, bp->b_bcount, 0); 1675 1676 /* 1677 * now we start the I/O, and if async, return. 1678 */ 1679 1680 VOP_STRATEGY(bp); 1681 if (async) 1682 return 0; 1683 1684 /* 1685 * must be sync i/o. wait for it to finish 1686 */ 1687 1688 error = biowait(bp); 1689 1690 /* 1691 * kill the pager mapping 1692 */ 1693 1694 uvm_pagermapout(kva, npages); 1695 1696 /* 1697 * now dispose of the buf and we're done. 1698 */ 1699 1700 s = splbio(); 1701 if (write) 1702 vwakeup(bp); 1703 pool_put(&bufpool, bp); 1704 splx(s); 1705 UVMHIST_LOG(pdhist, "<- done (sync) error=%d", error, 0, 0, 0); 1706 return (error); 1707 } 1708