1 /* $NetBSD: vfs_bio.c,v 1.147 2005/12/11 12:24:30 christos Exp $ */ 2 3 /*- 4 * Copyright (c) 1982, 1986, 1989, 1993 5 * The Regents of the University of California. All rights reserved. 6 * (c) UNIX System Laboratories, Inc. 7 * All or some portions of this file are derived from material licensed 8 * to the University of California by American Telephone and Telegraph 9 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 10 * the permission of UNIX System Laboratories, Inc. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)vfs_bio.c 8.6 (Berkeley) 1/11/94 37 */ 38 39 /*- 40 * Copyright (c) 1994 Christopher G. Demetriou 41 * 42 * Redistribution and use in source and binary forms, with or without 43 * modification, are permitted provided that the following conditions 44 * are met: 45 * 1. Redistributions of source code must retain the above copyright 46 * notice, this list of conditions and the following disclaimer. 47 * 2. Redistributions in binary form must reproduce the above copyright 48 * notice, this list of conditions and the following disclaimer in the 49 * documentation and/or other materials provided with the distribution. 50 * 3. All advertising materials mentioning features or use of this software 51 * must display the following acknowledgement: 52 * This product includes software developed by the University of 53 * California, Berkeley and its contributors. 54 * 4. Neither the name of the University nor the names of its contributors 55 * may be used to endorse or promote products derived from this software 56 * without specific prior written permission. 57 * 58 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 59 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 60 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 61 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 62 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 63 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 64 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 65 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 66 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 67 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 68 * SUCH DAMAGE. 69 * 70 * @(#)vfs_bio.c 8.6 (Berkeley) 1/11/94 71 */ 72 73 /* 74 * Some references: 75 * Bach: The Design of the UNIX Operating System (Prentice Hall, 1986) 76 * Leffler, et al.: The Design and Implementation of the 4.3BSD 77 * UNIX Operating System (Addison Welley, 1989) 78 */ 79 80 #include "opt_bufcache.h" 81 #include "opt_softdep.h" 82 83 #include <sys/cdefs.h> 84 __KERNEL_RCSID(0, "$NetBSD: vfs_bio.c,v 1.147 2005/12/11 12:24:30 christos Exp $"); 85 86 #include <sys/param.h> 87 #include <sys/systm.h> 88 #include <sys/kernel.h> 89 #include <sys/proc.h> 90 #include <sys/buf.h> 91 #include <sys/vnode.h> 92 #include <sys/mount.h> 93 #include <sys/malloc.h> 94 #include <sys/resourcevar.h> 95 #include <sys/sysctl.h> 96 #include <sys/conf.h> 97 98 #include <uvm/uvm.h> 99 100 #include <miscfs/specfs/specdev.h> 101 102 #ifndef BUFPAGES 103 # define BUFPAGES 0 104 #endif 105 106 #ifdef BUFCACHE 107 # if (BUFCACHE < 5) || (BUFCACHE > 95) 108 # error BUFCACHE is not between 5 and 95 109 # endif 110 #else 111 # define BUFCACHE 15 112 #endif 113 114 u_int nbuf; /* XXX - for softdep_lockedbufs */ 115 u_int bufpages = BUFPAGES; /* optional hardwired count */ 116 u_int bufcache = BUFCACHE; /* max % of RAM to use for buffer cache */ 117 118 /* Function prototypes */ 119 struct bqueue; 120 121 static void buf_setwm(void); 122 static int buf_trim(void); 123 static void *bufpool_page_alloc(struct pool *, int); 124 static void bufpool_page_free(struct pool *, void *); 125 static __inline struct buf *bio_doread(struct vnode *, daddr_t, int, 126 struct ucred *, int); 127 static int buf_lotsfree(void); 128 static int buf_canrelease(void); 129 static __inline u_long buf_mempoolidx(u_long); 130 static __inline u_long buf_roundsize(u_long); 131 static __inline caddr_t buf_malloc(size_t); 132 static void buf_mrelease(caddr_t, size_t); 133 static __inline void binsheadfree(struct buf *, struct bqueue *); 134 static __inline void binstailfree(struct buf *, struct bqueue *); 135 int count_lock_queue(void); /* XXX */ 136 #ifdef DEBUG 137 static int checkfreelist(struct buf *, struct bqueue *); 138 #endif 139 140 /* Macros to clear/set/test flags. */ 141 #define SET(t, f) (t) |= (f) 142 #define CLR(t, f) (t) &= ~(f) 143 #define ISSET(t, f) ((t) & (f)) 144 145 /* 146 * Definitions for the buffer hash lists. 147 */ 148 #define BUFHASH(dvp, lbn) \ 149 (&bufhashtbl[(((long)(dvp) >> 8) + (int)(lbn)) & bufhash]) 150 LIST_HEAD(bufhashhdr, buf) *bufhashtbl, invalhash; 151 u_long bufhash; 152 #if !defined(SOFTDEP) || !defined(FFS) 153 struct bio_ops bioops; /* I/O operation notification */ 154 #endif 155 156 /* 157 * Insq/Remq for the buffer hash lists. 158 */ 159 #define binshash(bp, dp) LIST_INSERT_HEAD(dp, bp, b_hash) 160 #define bremhash(bp) LIST_REMOVE(bp, b_hash) 161 162 /* 163 * Definitions for the buffer free lists. 164 */ 165 #define BQUEUES 3 /* number of free buffer queues */ 166 167 #define BQ_LOCKED 0 /* super-blocks &c */ 168 #define BQ_LRU 1 /* lru, useful buffers */ 169 #define BQ_AGE 2 /* rubbish */ 170 171 struct bqueue { 172 TAILQ_HEAD(, buf) bq_queue; 173 uint64_t bq_bytes; 174 } bufqueues[BQUEUES]; 175 int needbuffer; 176 177 /* 178 * Buffer queue lock. 179 * Take this lock first if also taking some buffer's b_interlock. 180 */ 181 struct simplelock bqueue_slock = SIMPLELOCK_INITIALIZER; 182 183 /* 184 * Buffer pool for I/O buffers. 185 */ 186 struct pool bufpool; 187 188 /* XXX - somewhat gross.. */ 189 #if MAXBSIZE == 0x2000 190 #define NMEMPOOLS 4 191 #elif MAXBSIZE == 0x4000 192 #define NMEMPOOLS 5 193 #elif MAXBSIZE == 0x8000 194 #define NMEMPOOLS 6 195 #else 196 #define NMEMPOOLS 7 197 #endif 198 199 #define MEMPOOL_INDEX_OFFSET 10 /* smallest pool is 1k */ 200 #if (1 << (NMEMPOOLS + MEMPOOL_INDEX_OFFSET - 1)) != MAXBSIZE 201 #error update vfs_bio buffer memory parameters 202 #endif 203 204 /* Buffer memory pools */ 205 static struct pool bmempools[NMEMPOOLS]; 206 207 struct vm_map *buf_map; 208 209 /* 210 * Buffer memory pool allocator. 211 */ 212 static void * 213 bufpool_page_alloc(struct pool *pp, int flags) 214 { 215 216 return (void *)uvm_km_alloc(buf_map, 217 MAXBSIZE, MAXBSIZE, 218 ((flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT | UVM_KMF_TRYLOCK) 219 | UVM_KMF_WIRED); 220 } 221 222 static void 223 bufpool_page_free(struct pool *pp, void *v) 224 { 225 226 uvm_km_free(buf_map, (vaddr_t)v, MAXBSIZE, UVM_KMF_WIRED); 227 } 228 229 static struct pool_allocator bufmempool_allocator = { 230 bufpool_page_alloc, bufpool_page_free, MAXBSIZE, 231 }; 232 233 /* Buffer memory management variables */ 234 u_long bufmem_valimit; 235 u_long bufmem_hiwater; 236 u_long bufmem_lowater; 237 u_long bufmem; 238 239 /* 240 * MD code can call this to set a hard limit on the amount 241 * of virtual memory used by the buffer cache. 242 */ 243 int 244 buf_setvalimit(vsize_t sz) 245 { 246 247 /* We need to accommodate at least NMEMPOOLS of MAXBSIZE each */ 248 if (sz < NMEMPOOLS * MAXBSIZE) 249 return EINVAL; 250 251 bufmem_valimit = sz; 252 return 0; 253 } 254 255 static void 256 buf_setwm(void) 257 { 258 259 bufmem_hiwater = buf_memcalc(); 260 /* lowater is approx. 2% of memory (with bufcache = 15) */ 261 #define BUFMEM_WMSHIFT 3 262 #define BUFMEM_HIWMMIN (64 * 1024 << BUFMEM_WMSHIFT) 263 if (bufmem_hiwater < BUFMEM_HIWMMIN) 264 /* Ensure a reasonable minimum value */ 265 bufmem_hiwater = BUFMEM_HIWMMIN; 266 bufmem_lowater = bufmem_hiwater >> BUFMEM_WMSHIFT; 267 } 268 269 #ifdef DEBUG 270 int debug_verify_freelist = 0; 271 static int 272 checkfreelist(struct buf *bp, struct bqueue *dp) 273 { 274 struct buf *b; 275 276 TAILQ_FOREACH(b, &dp->bq_queue, b_freelist) { 277 if (b == bp) 278 return 1; 279 } 280 return 0; 281 } 282 #endif 283 284 /* 285 * Insq/Remq for the buffer hash lists. 286 * Call with buffer queue locked. 287 */ 288 static __inline void 289 binsheadfree(struct buf *bp, struct bqueue *dp) 290 { 291 292 KASSERT(bp->b_freelistindex == -1); 293 TAILQ_INSERT_HEAD(&dp->bq_queue, bp, b_freelist); 294 dp->bq_bytes += bp->b_bufsize; 295 bp->b_freelistindex = dp - bufqueues; 296 } 297 298 static __inline void 299 binstailfree(struct buf *bp, struct bqueue *dp) 300 { 301 302 KASSERT(bp->b_freelistindex == -1); 303 TAILQ_INSERT_TAIL(&dp->bq_queue, bp, b_freelist); 304 dp->bq_bytes += bp->b_bufsize; 305 bp->b_freelistindex = dp - bufqueues; 306 } 307 308 void 309 bremfree(struct buf *bp) 310 { 311 struct bqueue *dp; 312 int bqidx = bp->b_freelistindex; 313 314 LOCK_ASSERT(simple_lock_held(&bqueue_slock)); 315 316 KASSERT(bqidx != -1); 317 dp = &bufqueues[bqidx]; 318 KDASSERT(!debug_verify_freelist || checkfreelist(bp, dp)); 319 KASSERT(dp->bq_bytes >= bp->b_bufsize); 320 TAILQ_REMOVE(&dp->bq_queue, bp, b_freelist); 321 dp->bq_bytes -= bp->b_bufsize; 322 #if defined(DIAGNOSTIC) 323 bp->b_freelistindex = -1; 324 #endif /* defined(DIAGNOSTIC) */ 325 } 326 327 u_long 328 buf_memcalc(void) 329 { 330 u_long n; 331 332 /* 333 * Determine the upper bound of memory to use for buffers. 334 * 335 * - If bufpages is specified, use that as the number 336 * pages. 337 * 338 * - Otherwise, use bufcache as the percentage of 339 * physical memory. 340 */ 341 if (bufpages != 0) { 342 n = bufpages; 343 } else { 344 if (bufcache < 5) { 345 printf("forcing bufcache %d -> 5", bufcache); 346 bufcache = 5; 347 } 348 if (bufcache > 95) { 349 printf("forcing bufcache %d -> 95", bufcache); 350 bufcache = 95; 351 } 352 n = physmem / 100 * bufcache; 353 } 354 355 n <<= PAGE_SHIFT; 356 if (bufmem_valimit != 0 && n > bufmem_valimit) 357 n = bufmem_valimit; 358 359 return (n); 360 } 361 362 /* 363 * Initialize buffers and hash links for buffers. 364 */ 365 void 366 bufinit(void) 367 { 368 struct bqueue *dp; 369 int use_std; 370 u_int i; 371 372 /* 373 * Initialize buffer cache memory parameters. 374 */ 375 bufmem = 0; 376 buf_setwm(); 377 378 if (bufmem_valimit != 0) { 379 vaddr_t minaddr = 0, maxaddr; 380 buf_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, 381 bufmem_valimit, VM_MAP_PAGEABLE, 382 FALSE, 0); 383 if (buf_map == NULL) 384 panic("bufinit: cannot allocate submap"); 385 } else 386 buf_map = kernel_map; 387 388 /* 389 * Initialize the buffer pools. 390 */ 391 pool_init(&bufpool, sizeof(struct buf), 0, 0, 0, "bufpl", NULL); 392 393 /* On "small" machines use small pool page sizes where possible */ 394 use_std = (physmem < atop(16*1024*1024)); 395 396 /* 397 * Also use them on systems that can map the pool pages using 398 * a direct-mapped segment. 399 */ 400 #ifdef PMAP_MAP_POOLPAGE 401 use_std = 1; 402 #endif 403 404 for (i = 0; i < NMEMPOOLS; i++) { 405 struct pool_allocator *pa; 406 struct pool *pp = &bmempools[i]; 407 u_int size = 1 << (i + MEMPOOL_INDEX_OFFSET); 408 char *name = malloc(8, M_TEMP, M_WAITOK); 409 snprintf(name, 8, "buf%dk", 1 << i); 410 pa = (size <= PAGE_SIZE && use_std) 411 ? &pool_allocator_nointr 412 : &bufmempool_allocator; 413 pool_init(pp, size, 0, 0, 0, name, pa); 414 pool_setlowat(pp, 1); 415 pool_sethiwat(pp, 1); 416 } 417 418 /* Initialize the buffer queues */ 419 for (dp = bufqueues; dp < &bufqueues[BQUEUES]; dp++) { 420 TAILQ_INIT(&dp->bq_queue); 421 dp->bq_bytes = 0; 422 } 423 424 /* 425 * Estimate hash table size based on the amount of memory we 426 * intend to use for the buffer cache. The average buffer 427 * size is dependent on our clients (i.e. filesystems). 428 * 429 * For now, use an empirical 3K per buffer. 430 */ 431 nbuf = (bufmem_hiwater / 1024) / 3; 432 bufhashtbl = hashinit(nbuf, HASH_LIST, M_CACHE, M_WAITOK, &bufhash); 433 } 434 435 static int 436 buf_lotsfree(void) 437 { 438 int try, thresh; 439 struct lwp *l = curlwp; 440 441 /* Always allocate if doing copy on write */ 442 if (l->l_flag & L_COWINPROGRESS) 443 return 1; 444 445 /* Always allocate if less than the low water mark. */ 446 if (bufmem < bufmem_lowater) 447 return 1; 448 449 /* Never allocate if greater than the high water mark. */ 450 if (bufmem > bufmem_hiwater) 451 return 0; 452 453 /* If there's anything on the AGE list, it should be eaten. */ 454 if (TAILQ_FIRST(&bufqueues[BQ_AGE].bq_queue) != NULL) 455 return 0; 456 457 /* 458 * The probabily of getting a new allocation is inversely 459 * proportional to the current size of the cache, using 460 * a granularity of 16 steps. 461 */ 462 try = random() & 0x0000000fL; 463 464 /* Don't use "16 * bufmem" here to avoid a 32-bit overflow. */ 465 thresh = (bufmem - bufmem_lowater) / 466 ((bufmem_hiwater - bufmem_lowater) / 16); 467 468 if (try >= thresh) 469 return 1; 470 471 /* Otherwise don't allocate. */ 472 return 0; 473 } 474 475 /* 476 * Return estimate of bytes we think need to be 477 * released to help resolve low memory conditions. 478 * 479 * => called at splbio. 480 * => called with bqueue_slock held. 481 */ 482 static int 483 buf_canrelease(void) 484 { 485 int pagedemand, ninvalid = 0; 486 487 LOCK_ASSERT(simple_lock_held(&bqueue_slock)); 488 489 if (bufmem < bufmem_lowater) 490 return 0; 491 492 if (bufmem > bufmem_hiwater) 493 return bufmem - bufmem_hiwater; 494 495 ninvalid += bufqueues[BQ_AGE].bq_bytes; 496 497 pagedemand = uvmexp.freetarg - uvmexp.free; 498 if (pagedemand < 0) 499 return ninvalid; 500 return MAX(ninvalid, MIN(2 * MAXBSIZE, 501 MIN((bufmem - bufmem_lowater) / 16, pagedemand * PAGE_SIZE))); 502 } 503 504 /* 505 * Buffer memory allocation helper functions 506 */ 507 static __inline u_long 508 buf_mempoolidx(u_long size) 509 { 510 u_int n = 0; 511 512 size -= 1; 513 size >>= MEMPOOL_INDEX_OFFSET; 514 while (size) { 515 size >>= 1; 516 n += 1; 517 } 518 if (n >= NMEMPOOLS) 519 panic("buf mem pool index %d", n); 520 return n; 521 } 522 523 static __inline u_long 524 buf_roundsize(u_long size) 525 { 526 /* Round up to nearest power of 2 */ 527 return (1 << (buf_mempoolidx(size) + MEMPOOL_INDEX_OFFSET)); 528 } 529 530 static __inline caddr_t 531 buf_malloc(size_t size) 532 { 533 u_int n = buf_mempoolidx(size); 534 caddr_t addr; 535 int s; 536 537 while (1) { 538 addr = pool_get(&bmempools[n], PR_NOWAIT); 539 if (addr != NULL) 540 break; 541 542 /* No memory, see if we can free some. If so, try again */ 543 if (buf_drain(1) > 0) 544 continue; 545 546 /* Wait for buffers to arrive on the LRU queue */ 547 s = splbio(); 548 simple_lock(&bqueue_slock); 549 needbuffer = 1; 550 ltsleep(&needbuffer, PNORELOCK | (PRIBIO + 1), 551 "buf_malloc", 0, &bqueue_slock); 552 splx(s); 553 } 554 555 return addr; 556 } 557 558 static void 559 buf_mrelease(caddr_t addr, size_t size) 560 { 561 562 pool_put(&bmempools[buf_mempoolidx(size)], addr); 563 } 564 565 /* 566 * bread()/breadn() helper. 567 */ 568 static __inline struct buf * 569 bio_doread(struct vnode *vp, daddr_t blkno, int size, struct ucred *cred, 570 int async) 571 { 572 struct buf *bp; 573 struct lwp *l = (curlwp != NULL ? curlwp : &lwp0); /* XXX */ 574 struct proc *p = l->l_proc; 575 struct mount *mp; 576 577 bp = getblk(vp, blkno, size, 0, 0); 578 579 #ifdef DIAGNOSTIC 580 if (bp == NULL) { 581 panic("bio_doread: no such buf"); 582 } 583 #endif 584 585 /* 586 * If buffer does not have data valid, start a read. 587 * Note that if buffer is B_INVAL, getblk() won't return it. 588 * Therefore, it's valid if its I/O has completed or been delayed. 589 */ 590 if (!ISSET(bp->b_flags, (B_DONE | B_DELWRI))) { 591 /* Start I/O for the buffer. */ 592 SET(bp->b_flags, B_READ | async); 593 if (async) 594 BIO_SETPRIO(bp, BPRIO_TIMELIMITED); 595 else 596 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); 597 VOP_STRATEGY(vp, bp); 598 599 /* Pay for the read. */ 600 p->p_stats->p_ru.ru_inblock++; 601 } else if (async) { 602 brelse(bp); 603 } 604 605 if (vp->v_type == VBLK) 606 mp = vp->v_specmountpoint; 607 else 608 mp = vp->v_mount; 609 610 /* 611 * Collect statistics on synchronous and asynchronous reads. 612 * Reads from block devices are charged to their associated 613 * filesystem (if any). 614 */ 615 if (mp != NULL) { 616 if (async == 0) 617 mp->mnt_stat.f_syncreads++; 618 else 619 mp->mnt_stat.f_asyncreads++; 620 } 621 622 return (bp); 623 } 624 625 /* 626 * Read a disk block. 627 * This algorithm described in Bach (p.54). 628 */ 629 int 630 bread(struct vnode *vp, daddr_t blkno, int size, struct ucred *cred, 631 struct buf **bpp) 632 { 633 struct buf *bp; 634 635 /* Get buffer for block. */ 636 bp = *bpp = bio_doread(vp, blkno, size, cred, 0); 637 638 /* Wait for the read to complete, and return result. */ 639 return (biowait(bp)); 640 } 641 642 /* 643 * Read-ahead multiple disk blocks. The first is sync, the rest async. 644 * Trivial modification to the breada algorithm presented in Bach (p.55). 645 */ 646 int 647 breadn(struct vnode *vp, daddr_t blkno, int size, daddr_t *rablks, 648 int *rasizes, int nrablks, struct ucred *cred, struct buf **bpp) 649 { 650 struct buf *bp; 651 int i; 652 653 bp = *bpp = bio_doread(vp, blkno, size, cred, 0); 654 655 /* 656 * For each of the read-ahead blocks, start a read, if necessary. 657 */ 658 for (i = 0; i < nrablks; i++) { 659 /* If it's in the cache, just go on to next one. */ 660 if (incore(vp, rablks[i])) 661 continue; 662 663 /* Get a buffer for the read-ahead block */ 664 (void) bio_doread(vp, rablks[i], rasizes[i], cred, B_ASYNC); 665 } 666 667 /* Otherwise, we had to start a read for it; wait until it's valid. */ 668 return (biowait(bp)); 669 } 670 671 /* 672 * Read with single-block read-ahead. Defined in Bach (p.55), but 673 * implemented as a call to breadn(). 674 * XXX for compatibility with old file systems. 675 */ 676 int 677 breada(struct vnode *vp, daddr_t blkno, int size, daddr_t rablkno, 678 int rabsize, struct ucred *cred, struct buf **bpp) 679 { 680 681 return (breadn(vp, blkno, size, &rablkno, &rabsize, 1, cred, bpp)); 682 } 683 684 /* 685 * Block write. Described in Bach (p.56) 686 */ 687 int 688 bwrite(struct buf *bp) 689 { 690 int rv, sync, wasdelayed, s; 691 struct lwp *l = (curlwp != NULL ? curlwp : &lwp0); /* XXX */ 692 struct proc *p = l->l_proc; 693 struct vnode *vp; 694 struct mount *mp; 695 696 KASSERT(ISSET(bp->b_flags, B_BUSY)); 697 698 vp = bp->b_vp; 699 if (vp != NULL) { 700 if (vp->v_type == VBLK) 701 mp = vp->v_specmountpoint; 702 else 703 mp = vp->v_mount; 704 } else { 705 mp = NULL; 706 } 707 708 /* 709 * Remember buffer type, to switch on it later. If the write was 710 * synchronous, but the file system was mounted with MNT_ASYNC, 711 * convert it to a delayed write. 712 * XXX note that this relies on delayed tape writes being converted 713 * to async, not sync writes (which is safe, but ugly). 714 */ 715 sync = !ISSET(bp->b_flags, B_ASYNC); 716 if (sync && mp != NULL && ISSET(mp->mnt_flag, MNT_ASYNC)) { 717 bdwrite(bp); 718 return (0); 719 } 720 721 /* 722 * Collect statistics on synchronous and asynchronous writes. 723 * Writes to block devices are charged to their associated 724 * filesystem (if any). 725 */ 726 if (mp != NULL) { 727 if (sync) 728 mp->mnt_stat.f_syncwrites++; 729 else 730 mp->mnt_stat.f_asyncwrites++; 731 } 732 733 s = splbio(); 734 simple_lock(&bp->b_interlock); 735 736 wasdelayed = ISSET(bp->b_flags, B_DELWRI); 737 738 CLR(bp->b_flags, (B_READ | B_DONE | B_ERROR | B_DELWRI)); 739 740 /* 741 * Pay for the I/O operation and make sure the buf is on the correct 742 * vnode queue. 743 */ 744 if (wasdelayed) 745 reassignbuf(bp, bp->b_vp); 746 else 747 p->p_stats->p_ru.ru_oublock++; 748 749 /* Initiate disk write. Make sure the appropriate party is charged. */ 750 V_INCR_NUMOUTPUT(bp->b_vp); 751 simple_unlock(&bp->b_interlock); 752 splx(s); 753 754 if (sync) 755 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); 756 else 757 BIO_SETPRIO(bp, BPRIO_TIMELIMITED); 758 759 VOP_STRATEGY(vp, bp); 760 761 if (sync) { 762 /* If I/O was synchronous, wait for it to complete. */ 763 rv = biowait(bp); 764 765 /* Release the buffer. */ 766 brelse(bp); 767 768 return (rv); 769 } else { 770 return (0); 771 } 772 } 773 774 int 775 vn_bwrite(void *v) 776 { 777 struct vop_bwrite_args *ap = v; 778 779 return (bwrite(ap->a_bp)); 780 } 781 782 /* 783 * Delayed write. 784 * 785 * The buffer is marked dirty, but is not queued for I/O. 786 * This routine should be used when the buffer is expected 787 * to be modified again soon, typically a small write that 788 * partially fills a buffer. 789 * 790 * NB: magnetic tapes cannot be delayed; they must be 791 * written in the order that the writes are requested. 792 * 793 * Described in Leffler, et al. (pp. 208-213). 794 */ 795 void 796 bdwrite(struct buf *bp) 797 { 798 struct lwp *l = (curlwp != NULL ? curlwp : &lwp0); /* XXX */ 799 struct proc *p = l->l_proc; 800 const struct bdevsw *bdev; 801 int s; 802 803 /* If this is a tape block, write the block now. */ 804 bdev = bdevsw_lookup(bp->b_dev); 805 if (bdev != NULL && bdev->d_type == D_TAPE) { 806 bawrite(bp); 807 return; 808 } 809 810 /* 811 * If the block hasn't been seen before: 812 * (1) Mark it as having been seen, 813 * (2) Charge for the write, 814 * (3) Make sure it's on its vnode's correct block list. 815 */ 816 s = splbio(); 817 simple_lock(&bp->b_interlock); 818 819 KASSERT(ISSET(bp->b_flags, B_BUSY)); 820 821 if (!ISSET(bp->b_flags, B_DELWRI)) { 822 SET(bp->b_flags, B_DELWRI); 823 p->p_stats->p_ru.ru_oublock++; 824 reassignbuf(bp, bp->b_vp); 825 } 826 827 /* Otherwise, the "write" is done, so mark and release the buffer. */ 828 CLR(bp->b_flags, B_DONE); 829 simple_unlock(&bp->b_interlock); 830 splx(s); 831 832 brelse(bp); 833 } 834 835 /* 836 * Asynchronous block write; just an asynchronous bwrite(). 837 */ 838 void 839 bawrite(struct buf *bp) 840 { 841 int s; 842 843 s = splbio(); 844 simple_lock(&bp->b_interlock); 845 846 KASSERT(ISSET(bp->b_flags, B_BUSY)); 847 848 SET(bp->b_flags, B_ASYNC); 849 simple_unlock(&bp->b_interlock); 850 splx(s); 851 VOP_BWRITE(bp); 852 } 853 854 /* 855 * Same as first half of bdwrite, mark buffer dirty, but do not release it. 856 * Call at splbio() and with the buffer interlock locked. 857 * Note: called only from biodone() through ffs softdep's bioops.io_complete() 858 */ 859 void 860 bdirty(struct buf *bp) 861 { 862 struct lwp *l = (curlwp != NULL ? curlwp : &lwp0); /* XXX */ 863 struct proc *p = l->l_proc; 864 865 LOCK_ASSERT(simple_lock_held(&bp->b_interlock)); 866 KASSERT(ISSET(bp->b_flags, B_BUSY)); 867 868 CLR(bp->b_flags, B_AGE); 869 870 if (!ISSET(bp->b_flags, B_DELWRI)) { 871 SET(bp->b_flags, B_DELWRI); 872 p->p_stats->p_ru.ru_oublock++; 873 reassignbuf(bp, bp->b_vp); 874 } 875 } 876 877 /* 878 * Release a buffer on to the free lists. 879 * Described in Bach (p. 46). 880 */ 881 void 882 brelse(struct buf *bp) 883 { 884 struct bqueue *bufq; 885 int s; 886 887 /* Block disk interrupts. */ 888 s = splbio(); 889 simple_lock(&bqueue_slock); 890 simple_lock(&bp->b_interlock); 891 892 KASSERT(ISSET(bp->b_flags, B_BUSY)); 893 KASSERT(!ISSET(bp->b_flags, B_CALL)); 894 895 /* Wake up any processes waiting for any buffer to become free. */ 896 if (needbuffer) { 897 needbuffer = 0; 898 wakeup(&needbuffer); 899 } 900 901 /* Wake up any proceeses waiting for _this_ buffer to become free. */ 902 if (ISSET(bp->b_flags, B_WANTED)) { 903 CLR(bp->b_flags, B_WANTED|B_AGE); 904 wakeup(bp); 905 } 906 907 /* 908 * Determine which queue the buffer should be on, then put it there. 909 */ 910 911 /* If it's locked, don't report an error; try again later. */ 912 if (ISSET(bp->b_flags, (B_LOCKED|B_ERROR)) == (B_LOCKED|B_ERROR)) 913 CLR(bp->b_flags, B_ERROR); 914 915 /* If it's not cacheable, or an error, mark it invalid. */ 916 if (ISSET(bp->b_flags, (B_NOCACHE|B_ERROR))) 917 SET(bp->b_flags, B_INVAL); 918 919 if (ISSET(bp->b_flags, B_VFLUSH)) { 920 /* 921 * This is a delayed write buffer that was just flushed to 922 * disk. It is still on the LRU queue. If it's become 923 * invalid, then we need to move it to a different queue; 924 * otherwise leave it in its current position. 925 */ 926 CLR(bp->b_flags, B_VFLUSH); 927 if (!ISSET(bp->b_flags, B_ERROR|B_INVAL|B_LOCKED|B_AGE)) { 928 KDASSERT(!debug_verify_freelist || checkfreelist(bp, &bufqueues[BQ_LRU])); 929 goto already_queued; 930 } else { 931 bremfree(bp); 932 } 933 } 934 935 KDASSERT(!debug_verify_freelist || !checkfreelist(bp, &bufqueues[BQ_AGE])); 936 KDASSERT(!debug_verify_freelist || !checkfreelist(bp, &bufqueues[BQ_LRU])); 937 KDASSERT(!debug_verify_freelist || !checkfreelist(bp, &bufqueues[BQ_LOCKED])); 938 939 if ((bp->b_bufsize <= 0) || ISSET(bp->b_flags, B_INVAL)) { 940 /* 941 * If it's invalid or empty, dissociate it from its vnode 942 * and put on the head of the appropriate queue. 943 */ 944 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_deallocate) 945 (*bioops.io_deallocate)(bp); 946 CLR(bp->b_flags, B_DONE|B_DELWRI); 947 if (bp->b_vp) { 948 reassignbuf(bp, bp->b_vp); 949 brelvp(bp); 950 } 951 if (bp->b_bufsize <= 0) 952 /* no data */ 953 goto already_queued; 954 else 955 /* invalid data */ 956 bufq = &bufqueues[BQ_AGE]; 957 binsheadfree(bp, bufq); 958 } else { 959 /* 960 * It has valid data. Put it on the end of the appropriate 961 * queue, so that it'll stick around for as long as possible. 962 * If buf is AGE, but has dependencies, must put it on last 963 * bufqueue to be scanned, ie LRU. This protects against the 964 * livelock where BQ_AGE only has buffers with dependencies, 965 * and we thus never get to the dependent buffers in BQ_LRU. 966 */ 967 if (ISSET(bp->b_flags, B_LOCKED)) 968 /* locked in core */ 969 bufq = &bufqueues[BQ_LOCKED]; 970 else if (!ISSET(bp->b_flags, B_AGE)) 971 /* valid data */ 972 bufq = &bufqueues[BQ_LRU]; 973 else { 974 /* stale but valid data */ 975 int has_deps; 976 977 if (LIST_FIRST(&bp->b_dep) != NULL && 978 bioops.io_countdeps) 979 has_deps = (*bioops.io_countdeps)(bp, 0); 980 else 981 has_deps = 0; 982 bufq = has_deps ? &bufqueues[BQ_LRU] : 983 &bufqueues[BQ_AGE]; 984 } 985 binstailfree(bp, bufq); 986 } 987 988 already_queued: 989 /* Unlock the buffer. */ 990 CLR(bp->b_flags, B_AGE|B_ASYNC|B_BUSY|B_NOCACHE); 991 SET(bp->b_flags, B_CACHE); 992 993 /* Allow disk interrupts. */ 994 simple_unlock(&bp->b_interlock); 995 simple_unlock(&bqueue_slock); 996 if (bp->b_bufsize <= 0) { 997 #ifdef DEBUG 998 memset((char *)bp, 0, sizeof(*bp)); 999 #endif 1000 pool_put(&bufpool, bp); 1001 } 1002 splx(s); 1003 } 1004 1005 /* 1006 * Determine if a block is in the cache. 1007 * Just look on what would be its hash chain. If it's there, return 1008 * a pointer to it, unless it's marked invalid. If it's marked invalid, 1009 * we normally don't return the buffer, unless the caller explicitly 1010 * wants us to. 1011 */ 1012 struct buf * 1013 incore(struct vnode *vp, daddr_t blkno) 1014 { 1015 struct buf *bp; 1016 1017 /* Search hash chain */ 1018 LIST_FOREACH(bp, BUFHASH(vp, blkno), b_hash) { 1019 if (bp->b_lblkno == blkno && bp->b_vp == vp && 1020 !ISSET(bp->b_flags, B_INVAL)) 1021 return (bp); 1022 } 1023 1024 return (NULL); 1025 } 1026 1027 /* 1028 * Get a block of requested size that is associated with 1029 * a given vnode and block offset. If it is found in the 1030 * block cache, mark it as having been found, make it busy 1031 * and return it. Otherwise, return an empty block of the 1032 * correct size. It is up to the caller to insure that the 1033 * cached blocks be of the correct size. 1034 */ 1035 struct buf * 1036 getblk(struct vnode *vp, daddr_t blkno, int size, int slpflag, int slptimeo) 1037 { 1038 struct buf *bp; 1039 int s, err; 1040 int preserve; 1041 1042 start: 1043 s = splbio(); 1044 simple_lock(&bqueue_slock); 1045 bp = incore(vp, blkno); 1046 if (bp != NULL) { 1047 simple_lock(&bp->b_interlock); 1048 if (ISSET(bp->b_flags, B_BUSY)) { 1049 simple_unlock(&bqueue_slock); 1050 if (curproc == uvm.pagedaemon_proc) { 1051 simple_unlock(&bp->b_interlock); 1052 splx(s); 1053 return NULL; 1054 } 1055 SET(bp->b_flags, B_WANTED); 1056 err = ltsleep(bp, slpflag | (PRIBIO + 1) | PNORELOCK, 1057 "getblk", slptimeo, &bp->b_interlock); 1058 splx(s); 1059 if (err) 1060 return (NULL); 1061 goto start; 1062 } 1063 #ifdef DIAGNOSTIC 1064 if (ISSET(bp->b_flags, B_DONE|B_DELWRI) && 1065 bp->b_bcount < size && vp->v_type != VBLK) 1066 panic("getblk: block size invariant failed"); 1067 #endif 1068 SET(bp->b_flags, B_BUSY); 1069 bremfree(bp); 1070 preserve = 1; 1071 } else { 1072 if ((bp = getnewbuf(slpflag, slptimeo, 0)) == NULL) { 1073 simple_unlock(&bqueue_slock); 1074 splx(s); 1075 goto start; 1076 } 1077 1078 binshash(bp, BUFHASH(vp, blkno)); 1079 bp->b_blkno = bp->b_lblkno = bp->b_rawblkno = blkno; 1080 bgetvp(vp, bp); 1081 preserve = 0; 1082 } 1083 simple_unlock(&bp->b_interlock); 1084 simple_unlock(&bqueue_slock); 1085 splx(s); 1086 /* 1087 * LFS can't track total size of B_LOCKED buffer (locked_queue_bytes) 1088 * if we re-size buffers here. 1089 */ 1090 if (ISSET(bp->b_flags, B_LOCKED)) { 1091 KASSERT(bp->b_bufsize >= size); 1092 } else { 1093 allocbuf(bp, size, preserve); 1094 } 1095 BIO_SETPRIO(bp, BPRIO_DEFAULT); 1096 return (bp); 1097 } 1098 1099 /* 1100 * Get an empty, disassociated buffer of given size. 1101 */ 1102 struct buf * 1103 geteblk(int size) 1104 { 1105 struct buf *bp; 1106 int s; 1107 1108 s = splbio(); 1109 simple_lock(&bqueue_slock); 1110 while ((bp = getnewbuf(0, 0, 0)) == 0) 1111 ; 1112 1113 SET(bp->b_flags, B_INVAL); 1114 binshash(bp, &invalhash); 1115 simple_unlock(&bqueue_slock); 1116 simple_unlock(&bp->b_interlock); 1117 splx(s); 1118 BIO_SETPRIO(bp, BPRIO_DEFAULT); 1119 allocbuf(bp, size, 0); 1120 return (bp); 1121 } 1122 1123 /* 1124 * Expand or contract the actual memory allocated to a buffer. 1125 * 1126 * If the buffer shrinks, data is lost, so it's up to the 1127 * caller to have written it out *first*; this routine will not 1128 * start a write. If the buffer grows, it's the callers 1129 * responsibility to fill out the buffer's additional contents. 1130 */ 1131 void 1132 allocbuf(struct buf *bp, int size, int preserve) 1133 { 1134 vsize_t oldsize, desired_size; 1135 caddr_t addr; 1136 int s, delta; 1137 1138 desired_size = buf_roundsize(size); 1139 if (desired_size > MAXBSIZE) 1140 printf("allocbuf: buffer larger than MAXBSIZE requested"); 1141 1142 bp->b_bcount = size; 1143 1144 oldsize = bp->b_bufsize; 1145 if (oldsize == desired_size) 1146 return; 1147 1148 /* 1149 * If we want a buffer of a different size, re-allocate the 1150 * buffer's memory; copy old content only if needed. 1151 */ 1152 addr = buf_malloc(desired_size); 1153 if (preserve) 1154 memcpy(addr, bp->b_data, MIN(oldsize,desired_size)); 1155 if (bp->b_data != NULL) 1156 buf_mrelease(bp->b_data, oldsize); 1157 bp->b_data = addr; 1158 bp->b_bufsize = desired_size; 1159 1160 /* 1161 * Update overall buffer memory counter (protected by bqueue_slock) 1162 */ 1163 delta = (long)desired_size - (long)oldsize; 1164 1165 s = splbio(); 1166 simple_lock(&bqueue_slock); 1167 if ((bufmem += delta) > bufmem_hiwater) { 1168 /* 1169 * Need to trim overall memory usage. 1170 */ 1171 while (buf_canrelease()) { 1172 if (buf_trim() == 0) 1173 break; 1174 } 1175 } 1176 1177 simple_unlock(&bqueue_slock); 1178 splx(s); 1179 } 1180 1181 /* 1182 * Find a buffer which is available for use. 1183 * Select something from a free list. 1184 * Preference is to AGE list, then LRU list. 1185 * 1186 * Called at splbio and with buffer queues locked. 1187 * Return buffer locked. 1188 */ 1189 struct buf * 1190 getnewbuf(int slpflag, int slptimeo, int from_bufq) 1191 { 1192 struct buf *bp; 1193 1194 start: 1195 LOCK_ASSERT(simple_lock_held(&bqueue_slock)); 1196 1197 /* 1198 * Get a new buffer from the pool; but use NOWAIT because 1199 * we have the buffer queues locked. 1200 */ 1201 if (!from_bufq && buf_lotsfree() && 1202 (bp = pool_get(&bufpool, PR_NOWAIT)) != NULL) { 1203 memset((char *)bp, 0, sizeof(*bp)); 1204 BUF_INIT(bp); 1205 bp->b_dev = NODEV; 1206 bp->b_vnbufs.le_next = NOLIST; 1207 bp->b_flags = B_BUSY; 1208 simple_lock(&bp->b_interlock); 1209 #if defined(DIAGNOSTIC) 1210 bp->b_freelistindex = -1; 1211 #endif /* defined(DIAGNOSTIC) */ 1212 return (bp); 1213 } 1214 1215 if ((bp = TAILQ_FIRST(&bufqueues[BQ_AGE].bq_queue)) != NULL || 1216 (bp = TAILQ_FIRST(&bufqueues[BQ_LRU].bq_queue)) != NULL) { 1217 simple_lock(&bp->b_interlock); 1218 bremfree(bp); 1219 } else { 1220 /* 1221 * XXX: !from_bufq should be removed. 1222 */ 1223 if (!from_bufq || curproc != uvm.pagedaemon_proc) { 1224 /* wait for a free buffer of any kind */ 1225 needbuffer = 1; 1226 ltsleep(&needbuffer, slpflag|(PRIBIO + 1), 1227 "getnewbuf", slptimeo, &bqueue_slock); 1228 } 1229 return (NULL); 1230 } 1231 1232 #ifdef DIAGNOSTIC 1233 if (bp->b_bufsize <= 0) 1234 panic("buffer %p: on queue but empty", bp); 1235 #endif 1236 1237 if (ISSET(bp->b_flags, B_VFLUSH)) { 1238 /* 1239 * This is a delayed write buffer being flushed to disk. Make 1240 * sure it gets aged out of the queue when it's finished, and 1241 * leave it off the LRU queue. 1242 */ 1243 CLR(bp->b_flags, B_VFLUSH); 1244 SET(bp->b_flags, B_AGE); 1245 simple_unlock(&bp->b_interlock); 1246 goto start; 1247 } 1248 1249 /* Buffer is no longer on free lists. */ 1250 SET(bp->b_flags, B_BUSY); 1251 1252 /* 1253 * If buffer was a delayed write, start it and return NULL 1254 * (since we might sleep while starting the write). 1255 */ 1256 if (ISSET(bp->b_flags, B_DELWRI)) { 1257 /* 1258 * This buffer has gone through the LRU, so make sure it gets 1259 * reused ASAP. 1260 */ 1261 SET(bp->b_flags, B_AGE); 1262 simple_unlock(&bp->b_interlock); 1263 simple_unlock(&bqueue_slock); 1264 bawrite(bp); 1265 simple_lock(&bqueue_slock); 1266 return (NULL); 1267 } 1268 1269 /* disassociate us from our vnode, if we had one... */ 1270 if (bp->b_vp) 1271 brelvp(bp); 1272 1273 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_deallocate) 1274 (*bioops.io_deallocate)(bp); 1275 1276 /* clear out various other fields */ 1277 bp->b_flags = B_BUSY; 1278 bp->b_dev = NODEV; 1279 bp->b_blkno = bp->b_lblkno = bp->b_rawblkno = 0; 1280 bp->b_iodone = 0; 1281 bp->b_error = 0; 1282 bp->b_resid = 0; 1283 bp->b_bcount = 0; 1284 1285 bremhash(bp); 1286 return (bp); 1287 } 1288 1289 /* 1290 * Attempt to free an aged buffer off the queues. 1291 * Called at splbio and with queue lock held. 1292 * Returns the amount of buffer memory freed. 1293 */ 1294 static int 1295 buf_trim(void) 1296 { 1297 struct buf *bp; 1298 long size = 0; 1299 1300 /* Instruct getnewbuf() to get buffers off the queues */ 1301 if ((bp = getnewbuf(PCATCH, 1, 1)) == NULL) 1302 return 0; 1303 1304 KASSERT(!ISSET(bp->b_flags, B_WANTED)); 1305 simple_unlock(&bp->b_interlock); 1306 size = bp->b_bufsize; 1307 bufmem -= size; 1308 simple_unlock(&bqueue_slock); 1309 if (size > 0) { 1310 buf_mrelease(bp->b_data, size); 1311 bp->b_bcount = bp->b_bufsize = 0; 1312 } 1313 /* brelse() will return the buffer to the global buffer pool */ 1314 brelse(bp); 1315 simple_lock(&bqueue_slock); 1316 return size; 1317 } 1318 1319 int 1320 buf_drain(int n) 1321 { 1322 int s, size = 0, sz; 1323 1324 s = splbio(); 1325 simple_lock(&bqueue_slock); 1326 1327 while (size < n && bufmem > bufmem_lowater) { 1328 sz = buf_trim(); 1329 if (sz <= 0) 1330 break; 1331 size += sz; 1332 } 1333 1334 simple_unlock(&bqueue_slock); 1335 splx(s); 1336 return size; 1337 } 1338 1339 /* 1340 * Wait for operations on the buffer to complete. 1341 * When they do, extract and return the I/O's error value. 1342 */ 1343 int 1344 biowait(struct buf *bp) 1345 { 1346 int s, error; 1347 1348 s = splbio(); 1349 simple_lock(&bp->b_interlock); 1350 while (!ISSET(bp->b_flags, B_DONE | B_DELWRI)) 1351 ltsleep(bp, PRIBIO + 1, "biowait", 0, &bp->b_interlock); 1352 1353 /* check for interruption of I/O (e.g. via NFS), then errors. */ 1354 if (ISSET(bp->b_flags, B_EINTR)) { 1355 CLR(bp->b_flags, B_EINTR); 1356 error = EINTR; 1357 } else if (ISSET(bp->b_flags, B_ERROR)) 1358 error = bp->b_error ? bp->b_error : EIO; 1359 else 1360 error = 0; 1361 1362 simple_unlock(&bp->b_interlock); 1363 splx(s); 1364 return (error); 1365 } 1366 1367 /* 1368 * Mark I/O complete on a buffer. 1369 * 1370 * If a callback has been requested, e.g. the pageout 1371 * daemon, do so. Otherwise, awaken waiting processes. 1372 * 1373 * [ Leffler, et al., says on p.247: 1374 * "This routine wakes up the blocked process, frees the buffer 1375 * for an asynchronous write, or, for a request by the pagedaemon 1376 * process, invokes a procedure specified in the buffer structure" ] 1377 * 1378 * In real life, the pagedaemon (or other system processes) wants 1379 * to do async stuff to, and doesn't want the buffer brelse()'d. 1380 * (for swap pager, that puts swap buffers on the free lists (!!!), 1381 * for the vn device, that puts malloc'd buffers on the free lists!) 1382 */ 1383 void 1384 biodone(struct buf *bp) 1385 { 1386 int s = splbio(); 1387 1388 simple_lock(&bp->b_interlock); 1389 if (ISSET(bp->b_flags, B_DONE)) 1390 panic("biodone already"); 1391 SET(bp->b_flags, B_DONE); /* note that it's done */ 1392 BIO_SETPRIO(bp, BPRIO_DEFAULT); 1393 1394 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_complete) 1395 (*bioops.io_complete)(bp); 1396 1397 if (!ISSET(bp->b_flags, B_READ)) /* wake up reader */ 1398 vwakeup(bp); 1399 1400 /* 1401 * If necessary, call out. Unlock the buffer before calling 1402 * iodone() as the buffer isn't valid any more when it return. 1403 */ 1404 if (ISSET(bp->b_flags, B_CALL)) { 1405 CLR(bp->b_flags, B_CALL); /* but note callout done */ 1406 simple_unlock(&bp->b_interlock); 1407 (*bp->b_iodone)(bp); 1408 } else { 1409 if (ISSET(bp->b_flags, B_ASYNC)) { /* if async, release */ 1410 simple_unlock(&bp->b_interlock); 1411 brelse(bp); 1412 } else { /* or just wakeup the buffer */ 1413 CLR(bp->b_flags, B_WANTED); 1414 wakeup(bp); 1415 simple_unlock(&bp->b_interlock); 1416 } 1417 } 1418 1419 splx(s); 1420 } 1421 1422 /* 1423 * Return a count of buffers on the "locked" queue. 1424 */ 1425 int 1426 count_lock_queue(void) 1427 { 1428 struct buf *bp; 1429 int n = 0; 1430 1431 simple_lock(&bqueue_slock); 1432 TAILQ_FOREACH(bp, &bufqueues[BQ_LOCKED].bq_queue, b_freelist) 1433 n++; 1434 simple_unlock(&bqueue_slock); 1435 return (n); 1436 } 1437 1438 /* 1439 * Wait for all buffers to complete I/O 1440 * Return the number of "stuck" buffers. 1441 */ 1442 int 1443 buf_syncwait(void) 1444 { 1445 struct buf *bp; 1446 int iter, nbusy, nbusy_prev = 0, dcount, s, ihash; 1447 1448 dcount = 10000; 1449 for (iter = 0; iter < 20;) { 1450 s = splbio(); 1451 simple_lock(&bqueue_slock); 1452 nbusy = 0; 1453 for (ihash = 0; ihash < bufhash+1; ihash++) { 1454 LIST_FOREACH(bp, &bufhashtbl[ihash], b_hash) { 1455 if ((bp->b_flags & (B_BUSY|B_INVAL|B_READ)) == B_BUSY) 1456 nbusy++; 1457 /* 1458 * With soft updates, some buffers that are 1459 * written will be remarked as dirty until other 1460 * buffers are written. 1461 */ 1462 if (bp->b_vp && bp->b_vp->v_mount 1463 && (bp->b_vp->v_mount->mnt_flag & MNT_SOFTDEP) 1464 && (bp->b_flags & B_DELWRI)) { 1465 simple_lock(&bp->b_interlock); 1466 bremfree(bp); 1467 bp->b_flags |= B_BUSY; 1468 nbusy++; 1469 simple_unlock(&bp->b_interlock); 1470 simple_unlock(&bqueue_slock); 1471 bawrite(bp); 1472 if (dcount-- <= 0) { 1473 printf("softdep "); 1474 splx(s); 1475 goto fail; 1476 } 1477 simple_lock(&bqueue_slock); 1478 } 1479 } 1480 } 1481 1482 simple_unlock(&bqueue_slock); 1483 splx(s); 1484 1485 if (nbusy == 0) 1486 break; 1487 if (nbusy_prev == 0) 1488 nbusy_prev = nbusy; 1489 printf("%d ", nbusy); 1490 tsleep(&nbusy, PRIBIO, "bflush", 1491 (iter == 0) ? 1 : hz / 25 * iter); 1492 if (nbusy >= nbusy_prev) /* we didn't flush anything */ 1493 iter++; 1494 else 1495 nbusy_prev = nbusy; 1496 } 1497 1498 if (nbusy) { 1499 fail:; 1500 #if defined(DEBUG) || defined(DEBUG_HALT_BUSY) 1501 printf("giving up\nPrinting vnodes for busy buffers\n"); 1502 s = splbio(); 1503 for (ihash = 0; ihash < bufhash+1; ihash++) { 1504 LIST_FOREACH(bp, &bufhashtbl[ihash], b_hash) { 1505 if ((bp->b_flags & (B_BUSY|B_INVAL|B_READ)) == B_BUSY) 1506 vprint(NULL, bp->b_vp); 1507 } 1508 } 1509 splx(s); 1510 #endif 1511 } 1512 1513 return nbusy; 1514 } 1515 1516 static void 1517 sysctl_fillbuf(struct buf *i, struct buf_sysctl *o) 1518 { 1519 1520 o->b_flags = i->b_flags; 1521 o->b_error = i->b_error; 1522 o->b_prio = i->b_prio; 1523 o->b_dev = i->b_dev; 1524 o->b_bufsize = i->b_bufsize; 1525 o->b_bcount = i->b_bcount; 1526 o->b_resid = i->b_resid; 1527 o->b_addr = PTRTOUINT64(i->b_un.b_addr); 1528 o->b_blkno = i->b_blkno; 1529 o->b_rawblkno = i->b_rawblkno; 1530 o->b_iodone = PTRTOUINT64(i->b_iodone); 1531 o->b_proc = PTRTOUINT64(i->b_proc); 1532 o->b_vp = PTRTOUINT64(i->b_vp); 1533 o->b_saveaddr = PTRTOUINT64(i->b_saveaddr); 1534 o->b_lblkno = i->b_lblkno; 1535 } 1536 1537 #define KERN_BUFSLOP 20 1538 static int 1539 sysctl_dobuf(SYSCTLFN_ARGS) 1540 { 1541 struct buf *bp; 1542 struct buf_sysctl bs; 1543 char *dp; 1544 u_int i, op, arg; 1545 size_t len, needed, elem_size, out_size; 1546 int error, s, elem_count; 1547 1548 if (namelen == 1 && name[0] == CTL_QUERY) 1549 return (sysctl_query(SYSCTLFN_CALL(rnode))); 1550 1551 if (namelen != 4) 1552 return (EINVAL); 1553 1554 dp = oldp; 1555 len = (oldp != NULL) ? *oldlenp : 0; 1556 op = name[0]; 1557 arg = name[1]; 1558 elem_size = name[2]; 1559 elem_count = name[3]; 1560 out_size = MIN(sizeof(bs), elem_size); 1561 1562 /* 1563 * at the moment, these are just "placeholders" to make the 1564 * API for retrieving kern.buf data more extensible in the 1565 * future. 1566 * 1567 * XXX kern.buf currently has "netbsd32" issues. hopefully 1568 * these will be resolved at a later point. 1569 */ 1570 if (op != KERN_BUF_ALL || arg != KERN_BUF_ALL || 1571 elem_size < 1 || elem_count < 0) 1572 return (EINVAL); 1573 1574 error = 0; 1575 needed = 0; 1576 s = splbio(); 1577 simple_lock(&bqueue_slock); 1578 for (i = 0; i < BQUEUES; i++) { 1579 TAILQ_FOREACH(bp, &bufqueues[i].bq_queue, b_freelist) { 1580 if (len >= elem_size && elem_count > 0) { 1581 sysctl_fillbuf(bp, &bs); 1582 error = copyout(&bs, dp, out_size); 1583 if (error) 1584 goto cleanup; 1585 dp += elem_size; 1586 len -= elem_size; 1587 } 1588 if (elem_count > 0) { 1589 needed += elem_size; 1590 if (elem_count != INT_MAX) 1591 elem_count--; 1592 } 1593 } 1594 } 1595 cleanup: 1596 simple_unlock(&bqueue_slock); 1597 splx(s); 1598 1599 *oldlenp = needed; 1600 if (oldp == NULL) 1601 *oldlenp += KERN_BUFSLOP * sizeof(struct buf); 1602 1603 return (error); 1604 } 1605 1606 static int 1607 sysctl_bufvm_update(SYSCTLFN_ARGS) 1608 { 1609 int t, error; 1610 struct sysctlnode node; 1611 1612 node = *rnode; 1613 node.sysctl_data = &t; 1614 t = *(int *)rnode->sysctl_data; 1615 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1616 if (error || newp == NULL) 1617 return (error); 1618 1619 if (t < 0) 1620 return EINVAL; 1621 if (rnode->sysctl_data == &bufcache) { 1622 if (t > 100) 1623 return (EINVAL); 1624 bufcache = t; 1625 buf_setwm(); 1626 } else if (rnode->sysctl_data == &bufmem_lowater) { 1627 if (bufmem_hiwater - t < 16) 1628 return (EINVAL); 1629 bufmem_lowater = t; 1630 } else if (rnode->sysctl_data == &bufmem_hiwater) { 1631 if (t - bufmem_lowater < 16) 1632 return (EINVAL); 1633 bufmem_hiwater = t; 1634 } else 1635 return (EINVAL); 1636 1637 /* Drain until below new high water mark */ 1638 while ((t = bufmem - bufmem_hiwater) >= 0) { 1639 if (buf_drain(t / (2 * 1024)) <= 0) 1640 break; 1641 } 1642 1643 return 0; 1644 } 1645 1646 SYSCTL_SETUP(sysctl_kern_buf_setup, "sysctl kern.buf subtree setup") 1647 { 1648 1649 sysctl_createv(clog, 0, NULL, NULL, 1650 CTLFLAG_PERMANENT, 1651 CTLTYPE_NODE, "kern", NULL, 1652 NULL, 0, NULL, 0, 1653 CTL_KERN, CTL_EOL); 1654 sysctl_createv(clog, 0, NULL, NULL, 1655 CTLFLAG_PERMANENT, 1656 CTLTYPE_NODE, "buf", 1657 SYSCTL_DESCR("Kernel buffer cache information"), 1658 sysctl_dobuf, 0, NULL, 0, 1659 CTL_KERN, KERN_BUF, CTL_EOL); 1660 } 1661 1662 SYSCTL_SETUP(sysctl_vm_buf_setup, "sysctl vm.buf* subtree setup") 1663 { 1664 1665 sysctl_createv(clog, 0, NULL, NULL, 1666 CTLFLAG_PERMANENT, 1667 CTLTYPE_NODE, "vm", NULL, 1668 NULL, 0, NULL, 0, 1669 CTL_VM, CTL_EOL); 1670 1671 sysctl_createv(clog, 0, NULL, NULL, 1672 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1673 CTLTYPE_INT, "bufcache", 1674 SYSCTL_DESCR("Percentage of physical memory to use for " 1675 "buffer cache"), 1676 sysctl_bufvm_update, 0, &bufcache, 0, 1677 CTL_VM, CTL_CREATE, CTL_EOL); 1678 sysctl_createv(clog, 0, NULL, NULL, 1679 CTLFLAG_PERMANENT|CTLFLAG_READONLY, 1680 CTLTYPE_INT, "bufmem", 1681 SYSCTL_DESCR("Amount of kernel memory used by buffer " 1682 "cache"), 1683 NULL, 0, &bufmem, 0, 1684 CTL_VM, CTL_CREATE, CTL_EOL); 1685 sysctl_createv(clog, 0, NULL, NULL, 1686 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1687 CTLTYPE_INT, "bufmem_lowater", 1688 SYSCTL_DESCR("Minimum amount of kernel memory to " 1689 "reserve for buffer cache"), 1690 sysctl_bufvm_update, 0, &bufmem_lowater, 0, 1691 CTL_VM, CTL_CREATE, CTL_EOL); 1692 sysctl_createv(clog, 0, NULL, NULL, 1693 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1694 CTLTYPE_INT, "bufmem_hiwater", 1695 SYSCTL_DESCR("Maximum amount of kernel memory to use " 1696 "for buffer cache"), 1697 sysctl_bufvm_update, 0, &bufmem_hiwater, 0, 1698 CTL_VM, CTL_CREATE, CTL_EOL); 1699 } 1700 1701 #ifdef DEBUG 1702 /* 1703 * Print out statistics on the current allocation of the buffer pool. 1704 * Can be enabled to print out on every ``sync'' by setting "syncprt" 1705 * in vfs_syscalls.c using sysctl. 1706 */ 1707 void 1708 vfs_bufstats(void) 1709 { 1710 int s, i, j, count; 1711 struct buf *bp; 1712 struct bqueue *dp; 1713 int counts[(MAXBSIZE / PAGE_SIZE) + 1]; 1714 static const char *bname[BQUEUES] = { "LOCKED", "LRU", "AGE" }; 1715 1716 for (dp = bufqueues, i = 0; dp < &bufqueues[BQUEUES]; dp++, i++) { 1717 count = 0; 1718 for (j = 0; j <= MAXBSIZE/PAGE_SIZE; j++) 1719 counts[j] = 0; 1720 s = splbio(); 1721 TAILQ_FOREACH(bp, &dp->bq_queue, b_freelist) { 1722 counts[bp->b_bufsize/PAGE_SIZE]++; 1723 count++; 1724 } 1725 splx(s); 1726 printf("%s: total-%d", bname[i], count); 1727 for (j = 0; j <= MAXBSIZE/PAGE_SIZE; j++) 1728 if (counts[j] != 0) 1729 printf(", %d-%d", j * PAGE_SIZE, counts[j]); 1730 printf("\n"); 1731 } 1732 } 1733 #endif /* DEBUG */ 1734