1 /* $NetBSD: vfs_bio.c,v 1.129 2004/09/08 10:20:15 yamt Exp $ */ 2 3 /*- 4 * Copyright (c) 1982, 1986, 1989, 1993 5 * The Regents of the University of California. All rights reserved. 6 * (c) UNIX System Laboratories, Inc. 7 * All or some portions of this file are derived from material licensed 8 * to the University of California by American Telephone and Telegraph 9 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 10 * the permission of UNIX System Laboratories, Inc. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)vfs_bio.c 8.6 (Berkeley) 1/11/94 37 */ 38 39 /*- 40 * Copyright (c) 1994 Christopher G. Demetriou 41 * 42 * Redistribution and use in source and binary forms, with or without 43 * modification, are permitted provided that the following conditions 44 * are met: 45 * 1. Redistributions of source code must retain the above copyright 46 * notice, this list of conditions and the following disclaimer. 47 * 2. Redistributions in binary form must reproduce the above copyright 48 * notice, this list of conditions and the following disclaimer in the 49 * documentation and/or other materials provided with the distribution. 50 * 3. All advertising materials mentioning features or use of this software 51 * must display the following acknowledgement: 52 * This product includes software developed by the University of 53 * California, Berkeley and its contributors. 54 * 4. Neither the name of the University nor the names of its contributors 55 * may be used to endorse or promote products derived from this software 56 * without specific prior written permission. 57 * 58 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 59 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 60 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 61 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 62 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 63 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 64 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 65 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 66 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 67 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 68 * SUCH DAMAGE. 69 * 70 * @(#)vfs_bio.c 8.6 (Berkeley) 1/11/94 71 */ 72 73 /* 74 * Some references: 75 * Bach: The Design of the UNIX Operating System (Prentice Hall, 1986) 76 * Leffler, et al.: The Design and Implementation of the 4.3BSD 77 * UNIX Operating System (Addison Welley, 1989) 78 */ 79 80 #include "opt_bufcache.h" 81 #include "opt_softdep.h" 82 83 #include <sys/cdefs.h> 84 __KERNEL_RCSID(0, "$NetBSD: vfs_bio.c,v 1.129 2004/09/08 10:20:15 yamt Exp $"); 85 86 #include <sys/param.h> 87 #include <sys/systm.h> 88 #include <sys/kernel.h> 89 #include <sys/proc.h> 90 #include <sys/buf.h> 91 #include <sys/vnode.h> 92 #include <sys/mount.h> 93 #include <sys/malloc.h> 94 #include <sys/resourcevar.h> 95 #include <sys/sysctl.h> 96 #include <sys/conf.h> 97 98 #include <uvm/uvm.h> 99 100 #include <miscfs/specfs/specdev.h> 101 102 #ifndef BUFPAGES 103 # define BUFPAGES 0 104 #endif 105 106 #ifdef BUFCACHE 107 # if (BUFCACHE < 5) || (BUFCACHE > 95) 108 # error BUFCACHE is not between 5 and 95 109 # endif 110 #else 111 # define BUFCACHE 15 112 #endif 113 114 u_int nbuf; /* XXX - for softdep_lockedbufs */ 115 u_int bufpages = BUFPAGES; /* optional hardwired count */ 116 u_int bufcache = BUFCACHE; /* max % of RAM to use for buffer cache */ 117 118 119 /* Macros to clear/set/test flags. */ 120 #define SET(t, f) (t) |= (f) 121 #define CLR(t, f) (t) &= ~(f) 122 #define ISSET(t, f) ((t) & (f)) 123 124 /* 125 * Definitions for the buffer hash lists. 126 */ 127 #define BUFHASH(dvp, lbn) \ 128 (&bufhashtbl[(((long)(dvp) >> 8) + (int)(lbn)) & bufhash]) 129 LIST_HEAD(bufhashhdr, buf) *bufhashtbl, invalhash; 130 u_long bufhash; 131 #ifndef SOFTDEP 132 struct bio_ops bioops; /* I/O operation notification */ 133 #endif 134 135 /* 136 * Insq/Remq for the buffer hash lists. 137 */ 138 #define binshash(bp, dp) LIST_INSERT_HEAD(dp, bp, b_hash) 139 #define bremhash(bp) LIST_REMOVE(bp, b_hash) 140 141 /* 142 * Definitions for the buffer free lists. 143 */ 144 #define BQUEUES 3 /* number of free buffer queues */ 145 146 #define BQ_LOCKED 0 /* super-blocks &c */ 147 #define BQ_LRU 1 /* lru, useful buffers */ 148 #define BQ_AGE 2 /* rubbish */ 149 150 TAILQ_HEAD(bqueues, buf) bufqueues[BQUEUES]; 151 int needbuffer; 152 153 /* 154 * Buffer queue lock. 155 * Take this lock first if also taking some buffer's b_interlock. 156 */ 157 struct simplelock bqueue_slock = SIMPLELOCK_INITIALIZER; 158 159 /* 160 * Buffer pool for I/O buffers. 161 */ 162 struct pool bufpool; 163 164 /* XXX - somewhat gross.. */ 165 #if MAXBSIZE == 0x2000 166 #define NMEMPOOLS 4 167 #elif MAXBSIZE == 0x4000 168 #define NMEMPOOLS 5 169 #elif MAXBSIZE == 0x8000 170 #define NMEMPOOLS 6 171 #else 172 #define NMEMPOOLS 7 173 #endif 174 175 #define MEMPOOL_INDEX_OFFSET 10 /* smallest pool is 1k */ 176 #if (1 << (NMEMPOOLS + MEMPOOL_INDEX_OFFSET - 1)) != MAXBSIZE 177 #error update vfs_bio buffer memory parameters 178 #endif 179 180 /* Buffer memory pools */ 181 static struct pool bmempools[NMEMPOOLS]; 182 183 struct vm_map *buf_map; 184 185 /* 186 * Buffer memory pool allocator. 187 */ 188 static void * 189 bufpool_page_alloc(struct pool *pp, int flags) 190 { 191 192 return (void *)uvm_km_kmemalloc1(buf_map, 193 uvm.kernel_object, MAXBSIZE, MAXBSIZE, UVM_UNKNOWN_OFFSET, 194 (flags & PR_WAITOK) ? 0 : UVM_KMF_NOWAIT | UVM_KMF_TRYLOCK); 195 } 196 197 static void 198 bufpool_page_free(struct pool *pp, void *v) 199 { 200 uvm_km_free(buf_map, (vaddr_t)v, MAXBSIZE); 201 } 202 203 static struct pool_allocator bufmempool_allocator = { 204 bufpool_page_alloc, bufpool_page_free, MAXBSIZE, 205 }; 206 207 /* Buffer memory management variables */ 208 u_long bufmem_valimit; 209 u_long bufmem_hiwater; 210 u_long bufmem_lowater; 211 u_long bufmem; 212 213 /* 214 * MD code can call this to set a hard limit on the amount 215 * of virtual memory used by the buffer cache. 216 */ 217 int 218 buf_setvalimit(vsize_t sz) 219 { 220 221 /* We need to accommodate at least NMEMPOOLS of MAXBSIZE each */ 222 if (sz < NMEMPOOLS * MAXBSIZE) 223 return EINVAL; 224 225 bufmem_valimit = sz; 226 return 0; 227 } 228 229 static int buf_trim(void); 230 231 /* 232 * bread()/breadn() helper. 233 */ 234 static __inline struct buf *bio_doread(struct vnode *, daddr_t, int, 235 struct ucred *, int); 236 int count_lock_queue(void); 237 238 /* 239 * Insq/Remq for the buffer free lists. 240 * Call with buffer queue locked. 241 */ 242 #define binsheadfree(bp, dp) TAILQ_INSERT_HEAD(dp, bp, b_freelist) 243 #define binstailfree(bp, dp) TAILQ_INSERT_TAIL(dp, bp, b_freelist) 244 245 #ifdef DEBUG 246 int debug_verify_freelist = 0; 247 static int checkfreelist(struct buf *bp, struct bqueues *dp) 248 { 249 struct buf *b; 250 TAILQ_FOREACH(b, dp, b_freelist) { 251 if (b == bp) 252 return 1; 253 } 254 return 0; 255 } 256 #endif 257 258 void 259 bremfree(struct buf *bp) 260 { 261 struct bqueues *dp = NULL; 262 263 LOCK_ASSERT(simple_lock_held(&bqueue_slock)); 264 265 KDASSERT(!debug_verify_freelist || 266 checkfreelist(bp, &bufqueues[BQ_AGE]) || 267 checkfreelist(bp, &bufqueues[BQ_LRU]) || 268 checkfreelist(bp, &bufqueues[BQ_LOCKED]) ); 269 270 /* 271 * We only calculate the head of the freelist when removing 272 * the last element of the list as that is the only time that 273 * it is needed (e.g. to reset the tail pointer). 274 * 275 * NB: This makes an assumption about how tailq's are implemented. 276 * 277 * We break the TAILQ abstraction in order to efficiently remove a 278 * buffer from its freelist without having to know exactly which 279 * freelist it is on. 280 */ 281 if (TAILQ_NEXT(bp, b_freelist) == NULL) { 282 for (dp = bufqueues; dp < &bufqueues[BQUEUES]; dp++) 283 if (dp->tqh_last == &bp->b_freelist.tqe_next) 284 break; 285 if (dp == &bufqueues[BQUEUES]) 286 panic("bremfree: lost tail"); 287 } 288 TAILQ_REMOVE(dp, bp, b_freelist); 289 } 290 291 u_long 292 buf_memcalc(void) 293 { 294 u_long n; 295 296 /* 297 * Determine the upper bound of memory to use for buffers. 298 * 299 * - If bufpages is specified, use that as the number 300 * pages. 301 * 302 * - Otherwise, use bufcache as the percentage of 303 * physical memory. 304 */ 305 if (bufpages != 0) { 306 n = bufpages; 307 } else { 308 if (bufcache < 5) { 309 printf("forcing bufcache %d -> 5", bufcache); 310 bufcache = 5; 311 } 312 if (bufcache > 95) { 313 printf("forcing bufcache %d -> 95", bufcache); 314 bufcache = 95; 315 } 316 n = physmem / 100 * bufcache; 317 } 318 319 n <<= PAGE_SHIFT; 320 if (bufmem_valimit != 0 && n > bufmem_valimit) 321 n = bufmem_valimit; 322 323 return (n); 324 } 325 326 /* 327 * Initialize buffers and hash links for buffers. 328 */ 329 void 330 bufinit(void) 331 { 332 struct bqueues *dp; 333 int use_std; 334 u_int i; 335 336 /* 337 * Initialize buffer cache memory parameters. 338 */ 339 bufmem = 0; 340 bufmem_hiwater = buf_memcalc(); 341 /* lowater is approx. 2% of memory (with bufcache=15) */ 342 bufmem_lowater = (bufmem_hiwater >> 3); 343 if (bufmem_lowater < 64 * 1024) 344 /* Ensure a reasonable minimum value */ 345 bufmem_lowater = 64 * 1024; 346 347 if (bufmem_valimit != 0) { 348 vaddr_t minaddr = 0, maxaddr; 349 buf_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, 350 bufmem_valimit, VM_MAP_PAGEABLE, 351 FALSE, 0); 352 if (buf_map == NULL) 353 panic("bufinit: cannot allocate submap"); 354 } else 355 buf_map = kernel_map; 356 357 /* 358 * Initialize the buffer pools. 359 */ 360 pool_init(&bufpool, sizeof(struct buf), 0, 0, 0, "bufpl", NULL); 361 362 /* On "small" machines use small pool page sizes where possible */ 363 use_std = (physmem < atop(16*1024*1024)); 364 365 /* 366 * Also use them on systems that can map the pool pages using 367 * a direct-mapped segment. 368 */ 369 #ifdef PMAP_MAP_POOLPAGE 370 use_std = 1; 371 #endif 372 373 for (i = 0; i < NMEMPOOLS; i++) { 374 struct pool_allocator *pa; 375 struct pool *pp = &bmempools[i]; 376 u_int size = 1 << (i + MEMPOOL_INDEX_OFFSET); 377 char *name = malloc(8, M_TEMP, M_WAITOK); 378 snprintf(name, 8, "buf%dk", 1 << i); 379 pa = (size <= PAGE_SIZE && use_std) 380 ? &pool_allocator_nointr 381 : &bufmempool_allocator; 382 pool_init(pp, size, 0, 0, 0, name, pa); 383 pool_setlowat(pp, 1); 384 pool_sethiwat(pp, 1); 385 } 386 387 /* Initialize the buffer queues */ 388 for (dp = bufqueues; dp < &bufqueues[BQUEUES]; dp++) 389 TAILQ_INIT(dp); 390 391 /* 392 * Estimate hash table size based on the amount of memory we 393 * intend to use for the buffer cache. The average buffer 394 * size is dependent on our clients (i.e. filesystems). 395 * 396 * For now, use an empirical 3K per buffer. 397 */ 398 nbuf = (bufmem_hiwater / 1024) / 3; 399 bufhashtbl = hashinit(nbuf, HASH_LIST, M_CACHE, M_WAITOK, &bufhash); 400 } 401 402 static int 403 buf_lotsfree(void) 404 { 405 int try, thresh; 406 struct lwp *l = curlwp; 407 408 /* Always allocate if doing copy on write */ 409 if (l->l_flag & L_COWINPROGRESS) 410 return 1; 411 412 /* Always allocate if less than the low water mark. */ 413 if (bufmem < bufmem_lowater) 414 return 1; 415 416 /* Never allocate if greater than the high water mark. */ 417 if (bufmem > bufmem_hiwater) 418 return 0; 419 420 /* If there's anything on the AGE list, it should be eaten. */ 421 if (TAILQ_FIRST(&bufqueues[BQ_AGE]) != NULL) 422 return 0; 423 424 /* 425 * The probabily of getting a new allocation is inversely 426 * proportional to the current size of the cache, using 427 * a granularity of 16 steps. 428 */ 429 try = random() & 0x0000000fL; 430 431 /* Don't use "16 * bufmem" here to avoid a 32-bit overflow. */ 432 thresh = bufmem / (bufmem_hiwater / 16); 433 434 if ((try > thresh) && (uvmexp.free > (2 * uvmexp.freetarg))) { 435 return 1; 436 } 437 438 /* Otherwise don't allocate. */ 439 return 0; 440 } 441 442 /* 443 * Return estimate of bytes we think need to be 444 * released to help resolve low memory conditions. 445 * 446 * => called at splbio. 447 * => called with bqueue_slock held. 448 */ 449 static int 450 buf_canrelease(void) 451 { 452 int pagedemand, ninvalid = 0; 453 struct buf *bp; 454 455 LOCK_ASSERT(simple_lock_held(&bqueue_slock)); 456 457 if (bufmem < bufmem_lowater) 458 return 0; 459 460 TAILQ_FOREACH(bp, &bufqueues[BQ_AGE], b_freelist) 461 ninvalid += bp->b_bufsize; 462 463 pagedemand = uvmexp.freetarg - uvmexp.free; 464 if (pagedemand < 0) 465 return ninvalid; 466 return MAX(ninvalid, MIN(2 * MAXBSIZE, 467 MIN((bufmem - bufmem_lowater) / 16, pagedemand * PAGE_SIZE))); 468 } 469 470 /* 471 * Buffer memory allocation helper functions 472 */ 473 static __inline u_long 474 buf_mempoolidx(u_long size) 475 { 476 u_int n = 0; 477 478 size -= 1; 479 size >>= MEMPOOL_INDEX_OFFSET; 480 while (size) { 481 size >>= 1; 482 n += 1; 483 } 484 if (n >= NMEMPOOLS) 485 panic("buf mem pool index %d", n); 486 return n; 487 } 488 489 static __inline u_long 490 buf_roundsize(u_long size) 491 { 492 /* Round up to nearest power of 2 */ 493 return (1 << (buf_mempoolidx(size) + MEMPOOL_INDEX_OFFSET)); 494 } 495 496 static __inline caddr_t 497 buf_malloc(size_t size) 498 { 499 u_int n = buf_mempoolidx(size); 500 caddr_t addr; 501 int s; 502 503 while (1) { 504 addr = pool_get(&bmempools[n], PR_NOWAIT); 505 if (addr != NULL) 506 break; 507 508 /* No memory, see if we can free some. If so, try again */ 509 if (buf_drain(1) > 0) 510 continue; 511 512 /* Wait for buffers to arrive on the LRU queue */ 513 s = splbio(); 514 simple_lock(&bqueue_slock); 515 needbuffer = 1; 516 ltsleep(&needbuffer, PNORELOCK | (PRIBIO + 1), 517 "buf_malloc", 0, &bqueue_slock); 518 splx(s); 519 } 520 521 return addr; 522 } 523 524 static void 525 buf_mrelease(caddr_t addr, size_t size) 526 { 527 528 pool_put(&bmempools[buf_mempoolidx(size)], addr); 529 } 530 531 532 static __inline struct buf * 533 bio_doread(struct vnode *vp, daddr_t blkno, int size, struct ucred *cred, 534 int async) 535 { 536 struct buf *bp; 537 struct lwp *l = (curlwp != NULL ? curlwp : &lwp0); /* XXX */ 538 struct proc *p = l->l_proc; 539 struct mount *mp; 540 541 bp = getblk(vp, blkno, size, 0, 0); 542 543 #ifdef DIAGNOSTIC 544 if (bp == NULL) { 545 panic("bio_doread: no such buf"); 546 } 547 #endif 548 549 /* 550 * If buffer does not have data valid, start a read. 551 * Note that if buffer is B_INVAL, getblk() won't return it. 552 * Therefore, it's valid if its I/O has completed or been delayed. 553 */ 554 if (!ISSET(bp->b_flags, (B_DONE | B_DELWRI))) { 555 /* Start I/O for the buffer. */ 556 SET(bp->b_flags, B_READ | async); 557 if (async) 558 BIO_SETPRIO(bp, BPRIO_TIMELIMITED); 559 else 560 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); 561 VOP_STRATEGY(vp, bp); 562 563 /* Pay for the read. */ 564 p->p_stats->p_ru.ru_inblock++; 565 } else if (async) { 566 brelse(bp); 567 } 568 569 if (vp->v_type == VBLK) 570 mp = vp->v_specmountpoint; 571 else 572 mp = vp->v_mount; 573 574 /* 575 * Collect statistics on synchronous and asynchronous reads. 576 * Reads from block devices are charged to their associated 577 * filesystem (if any). 578 */ 579 if (mp != NULL) { 580 if (async == 0) 581 mp->mnt_stat.f_syncreads++; 582 else 583 mp->mnt_stat.f_asyncreads++; 584 } 585 586 return (bp); 587 } 588 589 /* 590 * Read a disk block. 591 * This algorithm described in Bach (p.54). 592 */ 593 int 594 bread(struct vnode *vp, daddr_t blkno, int size, struct ucred *cred, 595 struct buf **bpp) 596 { 597 struct buf *bp; 598 599 /* Get buffer for block. */ 600 bp = *bpp = bio_doread(vp, blkno, size, cred, 0); 601 602 /* Wait for the read to complete, and return result. */ 603 return (biowait(bp)); 604 } 605 606 /* 607 * Read-ahead multiple disk blocks. The first is sync, the rest async. 608 * Trivial modification to the breada algorithm presented in Bach (p.55). 609 */ 610 int 611 breadn(struct vnode *vp, daddr_t blkno, int size, daddr_t *rablks, 612 int *rasizes, int nrablks, struct ucred *cred, struct buf **bpp) 613 { 614 struct buf *bp; 615 int i; 616 617 bp = *bpp = bio_doread(vp, blkno, size, cred, 0); 618 619 /* 620 * For each of the read-ahead blocks, start a read, if necessary. 621 */ 622 for (i = 0; i < nrablks; i++) { 623 /* If it's in the cache, just go on to next one. */ 624 if (incore(vp, rablks[i])) 625 continue; 626 627 /* Get a buffer for the read-ahead block */ 628 (void) bio_doread(vp, rablks[i], rasizes[i], cred, B_ASYNC); 629 } 630 631 /* Otherwise, we had to start a read for it; wait until it's valid. */ 632 return (biowait(bp)); 633 } 634 635 /* 636 * Read with single-block read-ahead. Defined in Bach (p.55), but 637 * implemented as a call to breadn(). 638 * XXX for compatibility with old file systems. 639 */ 640 int 641 breada(struct vnode *vp, daddr_t blkno, int size, daddr_t rablkno, 642 int rabsize, struct ucred *cred, struct buf **bpp) 643 { 644 645 return (breadn(vp, blkno, size, &rablkno, &rabsize, 1, cred, bpp)); 646 } 647 648 /* 649 * Block write. Described in Bach (p.56) 650 */ 651 int 652 bwrite(struct buf *bp) 653 { 654 int rv, sync, wasdelayed, s; 655 struct lwp *l = (curlwp != NULL ? curlwp : &lwp0); /* XXX */ 656 struct proc *p = l->l_proc; 657 struct vnode *vp; 658 struct mount *mp; 659 660 KASSERT(ISSET(bp->b_flags, B_BUSY)); 661 662 vp = bp->b_vp; 663 if (vp != NULL) { 664 if (vp->v_type == VBLK) 665 mp = vp->v_specmountpoint; 666 else 667 mp = vp->v_mount; 668 } else { 669 mp = NULL; 670 } 671 672 /* 673 * Remember buffer type, to switch on it later. If the write was 674 * synchronous, but the file system was mounted with MNT_ASYNC, 675 * convert it to a delayed write. 676 * XXX note that this relies on delayed tape writes being converted 677 * to async, not sync writes (which is safe, but ugly). 678 */ 679 sync = !ISSET(bp->b_flags, B_ASYNC); 680 if (sync && mp != NULL && ISSET(mp->mnt_flag, MNT_ASYNC)) { 681 bdwrite(bp); 682 return (0); 683 } 684 685 /* 686 * Collect statistics on synchronous and asynchronous writes. 687 * Writes to block devices are charged to their associated 688 * filesystem (if any). 689 */ 690 if (mp != NULL) { 691 if (sync) 692 mp->mnt_stat.f_syncwrites++; 693 else 694 mp->mnt_stat.f_asyncwrites++; 695 } 696 697 s = splbio(); 698 simple_lock(&bp->b_interlock); 699 700 wasdelayed = ISSET(bp->b_flags, B_DELWRI); 701 702 CLR(bp->b_flags, (B_READ | B_DONE | B_ERROR | B_DELWRI)); 703 704 /* 705 * Pay for the I/O operation and make sure the buf is on the correct 706 * vnode queue. 707 */ 708 if (wasdelayed) 709 reassignbuf(bp, bp->b_vp); 710 else 711 p->p_stats->p_ru.ru_oublock++; 712 713 /* Initiate disk write. Make sure the appropriate party is charged. */ 714 V_INCR_NUMOUTPUT(bp->b_vp); 715 simple_unlock(&bp->b_interlock); 716 splx(s); 717 718 if (sync) 719 BIO_SETPRIO(bp, BPRIO_TIMECRITICAL); 720 else 721 BIO_SETPRIO(bp, BPRIO_TIMELIMITED); 722 723 VOP_STRATEGY(vp, bp); 724 725 if (sync) { 726 /* If I/O was synchronous, wait for it to complete. */ 727 rv = biowait(bp); 728 729 /* Release the buffer. */ 730 brelse(bp); 731 732 return (rv); 733 } else { 734 return (0); 735 } 736 } 737 738 int 739 vn_bwrite(void *v) 740 { 741 struct vop_bwrite_args *ap = v; 742 743 return (bwrite(ap->a_bp)); 744 } 745 746 /* 747 * Delayed write. 748 * 749 * The buffer is marked dirty, but is not queued for I/O. 750 * This routine should be used when the buffer is expected 751 * to be modified again soon, typically a small write that 752 * partially fills a buffer. 753 * 754 * NB: magnetic tapes cannot be delayed; they must be 755 * written in the order that the writes are requested. 756 * 757 * Described in Leffler, et al. (pp. 208-213). 758 */ 759 void 760 bdwrite(struct buf *bp) 761 { 762 struct lwp *l = (curlwp != NULL ? curlwp : &lwp0); /* XXX */ 763 struct proc *p = l->l_proc; 764 const struct bdevsw *bdev; 765 int s; 766 767 /* If this is a tape block, write the block now. */ 768 bdev = bdevsw_lookup(bp->b_dev); 769 if (bdev != NULL && bdev->d_type == D_TAPE) { 770 bawrite(bp); 771 return; 772 } 773 774 /* 775 * If the block hasn't been seen before: 776 * (1) Mark it as having been seen, 777 * (2) Charge for the write, 778 * (3) Make sure it's on its vnode's correct block list. 779 */ 780 s = splbio(); 781 simple_lock(&bp->b_interlock); 782 783 KASSERT(ISSET(bp->b_flags, B_BUSY)); 784 785 if (!ISSET(bp->b_flags, B_DELWRI)) { 786 SET(bp->b_flags, B_DELWRI); 787 p->p_stats->p_ru.ru_oublock++; 788 reassignbuf(bp, bp->b_vp); 789 } 790 791 /* Otherwise, the "write" is done, so mark and release the buffer. */ 792 CLR(bp->b_flags, B_DONE); 793 simple_unlock(&bp->b_interlock); 794 splx(s); 795 796 brelse(bp); 797 } 798 799 /* 800 * Asynchronous block write; just an asynchronous bwrite(). 801 */ 802 void 803 bawrite(struct buf *bp) 804 { 805 int s; 806 807 s = splbio(); 808 simple_lock(&bp->b_interlock); 809 810 KASSERT(ISSET(bp->b_flags, B_BUSY)); 811 812 SET(bp->b_flags, B_ASYNC); 813 simple_unlock(&bp->b_interlock); 814 splx(s); 815 VOP_BWRITE(bp); 816 } 817 818 /* 819 * Same as first half of bdwrite, mark buffer dirty, but do not release it. 820 * Call at splbio() and with the buffer interlock locked. 821 * Note: called only from biodone() through ffs softdep's bioops.io_complete() 822 */ 823 void 824 bdirty(struct buf *bp) 825 { 826 struct lwp *l = (curlwp != NULL ? curlwp : &lwp0); /* XXX */ 827 struct proc *p = l->l_proc; 828 829 LOCK_ASSERT(simple_lock_held(&bp->b_interlock)); 830 KASSERT(ISSET(bp->b_flags, B_BUSY)); 831 832 CLR(bp->b_flags, B_AGE); 833 834 if (!ISSET(bp->b_flags, B_DELWRI)) { 835 SET(bp->b_flags, B_DELWRI); 836 p->p_stats->p_ru.ru_oublock++; 837 reassignbuf(bp, bp->b_vp); 838 } 839 } 840 841 /* 842 * Release a buffer on to the free lists. 843 * Described in Bach (p. 46). 844 */ 845 void 846 brelse(struct buf *bp) 847 { 848 struct bqueues *bufq; 849 int s; 850 851 /* Block disk interrupts. */ 852 s = splbio(); 853 simple_lock(&bqueue_slock); 854 simple_lock(&bp->b_interlock); 855 856 KASSERT(ISSET(bp->b_flags, B_BUSY)); 857 KASSERT(!ISSET(bp->b_flags, B_CALL)); 858 859 /* Wake up any processes waiting for any buffer to become free. */ 860 if (needbuffer) { 861 needbuffer = 0; 862 wakeup(&needbuffer); 863 } 864 865 /* Wake up any proceeses waiting for _this_ buffer to become free. */ 866 if (ISSET(bp->b_flags, B_WANTED)) { 867 CLR(bp->b_flags, B_WANTED|B_AGE); 868 wakeup(bp); 869 } 870 871 /* 872 * Determine which queue the buffer should be on, then put it there. 873 */ 874 875 /* If it's locked, don't report an error; try again later. */ 876 if (ISSET(bp->b_flags, (B_LOCKED|B_ERROR)) == (B_LOCKED|B_ERROR)) 877 CLR(bp->b_flags, B_ERROR); 878 879 /* If it's not cacheable, or an error, mark it invalid. */ 880 if (ISSET(bp->b_flags, (B_NOCACHE|B_ERROR))) 881 SET(bp->b_flags, B_INVAL); 882 883 if (ISSET(bp->b_flags, B_VFLUSH)) { 884 /* 885 * This is a delayed write buffer that was just flushed to 886 * disk. It is still on the LRU queue. If it's become 887 * invalid, then we need to move it to a different queue; 888 * otherwise leave it in its current position. 889 */ 890 CLR(bp->b_flags, B_VFLUSH); 891 if (!ISSET(bp->b_flags, B_ERROR|B_INVAL|B_LOCKED|B_AGE)) { 892 KDASSERT(!debug_verify_freelist || checkfreelist(bp, &bufqueues[BQ_LRU])); 893 goto already_queued; 894 } else { 895 bremfree(bp); 896 } 897 } 898 899 KDASSERT(!debug_verify_freelist || !checkfreelist(bp, &bufqueues[BQ_AGE])); 900 KDASSERT(!debug_verify_freelist || !checkfreelist(bp, &bufqueues[BQ_LRU])); 901 KDASSERT(!debug_verify_freelist || !checkfreelist(bp, &bufqueues[BQ_LOCKED])); 902 903 if ((bp->b_bufsize <= 0) || ISSET(bp->b_flags, B_INVAL)) { 904 /* 905 * If it's invalid or empty, dissociate it from its vnode 906 * and put on the head of the appropriate queue. 907 */ 908 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_deallocate) 909 (*bioops.io_deallocate)(bp); 910 CLR(bp->b_flags, B_DONE|B_DELWRI); 911 if (bp->b_vp) { 912 reassignbuf(bp, bp->b_vp); 913 brelvp(bp); 914 } 915 if (bp->b_bufsize <= 0) 916 /* no data */ 917 goto already_queued; 918 else 919 /* invalid data */ 920 bufq = &bufqueues[BQ_AGE]; 921 binsheadfree(bp, bufq); 922 } else { 923 /* 924 * It has valid data. Put it on the end of the appropriate 925 * queue, so that it'll stick around for as long as possible. 926 * If buf is AGE, but has dependencies, must put it on last 927 * bufqueue to be scanned, ie LRU. This protects against the 928 * livelock where BQ_AGE only has buffers with dependencies, 929 * and we thus never get to the dependent buffers in BQ_LRU. 930 */ 931 if (ISSET(bp->b_flags, B_LOCKED)) 932 /* locked in core */ 933 bufq = &bufqueues[BQ_LOCKED]; 934 else if (!ISSET(bp->b_flags, B_AGE)) 935 /* valid data */ 936 bufq = &bufqueues[BQ_LRU]; 937 else { 938 /* stale but valid data */ 939 int has_deps; 940 941 if (LIST_FIRST(&bp->b_dep) != NULL && 942 bioops.io_countdeps) 943 has_deps = (*bioops.io_countdeps)(bp, 0); 944 else 945 has_deps = 0; 946 bufq = has_deps ? &bufqueues[BQ_LRU] : 947 &bufqueues[BQ_AGE]; 948 } 949 binstailfree(bp, bufq); 950 } 951 952 already_queued: 953 /* Unlock the buffer. */ 954 CLR(bp->b_flags, B_AGE|B_ASYNC|B_BUSY|B_NOCACHE); 955 SET(bp->b_flags, B_CACHE); 956 957 /* Allow disk interrupts. */ 958 simple_unlock(&bp->b_interlock); 959 simple_unlock(&bqueue_slock); 960 if (bp->b_bufsize <= 0) { 961 #ifdef DEBUG 962 memset((char *)bp, 0, sizeof(*bp)); 963 #endif 964 pool_put(&bufpool, bp); 965 } 966 splx(s); 967 } 968 969 /* 970 * Determine if a block is in the cache. 971 * Just look on what would be its hash chain. If it's there, return 972 * a pointer to it, unless it's marked invalid. If it's marked invalid, 973 * we normally don't return the buffer, unless the caller explicitly 974 * wants us to. 975 */ 976 struct buf * 977 incore(struct vnode *vp, daddr_t blkno) 978 { 979 struct buf *bp; 980 981 /* Search hash chain */ 982 LIST_FOREACH(bp, BUFHASH(vp, blkno), b_hash) { 983 if (bp->b_lblkno == blkno && bp->b_vp == vp && 984 !ISSET(bp->b_flags, B_INVAL)) 985 return (bp); 986 } 987 988 return (NULL); 989 } 990 991 /* 992 * Get a block of requested size that is associated with 993 * a given vnode and block offset. If it is found in the 994 * block cache, mark it as having been found, make it busy 995 * and return it. Otherwise, return an empty block of the 996 * correct size. It is up to the caller to insure that the 997 * cached blocks be of the correct size. 998 */ 999 struct buf * 1000 getblk(struct vnode *vp, daddr_t blkno, int size, int slpflag, int slptimeo) 1001 { 1002 struct buf *bp; 1003 int s, err; 1004 int preserve; 1005 1006 start: 1007 s = splbio(); 1008 simple_lock(&bqueue_slock); 1009 bp = incore(vp, blkno); 1010 if (bp != NULL) { 1011 simple_lock(&bp->b_interlock); 1012 if (ISSET(bp->b_flags, B_BUSY)) { 1013 simple_unlock(&bqueue_slock); 1014 if (curproc == uvm.pagedaemon_proc) { 1015 simple_unlock(&bp->b_interlock); 1016 splx(s); 1017 return NULL; 1018 } 1019 SET(bp->b_flags, B_WANTED); 1020 err = ltsleep(bp, slpflag | (PRIBIO + 1) | PNORELOCK, 1021 "getblk", slptimeo, &bp->b_interlock); 1022 splx(s); 1023 if (err) 1024 return (NULL); 1025 goto start; 1026 } 1027 #ifdef DIAGNOSTIC 1028 if (ISSET(bp->b_flags, B_DONE|B_DELWRI) && 1029 bp->b_bcount < size && vp->v_type != VBLK) 1030 panic("getblk: block size invariant failed"); 1031 #endif 1032 SET(bp->b_flags, B_BUSY); 1033 bremfree(bp); 1034 preserve = 1; 1035 } else { 1036 if ((bp = getnewbuf(slpflag, slptimeo, 0)) == NULL) { 1037 simple_unlock(&bqueue_slock); 1038 splx(s); 1039 goto start; 1040 } 1041 1042 binshash(bp, BUFHASH(vp, blkno)); 1043 bp->b_blkno = bp->b_lblkno = bp->b_rawblkno = blkno; 1044 bgetvp(vp, bp); 1045 preserve = 0; 1046 } 1047 simple_unlock(&bp->b_interlock); 1048 simple_unlock(&bqueue_slock); 1049 splx(s); 1050 /* 1051 * LFS can't track total size of B_LOCKED buffer (locked_queue_bytes) 1052 * if we re-size buffers here. 1053 */ 1054 if (ISSET(bp->b_flags, B_LOCKED)) { 1055 KASSERT(bp->b_bufsize >= size); 1056 } else { 1057 allocbuf(bp, size, preserve); 1058 } 1059 BIO_SETPRIO(bp, BPRIO_DEFAULT); 1060 return (bp); 1061 } 1062 1063 /* 1064 * Get an empty, disassociated buffer of given size. 1065 */ 1066 struct buf * 1067 geteblk(int size) 1068 { 1069 struct buf *bp; 1070 int s; 1071 1072 s = splbio(); 1073 simple_lock(&bqueue_slock); 1074 while ((bp = getnewbuf(0, 0, 0)) == 0) 1075 ; 1076 1077 SET(bp->b_flags, B_INVAL); 1078 binshash(bp, &invalhash); 1079 simple_unlock(&bqueue_slock); 1080 simple_unlock(&bp->b_interlock); 1081 splx(s); 1082 BIO_SETPRIO(bp, BPRIO_DEFAULT); 1083 allocbuf(bp, size, 0); 1084 return (bp); 1085 } 1086 1087 /* 1088 * Expand or contract the actual memory allocated to a buffer. 1089 * 1090 * If the buffer shrinks, data is lost, so it's up to the 1091 * caller to have written it out *first*; this routine will not 1092 * start a write. If the buffer grows, it's the callers 1093 * responsibility to fill out the buffer's additional contents. 1094 */ 1095 void 1096 allocbuf(struct buf *bp, int size, int preserve) 1097 { 1098 vsize_t oldsize, desired_size; 1099 caddr_t addr; 1100 int s, delta; 1101 1102 desired_size = buf_roundsize(size); 1103 if (desired_size > MAXBSIZE) 1104 printf("allocbuf: buffer larger than MAXBSIZE requested"); 1105 1106 bp->b_bcount = size; 1107 1108 oldsize = bp->b_bufsize; 1109 if (oldsize == desired_size) 1110 return; 1111 1112 /* 1113 * If we want a buffer of a different size, re-allocate the 1114 * buffer's memory; copy old content only if needed. 1115 */ 1116 addr = buf_malloc(desired_size); 1117 if (preserve) 1118 memcpy(addr, bp->b_data, MIN(oldsize,desired_size)); 1119 if (bp->b_data != NULL) 1120 buf_mrelease(bp->b_data, oldsize); 1121 bp->b_data = addr; 1122 bp->b_bufsize = desired_size; 1123 1124 /* 1125 * Update overall buffer memory counter (protected by bqueue_slock) 1126 */ 1127 delta = (long)desired_size - (long)oldsize; 1128 1129 s = splbio(); 1130 simple_lock(&bqueue_slock); 1131 if ((bufmem += delta) > bufmem_hiwater) { 1132 /* 1133 * Need to trim overall memory usage. 1134 */ 1135 while (buf_canrelease()) { 1136 if (buf_trim() == 0) 1137 break; 1138 } 1139 } 1140 1141 simple_unlock(&bqueue_slock); 1142 splx(s); 1143 } 1144 1145 /* 1146 * Find a buffer which is available for use. 1147 * Select something from a free list. 1148 * Preference is to AGE list, then LRU list. 1149 * 1150 * Called at splbio and with buffer queues locked. 1151 * Return buffer locked. 1152 */ 1153 struct buf * 1154 getnewbuf(int slpflag, int slptimeo, int from_bufq) 1155 { 1156 struct buf *bp; 1157 1158 start: 1159 LOCK_ASSERT(simple_lock_held(&bqueue_slock)); 1160 1161 /* 1162 * Get a new buffer from the pool; but use NOWAIT because 1163 * we have the buffer queues locked. 1164 */ 1165 if (buf_lotsfree() && !from_bufq && 1166 (bp = pool_get(&bufpool, PR_NOWAIT)) != NULL) { 1167 memset((char *)bp, 0, sizeof(*bp)); 1168 BUF_INIT(bp); 1169 bp->b_dev = NODEV; 1170 bp->b_vnbufs.le_next = NOLIST; 1171 bp->b_flags = B_BUSY; 1172 simple_lock(&bp->b_interlock); 1173 return (bp); 1174 } 1175 1176 if ((bp = TAILQ_FIRST(&bufqueues[BQ_AGE])) != NULL || 1177 (bp = TAILQ_FIRST(&bufqueues[BQ_LRU])) != NULL) { 1178 simple_lock(&bp->b_interlock); 1179 bremfree(bp); 1180 } else { 1181 /* wait for a free buffer of any kind */ 1182 needbuffer = 1; 1183 ltsleep(&needbuffer, slpflag|(PRIBIO + 1), 1184 "getnewbuf", slptimeo, &bqueue_slock); 1185 return (NULL); 1186 } 1187 1188 #ifdef DIAGNOSTIC 1189 if (bp->b_bufsize <= 0) 1190 panic("buffer %p: on queue but empty", bp); 1191 #endif 1192 1193 if (ISSET(bp->b_flags, B_VFLUSH)) { 1194 /* 1195 * This is a delayed write buffer being flushed to disk. Make 1196 * sure it gets aged out of the queue when it's finished, and 1197 * leave it off the LRU queue. 1198 */ 1199 CLR(bp->b_flags, B_VFLUSH); 1200 SET(bp->b_flags, B_AGE); 1201 simple_unlock(&bp->b_interlock); 1202 goto start; 1203 } 1204 1205 /* Buffer is no longer on free lists. */ 1206 SET(bp->b_flags, B_BUSY); 1207 1208 /* 1209 * If buffer was a delayed write, start it and return NULL 1210 * (since we might sleep while starting the write). 1211 */ 1212 if (ISSET(bp->b_flags, B_DELWRI)) { 1213 /* 1214 * This buffer has gone through the LRU, so make sure it gets 1215 * reused ASAP. 1216 */ 1217 SET(bp->b_flags, B_AGE); 1218 simple_unlock(&bp->b_interlock); 1219 simple_unlock(&bqueue_slock); 1220 bawrite(bp); 1221 simple_lock(&bqueue_slock); 1222 return (NULL); 1223 } 1224 1225 /* disassociate us from our vnode, if we had one... */ 1226 if (bp->b_vp) 1227 brelvp(bp); 1228 1229 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_deallocate) 1230 (*bioops.io_deallocate)(bp); 1231 1232 /* clear out various other fields */ 1233 bp->b_flags = B_BUSY; 1234 bp->b_dev = NODEV; 1235 bp->b_blkno = bp->b_lblkno = bp->b_rawblkno = 0; 1236 bp->b_iodone = 0; 1237 bp->b_error = 0; 1238 bp->b_resid = 0; 1239 bp->b_bcount = 0; 1240 1241 bremhash(bp); 1242 return (bp); 1243 } 1244 1245 /* 1246 * Attempt to free an aged buffer off the queues. 1247 * Called at splbio and with queue lock held. 1248 * Returns the amount of buffer memory freed. 1249 */ 1250 int 1251 buf_trim(void) 1252 { 1253 struct buf *bp; 1254 long size = 0; 1255 1256 /* Instruct getnewbuf() to get buffers off the queues */ 1257 if ((bp = getnewbuf(PCATCH, 1, 1)) == NULL) 1258 return 0; 1259 1260 KASSERT(!ISSET(bp->b_flags, B_WANTED)); 1261 simple_unlock(&bp->b_interlock); 1262 size = bp->b_bufsize; 1263 bufmem -= size; 1264 simple_unlock(&bqueue_slock); 1265 if (size > 0) { 1266 buf_mrelease(bp->b_data, size); 1267 bp->b_bcount = bp->b_bufsize = 0; 1268 } 1269 /* brelse() will return the buffer to the global buffer pool */ 1270 brelse(bp); 1271 simple_lock(&bqueue_slock); 1272 return size; 1273 } 1274 1275 int 1276 buf_drain(int n) 1277 { 1278 int s, size = 0; 1279 1280 s = splbio(); 1281 simple_lock(&bqueue_slock); 1282 1283 /* If not asked for a specific amount, make our own estimate */ 1284 if (n == 0) 1285 n = buf_canrelease(); 1286 1287 while (size < n && bufmem > bufmem_lowater) 1288 size += buf_trim(); 1289 1290 simple_unlock(&bqueue_slock); 1291 splx(s); 1292 return size; 1293 } 1294 1295 /* 1296 * Wait for operations on the buffer to complete. 1297 * When they do, extract and return the I/O's error value. 1298 */ 1299 int 1300 biowait(struct buf *bp) 1301 { 1302 int s, error; 1303 1304 s = splbio(); 1305 simple_lock(&bp->b_interlock); 1306 while (!ISSET(bp->b_flags, B_DONE | B_DELWRI)) 1307 ltsleep(bp, PRIBIO + 1, "biowait", 0, &bp->b_interlock); 1308 1309 /* check for interruption of I/O (e.g. via NFS), then errors. */ 1310 if (ISSET(bp->b_flags, B_EINTR)) { 1311 CLR(bp->b_flags, B_EINTR); 1312 error = EINTR; 1313 } else if (ISSET(bp->b_flags, B_ERROR)) 1314 error = bp->b_error ? bp->b_error : EIO; 1315 else 1316 error = 0; 1317 1318 simple_unlock(&bp->b_interlock); 1319 splx(s); 1320 return (error); 1321 } 1322 1323 /* 1324 * Mark I/O complete on a buffer. 1325 * 1326 * If a callback has been requested, e.g. the pageout 1327 * daemon, do so. Otherwise, awaken waiting processes. 1328 * 1329 * [ Leffler, et al., says on p.247: 1330 * "This routine wakes up the blocked process, frees the buffer 1331 * for an asynchronous write, or, for a request by the pagedaemon 1332 * process, invokes a procedure specified in the buffer structure" ] 1333 * 1334 * In real life, the pagedaemon (or other system processes) wants 1335 * to do async stuff to, and doesn't want the buffer brelse()'d. 1336 * (for swap pager, that puts swap buffers on the free lists (!!!), 1337 * for the vn device, that puts malloc'd buffers on the free lists!) 1338 */ 1339 void 1340 biodone(struct buf *bp) 1341 { 1342 int s = splbio(); 1343 1344 simple_lock(&bp->b_interlock); 1345 if (ISSET(bp->b_flags, B_DONE)) 1346 panic("biodone already"); 1347 SET(bp->b_flags, B_DONE); /* note that it's done */ 1348 BIO_SETPRIO(bp, BPRIO_DEFAULT); 1349 1350 if (LIST_FIRST(&bp->b_dep) != NULL && bioops.io_complete) 1351 (*bioops.io_complete)(bp); 1352 1353 if (!ISSET(bp->b_flags, B_READ)) /* wake up reader */ 1354 vwakeup(bp); 1355 1356 /* 1357 * If necessary, call out. Unlock the buffer before calling 1358 * iodone() as the buffer isn't valid any more when it return. 1359 */ 1360 if (ISSET(bp->b_flags, B_CALL)) { 1361 CLR(bp->b_flags, B_CALL); /* but note callout done */ 1362 simple_unlock(&bp->b_interlock); 1363 (*bp->b_iodone)(bp); 1364 } else { 1365 if (ISSET(bp->b_flags, B_ASYNC)) { /* if async, release */ 1366 simple_unlock(&bp->b_interlock); 1367 brelse(bp); 1368 } else { /* or just wakeup the buffer */ 1369 CLR(bp->b_flags, B_WANTED); 1370 wakeup(bp); 1371 simple_unlock(&bp->b_interlock); 1372 } 1373 } 1374 1375 splx(s); 1376 } 1377 1378 /* 1379 * Return a count of buffers on the "locked" queue. 1380 */ 1381 int 1382 count_lock_queue(void) 1383 { 1384 struct buf *bp; 1385 int n = 0; 1386 1387 simple_lock(&bqueue_slock); 1388 TAILQ_FOREACH(bp, &bufqueues[BQ_LOCKED], b_freelist) 1389 n++; 1390 simple_unlock(&bqueue_slock); 1391 return (n); 1392 } 1393 1394 /* 1395 * Wait for all buffers to complete I/O 1396 * Return the number of "stuck" buffers. 1397 */ 1398 int 1399 buf_syncwait(void) 1400 { 1401 struct buf *bp; 1402 int iter, nbusy, nbusy_prev = 0, dcount, s, ihash; 1403 1404 dcount = 10000; 1405 for (iter = 0; iter < 20;) { 1406 s = splbio(); 1407 simple_lock(&bqueue_slock); 1408 nbusy = 0; 1409 for (ihash = 0; ihash < bufhash+1; ihash++) { 1410 LIST_FOREACH(bp, &bufhashtbl[ihash], b_hash) { 1411 if ((bp->b_flags & (B_BUSY|B_INVAL|B_READ)) == B_BUSY) 1412 nbusy++; 1413 /* 1414 * With soft updates, some buffers that are 1415 * written will be remarked as dirty until other 1416 * buffers are written. 1417 */ 1418 if (bp->b_vp && bp->b_vp->v_mount 1419 && (bp->b_vp->v_mount->mnt_flag & MNT_SOFTDEP) 1420 && (bp->b_flags & B_DELWRI)) { 1421 simple_lock(&bp->b_interlock); 1422 bremfree(bp); 1423 bp->b_flags |= B_BUSY; 1424 nbusy++; 1425 simple_unlock(&bp->b_interlock); 1426 simple_unlock(&bqueue_slock); 1427 bawrite(bp); 1428 if (dcount-- <= 0) { 1429 printf("softdep "); 1430 goto fail; 1431 } 1432 simple_lock(&bqueue_slock); 1433 } 1434 } 1435 } 1436 1437 simple_unlock(&bqueue_slock); 1438 splx(s); 1439 1440 if (nbusy == 0) 1441 break; 1442 if (nbusy_prev == 0) 1443 nbusy_prev = nbusy; 1444 printf("%d ", nbusy); 1445 tsleep(&nbusy, PRIBIO, "bflush", 1446 (iter == 0) ? 1 : hz / 25 * iter); 1447 if (nbusy >= nbusy_prev) /* we didn't flush anything */ 1448 iter++; 1449 else 1450 nbusy_prev = nbusy; 1451 } 1452 1453 if (nbusy) { 1454 fail:; 1455 #if defined(DEBUG) || defined(DEBUG_HALT_BUSY) 1456 printf("giving up\nPrinting vnodes for busy buffers\n"); 1457 for (ihash = 0; ihash < bufhash+1; ihash++) { 1458 LIST_FOREACH(bp, &bufhashtbl[ihash], b_hash) { 1459 if ((bp->b_flags & (B_BUSY|B_INVAL|B_READ)) == B_BUSY) 1460 vprint(NULL, bp->b_vp); 1461 } 1462 } 1463 #endif 1464 } 1465 1466 return nbusy; 1467 } 1468 1469 static void 1470 sysctl_fillbuf(struct buf *i, struct buf_sysctl *o) 1471 { 1472 1473 o->b_flags = i->b_flags; 1474 o->b_error = i->b_error; 1475 o->b_prio = i->b_prio; 1476 o->b_dev = i->b_dev; 1477 o->b_bufsize = i->b_bufsize; 1478 o->b_bcount = i->b_bcount; 1479 o->b_resid = i->b_resid; 1480 o->b_addr = PTRTOUINT64(i->b_un.b_addr); 1481 o->b_blkno = i->b_blkno; 1482 o->b_rawblkno = i->b_rawblkno; 1483 o->b_iodone = PTRTOUINT64(i->b_iodone); 1484 o->b_proc = PTRTOUINT64(i->b_proc); 1485 o->b_vp = PTRTOUINT64(i->b_vp); 1486 o->b_saveaddr = PTRTOUINT64(i->b_saveaddr); 1487 o->b_lblkno = i->b_lblkno; 1488 } 1489 1490 #define KERN_BUFSLOP 20 1491 static int 1492 sysctl_dobuf(SYSCTLFN_ARGS) 1493 { 1494 struct buf *bp; 1495 struct buf_sysctl bs; 1496 char *dp; 1497 u_int i, op, arg; 1498 size_t len, needed, elem_size, out_size; 1499 int error, s, elem_count; 1500 1501 if (namelen == 1 && name[0] == CTL_QUERY) 1502 return (sysctl_query(SYSCTLFN_CALL(rnode))); 1503 1504 if (namelen != 4) 1505 return (EINVAL); 1506 1507 dp = oldp; 1508 len = (oldp != NULL) ? *oldlenp : 0; 1509 op = name[0]; 1510 arg = name[1]; 1511 elem_size = name[2]; 1512 elem_count = name[3]; 1513 out_size = MIN(sizeof(bs), elem_size); 1514 1515 /* 1516 * at the moment, these are just "placeholders" to make the 1517 * API for retrieving kern.buf data more extensible in the 1518 * future. 1519 * 1520 * XXX kern.buf currently has "netbsd32" issues. hopefully 1521 * these will be resolved at a later point. 1522 */ 1523 if (op != KERN_BUF_ALL || arg != KERN_BUF_ALL || 1524 elem_size < 1 || elem_count < 0) 1525 return (EINVAL); 1526 1527 error = 0; 1528 needed = 0; 1529 s = splbio(); 1530 simple_lock(&bqueue_slock); 1531 for (i = 0; i < BQUEUES; i++) { 1532 TAILQ_FOREACH(bp, &bufqueues[i], b_freelist) { 1533 if (len >= elem_size && elem_count > 0) { 1534 sysctl_fillbuf(bp, &bs); 1535 error = copyout(&bs, dp, out_size); 1536 if (error) 1537 goto cleanup; 1538 dp += elem_size; 1539 len -= elem_size; 1540 } 1541 if (elem_count > 0) { 1542 needed += elem_size; 1543 if (elem_count != INT_MAX) 1544 elem_count--; 1545 } 1546 } 1547 } 1548 cleanup: 1549 simple_unlock(&bqueue_slock); 1550 splx(s); 1551 1552 *oldlenp = needed; 1553 if (oldp == NULL) 1554 *oldlenp += KERN_BUFSLOP * sizeof(struct buf); 1555 1556 return (error); 1557 } 1558 1559 static int 1560 sysctl_bufvm_update(SYSCTLFN_ARGS) 1561 { 1562 int t, error; 1563 struct sysctlnode node; 1564 1565 node = *rnode; 1566 node.sysctl_data = &t; 1567 t = *(int*)rnode->sysctl_data; 1568 error = sysctl_lookup(SYSCTLFN_CALL(&node)); 1569 if (error || newp == NULL) 1570 return (error); 1571 1572 if (rnode->sysctl_data == &bufcache) { 1573 if (t < 0 || t > 100) 1574 return (EINVAL); 1575 bufcache = t; 1576 bufmem_hiwater = buf_memcalc(); 1577 bufmem_lowater = (bufmem_hiwater >> 3); 1578 if (bufmem_lowater < 64 * 1024) 1579 /* Ensure a reasonable minimum value */ 1580 bufmem_lowater = 64 * 1024; 1581 1582 } else if (rnode->sysctl_data == &bufmem_lowater) { 1583 bufmem_lowater = t; 1584 } else if (rnode->sysctl_data == &bufmem_hiwater) { 1585 bufmem_hiwater = t; 1586 } else 1587 return (EINVAL); 1588 1589 /* Drain until below new high water mark */ 1590 while ((t = bufmem - bufmem_hiwater) >= 0) { 1591 if (buf_drain(t / (2*1024)) <= 0) 1592 break; 1593 } 1594 1595 return 0; 1596 } 1597 1598 SYSCTL_SETUP(sysctl_kern_buf_setup, "sysctl kern.buf subtree setup") 1599 { 1600 1601 sysctl_createv(clog, 0, NULL, NULL, 1602 CTLFLAG_PERMANENT, 1603 CTLTYPE_NODE, "kern", NULL, 1604 NULL, 0, NULL, 0, 1605 CTL_KERN, CTL_EOL); 1606 sysctl_createv(clog, 0, NULL, NULL, 1607 CTLFLAG_PERMANENT, 1608 CTLTYPE_NODE, "buf", 1609 SYSCTL_DESCR("Kernel buffer cache information"), 1610 sysctl_dobuf, 0, NULL, 0, 1611 CTL_KERN, KERN_BUF, CTL_EOL); 1612 } 1613 1614 SYSCTL_SETUP(sysctl_vm_buf_setup, "sysctl vm.buf* subtree setup") 1615 { 1616 1617 sysctl_createv(clog, 0, NULL, NULL, 1618 CTLFLAG_PERMANENT, 1619 CTLTYPE_NODE, "vm", NULL, 1620 NULL, 0, NULL, 0, 1621 CTL_VM, CTL_EOL); 1622 1623 sysctl_createv(clog, 0, NULL, NULL, 1624 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1625 CTLTYPE_INT, "bufcache", 1626 SYSCTL_DESCR("Percentage of kernel memory to use for " 1627 "buffer cache"), 1628 sysctl_bufvm_update, 0, &bufcache, 0, 1629 CTL_VM, CTL_CREATE, CTL_EOL); 1630 sysctl_createv(clog, 0, NULL, NULL, 1631 CTLFLAG_PERMANENT|CTLFLAG_READONLY, 1632 CTLTYPE_INT, "bufmem", 1633 SYSCTL_DESCR("Amount of kernel memory used by buffer " 1634 "cache"), 1635 NULL, 0, &bufmem, 0, 1636 CTL_VM, CTL_CREATE, CTL_EOL); 1637 sysctl_createv(clog, 0, NULL, NULL, 1638 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1639 CTLTYPE_INT, "bufmem_lowater", 1640 SYSCTL_DESCR("Minimum amount of kernel memory to " 1641 "reserve for buffer cache"), 1642 sysctl_bufvm_update, 0, &bufmem_lowater, 0, 1643 CTL_VM, CTL_CREATE, CTL_EOL); 1644 sysctl_createv(clog, 0, NULL, NULL, 1645 CTLFLAG_PERMANENT|CTLFLAG_READWRITE, 1646 CTLTYPE_INT, "bufmem_hiwater", 1647 SYSCTL_DESCR("Maximum amount of kernel memory to use " 1648 "for buffer cache"), 1649 sysctl_bufvm_update, 0, &bufmem_hiwater, 0, 1650 CTL_VM, CTL_CREATE, CTL_EOL); 1651 } 1652 1653 #ifdef DEBUG 1654 /* 1655 * Print out statistics on the current allocation of the buffer pool. 1656 * Can be enabled to print out on every ``sync'' by setting "syncprt" 1657 * in vfs_syscalls.c using sysctl. 1658 */ 1659 void 1660 vfs_bufstats(void) 1661 { 1662 int s, i, j, count; 1663 struct buf *bp; 1664 struct bqueues *dp; 1665 int counts[(MAXBSIZE / PAGE_SIZE) + 1]; 1666 static char *bname[BQUEUES] = { "LOCKED", "LRU", "AGE" }; 1667 1668 for (dp = bufqueues, i = 0; dp < &bufqueues[BQUEUES]; dp++, i++) { 1669 count = 0; 1670 for (j = 0; j <= MAXBSIZE/PAGE_SIZE; j++) 1671 counts[j] = 0; 1672 s = splbio(); 1673 TAILQ_FOREACH(bp, dp, b_freelist) { 1674 counts[bp->b_bufsize/PAGE_SIZE]++; 1675 count++; 1676 } 1677 splx(s); 1678 printf("%s: total-%d", bname[i], count); 1679 for (j = 0; j <= MAXBSIZE/PAGE_SIZE; j++) 1680 if (counts[j] != 0) 1681 printf(", %d-%d", j * PAGE_SIZE, counts[j]); 1682 printf("\n"); 1683 } 1684 } 1685 #endif /* DEBUG */ 1686