1 /* $OpenBSD: vfs_bio.c,v 1.190 2019/05/09 15:09:40 beck Exp $ */ 2 /* $NetBSD: vfs_bio.c,v 1.44 1996/06/11 11:15:36 pk Exp $ */ 3 4 /* 5 * Copyright (c) 1994 Christopher G. Demetriou 6 * Copyright (c) 1982, 1986, 1989, 1993 7 * The Regents of the University of California. All rights reserved. 8 * (c) UNIX System Laboratories, Inc. 9 * All or some portions of this file are derived from material licensed 10 * to the University of California by American Telephone and Telegraph 11 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 12 * the permission of UNIX System Laboratories, Inc. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 3. Neither the name of the University nor the names of its contributors 23 * may be used to endorse or promote products derived from this software 24 * without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 29 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 36 * SUCH DAMAGE. 37 * 38 * @(#)vfs_bio.c 8.6 (Berkeley) 1/11/94 39 */ 40 41 /* 42 * Some references: 43 * Bach: The Design of the UNIX Operating System (Prentice Hall, 1986) 44 * Leffler, et al.: The Design and Implementation of the 4.3BSD 45 * UNIX Operating System (Addison Welley, 1989) 46 */ 47 48 #include <sys/param.h> 49 #include <sys/systm.h> 50 #include <sys/proc.h> 51 #include <sys/buf.h> 52 #include <sys/vnode.h> 53 #include <sys/mount.h> 54 #include <sys/malloc.h> 55 #include <sys/pool.h> 56 #include <sys/resourcevar.h> 57 #include <sys/conf.h> 58 #include <sys/kernel.h> 59 #include <sys/specdev.h> 60 #include <uvm/uvm_extern.h> 61 62 /* XXX Should really be in buf.h, but for uvm_constraint_range.. */ 63 int buf_realloc_pages(struct buf *, struct uvm_constraint_range *, int); 64 65 struct uvm_constraint_range high_constraint; 66 int fliphigh; 67 68 int nobuffers; 69 int needbuffer; 70 struct bio_ops bioops; 71 72 /* private bufcache functions */ 73 void bufcache_init(void); 74 void bufcache_adjust(void); 75 struct buf *bufcache_gethighcleanbuf(void); 76 struct buf *bufcache_getdmacleanbuf(void); 77 78 /* 79 * Buffer pool for I/O buffers. 80 */ 81 struct pool bufpool; 82 struct bufhead bufhead = LIST_HEAD_INITIALIZER(bufhead); 83 void buf_put(struct buf *); 84 85 struct buf *bio_doread(struct vnode *, daddr_t, int, int); 86 struct buf *buf_get(struct vnode *, daddr_t, size_t); 87 void bread_cluster_callback(struct buf *); 88 int64_t bufcache_recover_dmapages(int discard, int64_t howmany); 89 90 struct bcachestats bcstats; /* counters */ 91 long lodirtypages; /* dirty page count low water mark */ 92 long hidirtypages; /* dirty page count high water mark */ 93 long targetpages; /* target number of pages for cache size */ 94 long buflowpages; /* smallest size cache allowed */ 95 long bufhighpages; /* largest size cache allowed */ 96 long bufbackpages; /* minimum number of pages we shrink when asked to */ 97 98 vsize_t bufkvm; 99 100 struct proc *cleanerproc; 101 int bd_req; /* Sleep point for cleaner daemon. */ 102 103 #define NUM_CACHES 2 104 #define DMA_CACHE 0 105 struct bufcache cleancache[NUM_CACHES]; 106 struct bufqueue dirtyqueue; 107 108 void 109 buf_put(struct buf *bp) 110 { 111 splassert(IPL_BIO); 112 113 #ifdef DIAGNOSTIC 114 if (bp->b_pobj != NULL) 115 KASSERT(bp->b_bufsize > 0); 116 if (ISSET(bp->b_flags, B_DELWRI)) 117 panic("buf_put: releasing dirty buffer"); 118 if (bp->b_freelist.tqe_next != NOLIST && 119 bp->b_freelist.tqe_next != (void *)-1) 120 panic("buf_put: still on the free list"); 121 if (bp->b_vnbufs.le_next != NOLIST && 122 bp->b_vnbufs.le_next != (void *)-1) 123 panic("buf_put: still on the vnode list"); 124 if (!LIST_EMPTY(&bp->b_dep)) 125 panic("buf_put: b_dep is not empty"); 126 #endif 127 128 LIST_REMOVE(bp, b_list); 129 bcstats.numbufs--; 130 131 if (buf_dealloc_mem(bp) != 0) 132 return; 133 pool_put(&bufpool, bp); 134 } 135 136 /* 137 * Initialize buffers and hash links for buffers. 138 */ 139 void 140 bufinit(void) 141 { 142 u_int64_t dmapages; 143 u_int64_t highpages; 144 145 dmapages = uvm_pagecount(&dma_constraint); 146 /* take away a guess at how much of this the kernel will consume */ 147 dmapages -= (atop(physmem) - atop(uvmexp.free)); 148 149 /* See if we have memory above the dma accessible region. */ 150 high_constraint.ucr_low = dma_constraint.ucr_high; 151 high_constraint.ucr_high = no_constraint.ucr_high; 152 if (high_constraint.ucr_low != high_constraint.ucr_high) 153 high_constraint.ucr_low++; 154 highpages = uvm_pagecount(&high_constraint); 155 156 /* 157 * Do we have any significant amount of high memory above 158 * the DMA region? if so enable moving buffers there, if not, 159 * don't bother. 160 */ 161 if (highpages > dmapages / 4) 162 fliphigh = 1; 163 else 164 fliphigh = 0; 165 166 /* 167 * If MD code doesn't say otherwise, use up to 10% of DMA'able 168 * memory for buffers. 169 */ 170 if (bufcachepercent == 0) 171 bufcachepercent = 10; 172 173 /* 174 * XXX these values and their same use in kern_sysctl 175 * need to move into buf.h 176 */ 177 KASSERT(bufcachepercent <= 90); 178 KASSERT(bufcachepercent >= 5); 179 if (bufpages == 0) 180 bufpages = dmapages * bufcachepercent / 100; 181 if (bufpages < BCACHE_MIN) 182 bufpages = BCACHE_MIN; 183 KASSERT(bufpages < dmapages); 184 185 bufhighpages = bufpages; 186 187 /* 188 * Set the base backoff level for the buffer cache. We will 189 * not allow uvm to steal back more than this number of pages. 190 */ 191 buflowpages = dmapages * 5 / 100; 192 if (buflowpages < BCACHE_MIN) 193 buflowpages = BCACHE_MIN; 194 195 /* 196 * set bufbackpages to 100 pages, or 10 percent of the low water mark 197 * if we don't have that many pages. 198 */ 199 200 bufbackpages = buflowpages * 10 / 100; 201 if (bufbackpages > 100) 202 bufbackpages = 100; 203 204 /* 205 * If the MD code does not say otherwise, reserve 10% of kva 206 * space for mapping buffers. 207 */ 208 if (bufkvm == 0) 209 bufkvm = VM_KERNEL_SPACE_SIZE / 10; 210 211 /* 212 * Don't use more than twice the amount of bufpages for mappings. 213 * It's twice since we map things sparsely. 214 */ 215 if (bufkvm > bufpages * PAGE_SIZE) 216 bufkvm = bufpages * PAGE_SIZE; 217 /* 218 * Round bufkvm to MAXPHYS because we allocate chunks of va space 219 * in MAXPHYS chunks. 220 */ 221 bufkvm &= ~(MAXPHYS - 1); 222 223 pool_init(&bufpool, sizeof(struct buf), 0, IPL_BIO, 0, "bufpl", NULL); 224 225 bufcache_init(); 226 227 /* 228 * hmm - bufkvm is an argument because it's static, while 229 * bufpages is global because it can change while running. 230 */ 231 buf_mem_init(bufkvm); 232 233 /* 234 * Set the dirty page high water mark to be less than the low 235 * water mark for pages in the buffer cache. This ensures we 236 * can always back off by throwing away clean pages, and give 237 * ourselves a chance to write out the dirty pages eventually. 238 */ 239 hidirtypages = (buflowpages / 4) * 3; 240 lodirtypages = buflowpages / 2; 241 242 /* 243 * We are allowed to use up to the reserve. 244 */ 245 targetpages = bufpages - RESERVE_PAGES; 246 } 247 248 /* 249 * Change cachepct 250 */ 251 void 252 bufadjust(int newbufpages) 253 { 254 int s; 255 int64_t npages; 256 257 if (newbufpages < buflowpages) 258 newbufpages = buflowpages; 259 260 s = splbio(); 261 bufpages = newbufpages; 262 263 /* 264 * We are allowed to use up to the reserve 265 */ 266 targetpages = bufpages - RESERVE_PAGES; 267 268 npages = bcstats.dmapages - targetpages; 269 270 /* 271 * Shrinking the cache happens here only if someone has manually 272 * adjusted bufcachepercent - or the pagedaemon has told us 273 * to give back memory *now* - so we give it all back. 274 */ 275 if (bcstats.dmapages > targetpages) 276 (void) bufcache_recover_dmapages(0, bcstats.dmapages - targetpages); 277 bufcache_adjust(); 278 279 /* 280 * Wake up the cleaner if we have lots of dirty pages, 281 * or if we are getting low on buffer cache kva. 282 */ 283 if ((UNCLEAN_PAGES >= hidirtypages) || 284 bcstats.kvaslots_avail <= 2 * RESERVE_SLOTS) 285 wakeup(&bd_req); 286 287 splx(s); 288 } 289 290 /* 291 * Make the buffer cache back off from cachepct. 292 */ 293 int 294 bufbackoff(struct uvm_constraint_range *range, long size) 295 { 296 /* 297 * Back off "size" buffer cache pages. Called by the page 298 * daemon to consume buffer cache pages rather than scanning. 299 * 300 * It returns 0 to the pagedaemon to indicate that it has 301 * succeeded in freeing enough pages. It returns -1 to 302 * indicate that it could not and the pagedaemon should take 303 * other measures. 304 * 305 */ 306 long pdelta, oldbufpages; 307 308 /* 309 * If we will accept high memory for this backoff 310 * try to steal it from the high memory buffer cache. 311 */ 312 if (range->ucr_high > dma_constraint.ucr_high) { 313 struct buf *bp; 314 int64_t start = bcstats.numbufpages, recovered = 0; 315 int s = splbio(); 316 317 while ((recovered < size) && 318 (bp = bufcache_gethighcleanbuf())) { 319 bufcache_take(bp); 320 if (bp->b_vp) { 321 RBT_REMOVE(buf_rb_bufs, 322 &bp->b_vp->v_bufs_tree, bp); 323 brelvp(bp); 324 } 325 buf_put(bp); 326 recovered = start - bcstats.numbufpages; 327 } 328 bufcache_adjust(); 329 splx(s); 330 331 /* If we got enough, return success */ 332 if (recovered >= size) 333 return 0; 334 335 /* 336 * If we needed only memory above DMA, 337 * return failure 338 */ 339 if (range->ucr_low > dma_constraint.ucr_high) 340 return -1; 341 342 /* Otherwise get the rest from DMA */ 343 size -= recovered; 344 } 345 346 /* 347 * XXX Otherwise do the dma memory cache dance. this needs 348 * refactoring later to get rid of 'bufpages' 349 */ 350 351 /* 352 * Back off by at least bufbackpages. If the page daemon gave us 353 * a larger size, back off by that much. 354 */ 355 pdelta = (size > bufbackpages) ? size : bufbackpages; 356 357 if (bufpages <= buflowpages) 358 return(-1); 359 if (bufpages - pdelta < buflowpages) 360 pdelta = bufpages - buflowpages; 361 oldbufpages = bufpages; 362 bufadjust(bufpages - pdelta); 363 if (oldbufpages - bufpages < size) 364 return (-1); /* we did not free what we were asked */ 365 else 366 return(0); 367 } 368 369 370 /* 371 * Opportunistically flip a buffer into high memory. Will move the buffer 372 * if memory is available without sleeping, and return 0, otherwise will 373 * fail and return -1 with the buffer unchanged. 374 */ 375 376 int 377 buf_flip_high(struct buf *bp) 378 { 379 int s; 380 int ret = -1; 381 382 KASSERT(ISSET(bp->b_flags, B_BC)); 383 KASSERT(ISSET(bp->b_flags, B_DMA)); 384 KASSERT(bp->cache == DMA_CACHE); 385 KASSERT(fliphigh); 386 387 /* Attempt to move the buffer to high memory if we can */ 388 s = splbio(); 389 if (buf_realloc_pages(bp, &high_constraint, UVM_PLA_NOWAIT) == 0) { 390 KASSERT(!ISSET(bp->b_flags, B_DMA)); 391 bcstats.highflips++; 392 ret = 0; 393 } else 394 bcstats.highflops++; 395 splx(s); 396 397 return ret; 398 } 399 400 /* 401 * Flip a buffer to dma reachable memory, when we need it there for 402 * I/O. This can sleep since it will wait for memory alloacation in the 403 * DMA reachable area since we have to have the buffer there to proceed. 404 */ 405 void 406 buf_flip_dma(struct buf *bp) 407 { 408 KASSERT(ISSET(bp->b_flags, B_BC)); 409 KASSERT(ISSET(bp->b_flags, B_BUSY)); 410 KASSERT(bp->cache < NUM_CACHES); 411 412 if (!ISSET(bp->b_flags, B_DMA)) { 413 int s = splbio(); 414 415 /* move buf to dma reachable memory */ 416 (void) buf_realloc_pages(bp, &dma_constraint, UVM_PLA_WAITOK); 417 KASSERT(ISSET(bp->b_flags, B_DMA)); 418 bcstats.dmaflips++; 419 splx(s); 420 } 421 422 if (bp->cache > DMA_CACHE) { 423 CLR(bp->b_flags, B_COLD); 424 CLR(bp->b_flags, B_WARM); 425 bp->cache = DMA_CACHE; 426 } 427 } 428 429 struct buf * 430 bio_doread(struct vnode *vp, daddr_t blkno, int size, int async) 431 { 432 struct buf *bp; 433 struct mount *mp; 434 435 bp = getblk(vp, blkno, size, 0, 0); 436 437 /* 438 * If buffer does not have valid data, start a read. 439 * Note that if buffer is B_INVAL, getblk() won't return it. 440 * Therefore, it's valid if its I/O has completed or been delayed. 441 */ 442 if (!ISSET(bp->b_flags, (B_DONE | B_DELWRI))) { 443 SET(bp->b_flags, B_READ | async); 444 bcstats.pendingreads++; 445 bcstats.numreads++; 446 VOP_STRATEGY(bp); 447 /* Pay for the read. */ 448 curproc->p_ru.ru_inblock++; /* XXX */ 449 } else if (async) { 450 brelse(bp); 451 } 452 453 mp = vp->v_type == VBLK? vp->v_specmountpoint : vp->v_mount; 454 455 /* 456 * Collect statistics on synchronous and asynchronous reads. 457 * Reads from block devices are charged to their associated 458 * filesystem (if any). 459 */ 460 if (mp != NULL) { 461 if (async == 0) 462 mp->mnt_stat.f_syncreads++; 463 else 464 mp->mnt_stat.f_asyncreads++; 465 } 466 467 return (bp); 468 } 469 470 /* 471 * Read a disk block. 472 * This algorithm described in Bach (p.54). 473 */ 474 int 475 bread(struct vnode *vp, daddr_t blkno, int size, struct buf **bpp) 476 { 477 struct buf *bp; 478 479 /* Get buffer for block. */ 480 bp = *bpp = bio_doread(vp, blkno, size, 0); 481 482 /* Wait for the read to complete, and return result. */ 483 return (biowait(bp)); 484 } 485 486 /* 487 * Read-ahead multiple disk blocks. The first is sync, the rest async. 488 * Trivial modification to the breada algorithm presented in Bach (p.55). 489 */ 490 int 491 breadn(struct vnode *vp, daddr_t blkno, int size, daddr_t rablks[], 492 int rasizes[], int nrablks, struct buf **bpp) 493 { 494 struct buf *bp; 495 int i; 496 497 bp = *bpp = bio_doread(vp, blkno, size, 0); 498 499 /* 500 * For each of the read-ahead blocks, start a read, if necessary. 501 */ 502 for (i = 0; i < nrablks; i++) { 503 /* If it's in the cache, just go on to next one. */ 504 if (incore(vp, rablks[i])) 505 continue; 506 507 /* Get a buffer for the read-ahead block */ 508 (void) bio_doread(vp, rablks[i], rasizes[i], B_ASYNC); 509 } 510 511 /* Otherwise, we had to start a read for it; wait until it's valid. */ 512 return (biowait(bp)); 513 } 514 515 /* 516 * Called from interrupt context. 517 */ 518 void 519 bread_cluster_callback(struct buf *bp) 520 { 521 struct buf **xbpp = bp->b_saveaddr; 522 int i; 523 524 if (xbpp[1] != NULL) { 525 size_t newsize = xbpp[1]->b_bufsize; 526 527 /* 528 * Shrink this buffer's mapping to only cover its part of 529 * the total I/O. 530 */ 531 buf_fix_mapping(bp, newsize); 532 bp->b_bcount = newsize; 533 } 534 535 /* Invalidate read-ahead buffers if read short */ 536 if (bp->b_resid > 0) { 537 for (i = 1; xbpp[i] != NULL; i++) 538 continue; 539 for (i = i - 1; i != 0; i--) { 540 if (xbpp[i]->b_bufsize <= bp->b_resid) { 541 bp->b_resid -= xbpp[i]->b_bufsize; 542 SET(xbpp[i]->b_flags, B_INVAL); 543 } else if (bp->b_resid > 0) { 544 bp->b_resid = 0; 545 SET(xbpp[i]->b_flags, B_INVAL); 546 } else 547 break; 548 } 549 } 550 551 for (i = 1; xbpp[i] != NULL; i++) { 552 if (ISSET(bp->b_flags, B_ERROR)) 553 SET(xbpp[i]->b_flags, B_INVAL | B_ERROR); 554 biodone(xbpp[i]); 555 } 556 557 free(xbpp, M_TEMP, (i + 1) * sizeof(*xbpp)); 558 559 if (ISSET(bp->b_flags, B_ASYNC)) { 560 brelse(bp); 561 } else { 562 CLR(bp->b_flags, B_WANTED); 563 wakeup(bp); 564 } 565 } 566 567 /* 568 * Read-ahead multiple disk blocks, but make sure only one (big) I/O 569 * request is sent to the disk. 570 * XXX This should probably be dropped and breadn should instead be optimized 571 * XXX to do fewer I/O requests. 572 */ 573 int 574 bread_cluster(struct vnode *vp, daddr_t blkno, int size, struct buf **rbpp) 575 { 576 struct buf *bp, **xbpp; 577 int howmany, maxra, i, inc; 578 daddr_t sblkno; 579 580 *rbpp = bio_doread(vp, blkno, size, 0); 581 582 /* 583 * If the buffer is in the cache skip any I/O operation. 584 */ 585 if (ISSET((*rbpp)->b_flags, B_CACHE)) 586 goto out; 587 588 if (size != round_page(size)) 589 goto out; 590 591 if (VOP_BMAP(vp, blkno + 1, NULL, &sblkno, &maxra)) 592 goto out; 593 594 maxra++; 595 if (sblkno == -1 || maxra < 2) 596 goto out; 597 598 howmany = MAXPHYS / size; 599 if (howmany > maxra) 600 howmany = maxra; 601 602 xbpp = mallocarray(howmany + 1, sizeof(*xbpp), M_TEMP, M_NOWAIT); 603 if (xbpp == NULL) 604 goto out; 605 606 for (i = howmany - 1; i >= 0; i--) { 607 size_t sz; 608 609 /* 610 * First buffer allocates big enough size to cover what 611 * all the other buffers need. 612 */ 613 sz = i == 0 ? howmany * size : 0; 614 615 xbpp[i] = buf_get(vp, blkno + i + 1, sz); 616 if (xbpp[i] == NULL) { 617 for (++i; i < howmany; i++) { 618 SET(xbpp[i]->b_flags, B_INVAL); 619 brelse(xbpp[i]); 620 } 621 free(xbpp, M_TEMP, (howmany + 1) * sizeof(*xbpp)); 622 goto out; 623 } 624 } 625 626 bp = xbpp[0]; 627 628 xbpp[howmany] = NULL; 629 630 inc = btodb(size); 631 632 for (i = 1; i < howmany; i++) { 633 bcstats.pendingreads++; 634 bcstats.numreads++; 635 /* 636 * We set B_DMA here because bp above will be B_DMA, 637 * and we are playing buffer slice-n-dice games from 638 * the memory allocated in bp. 639 */ 640 SET(xbpp[i]->b_flags, B_DMA | B_READ | B_ASYNC); 641 xbpp[i]->b_blkno = sblkno + (i * inc); 642 xbpp[i]->b_bufsize = xbpp[i]->b_bcount = size; 643 xbpp[i]->b_data = NULL; 644 xbpp[i]->b_pobj = bp->b_pobj; 645 xbpp[i]->b_poffs = bp->b_poffs + (i * size); 646 } 647 648 KASSERT(bp->b_lblkno == blkno + 1); 649 KASSERT(bp->b_vp == vp); 650 651 bp->b_blkno = sblkno; 652 SET(bp->b_flags, B_READ | B_ASYNC | B_CALL); 653 654 bp->b_saveaddr = (void *)xbpp; 655 bp->b_iodone = bread_cluster_callback; 656 657 bcstats.pendingreads++; 658 bcstats.numreads++; 659 VOP_STRATEGY(bp); 660 curproc->p_ru.ru_inblock++; 661 662 out: 663 return (biowait(*rbpp)); 664 } 665 666 /* 667 * Block write. Described in Bach (p.56) 668 */ 669 int 670 bwrite(struct buf *bp) 671 { 672 int rv, async, wasdelayed, s; 673 struct vnode *vp; 674 struct mount *mp; 675 676 vp = bp->b_vp; 677 if (vp != NULL) 678 mp = vp->v_type == VBLK? vp->v_specmountpoint : vp->v_mount; 679 else 680 mp = NULL; 681 682 /* 683 * Remember buffer type, to switch on it later. If the write was 684 * synchronous, but the file system was mounted with MNT_ASYNC, 685 * convert it to a delayed write. 686 * XXX note that this relies on delayed tape writes being converted 687 * to async, not sync writes (which is safe, but ugly). 688 */ 689 async = ISSET(bp->b_flags, B_ASYNC); 690 if (!async && mp && ISSET(mp->mnt_flag, MNT_ASYNC)) { 691 bdwrite(bp); 692 return (0); 693 } 694 695 /* 696 * Collect statistics on synchronous and asynchronous writes. 697 * Writes to block devices are charged to their associated 698 * filesystem (if any). 699 */ 700 if (mp != NULL) { 701 if (async) 702 mp->mnt_stat.f_asyncwrites++; 703 else 704 mp->mnt_stat.f_syncwrites++; 705 } 706 bcstats.pendingwrites++; 707 bcstats.numwrites++; 708 709 wasdelayed = ISSET(bp->b_flags, B_DELWRI); 710 CLR(bp->b_flags, (B_READ | B_DONE | B_ERROR | B_DELWRI)); 711 712 s = splbio(); 713 714 /* 715 * If not synchronous, pay for the I/O operation and make 716 * sure the buf is on the correct vnode queue. We have 717 * to do this now, because if we don't, the vnode may not 718 * be properly notified that its I/O has completed. 719 */ 720 if (wasdelayed) { 721 reassignbuf(bp); 722 } else 723 curproc->p_ru.ru_oublock++; 724 725 726 /* Initiate disk write. Make sure the appropriate party is charged. */ 727 bp->b_vp->v_numoutput++; 728 splx(s); 729 buf_flip_dma(bp); 730 SET(bp->b_flags, B_WRITEINPROG); 731 VOP_STRATEGY(bp); 732 733 /* 734 * If the queue is above the high water mark, wait till 735 * the number of outstanding write bufs drops below the low 736 * water mark. 737 */ 738 if (bp->b_bq) 739 bufq_wait(bp->b_bq); 740 741 if (async) 742 return (0); 743 744 /* 745 * If I/O was synchronous, wait for it to complete. 746 */ 747 rv = biowait(bp); 748 749 /* Release the buffer. */ 750 brelse(bp); 751 752 return (rv); 753 } 754 755 756 /* 757 * Delayed write. 758 * 759 * The buffer is marked dirty, but is not queued for I/O. 760 * This routine should be used when the buffer is expected 761 * to be modified again soon, typically a small write that 762 * partially fills a buffer. 763 * 764 * NB: magnetic tapes cannot be delayed; they must be 765 * written in the order that the writes are requested. 766 * 767 * Described in Leffler, et al. (pp. 208-213). 768 */ 769 void 770 bdwrite(struct buf *bp) 771 { 772 int s; 773 774 /* 775 * If the block hasn't been seen before: 776 * (1) Mark it as having been seen, 777 * (2) Charge for the write. 778 * (3) Make sure it's on its vnode's correct block list, 779 * (4) If a buffer is rewritten, move it to end of dirty list 780 */ 781 if (!ISSET(bp->b_flags, B_DELWRI)) { 782 SET(bp->b_flags, B_DELWRI); 783 s = splbio(); 784 buf_flip_dma(bp); 785 reassignbuf(bp); 786 splx(s); 787 curproc->p_ru.ru_oublock++; /* XXX */ 788 } 789 790 /* The "write" is done, so mark and release the buffer. */ 791 CLR(bp->b_flags, B_NEEDCOMMIT); 792 SET(bp->b_flags, B_DONE); 793 brelse(bp); 794 } 795 796 /* 797 * Asynchronous block write; just an asynchronous bwrite(). 798 */ 799 void 800 bawrite(struct buf *bp) 801 { 802 803 SET(bp->b_flags, B_ASYNC); 804 VOP_BWRITE(bp); 805 } 806 807 /* 808 * Must be called at splbio() 809 */ 810 void 811 buf_dirty(struct buf *bp) 812 { 813 splassert(IPL_BIO); 814 815 #ifdef DIAGNOSTIC 816 if (!ISSET(bp->b_flags, B_BUSY)) 817 panic("Trying to dirty buffer on freelist!"); 818 #endif 819 820 if (ISSET(bp->b_flags, B_DELWRI) == 0) { 821 SET(bp->b_flags, B_DELWRI); 822 buf_flip_dma(bp); 823 reassignbuf(bp); 824 } 825 } 826 827 /* 828 * Must be called at splbio() 829 */ 830 void 831 buf_undirty(struct buf *bp) 832 { 833 splassert(IPL_BIO); 834 835 #ifdef DIAGNOSTIC 836 if (!ISSET(bp->b_flags, B_BUSY)) 837 panic("Trying to undirty buffer on freelist!"); 838 #endif 839 if (ISSET(bp->b_flags, B_DELWRI)) { 840 CLR(bp->b_flags, B_DELWRI); 841 reassignbuf(bp); 842 } 843 } 844 845 /* 846 * Release a buffer on to the free lists. 847 * Described in Bach (p. 46). 848 */ 849 void 850 brelse(struct buf *bp) 851 { 852 int s; 853 854 s = splbio(); 855 856 if (bp->b_data != NULL) 857 KASSERT(bp->b_bufsize > 0); 858 859 /* 860 * Determine which queue the buffer should be on, then put it there. 861 */ 862 863 /* If it's not cacheable, or an error, mark it invalid. */ 864 if (ISSET(bp->b_flags, (B_NOCACHE|B_ERROR))) 865 SET(bp->b_flags, B_INVAL); 866 /* If it's a write error, also mark the vnode as damaged. */ 867 if (ISSET(bp->b_flags, B_ERROR) && !ISSET(bp->b_flags, B_READ)) { 868 if (bp->b_vp && bp->b_vp->v_type == VREG) 869 SET(bp->b_vp->v_bioflag, VBIOERROR); 870 } 871 872 if (ISSET(bp->b_flags, B_INVAL)) { 873 /* 874 * If the buffer is invalid, free it now rather than leaving 875 * it in a queue and wasting memory. 876 */ 877 if (LIST_FIRST(&bp->b_dep) != NULL) 878 buf_deallocate(bp); 879 880 if (ISSET(bp->b_flags, B_DELWRI)) { 881 CLR(bp->b_flags, B_DELWRI); 882 } 883 884 if (bp->b_vp) { 885 RBT_REMOVE(buf_rb_bufs, &bp->b_vp->v_bufs_tree, bp); 886 brelvp(bp); 887 } 888 bp->b_vp = NULL; 889 890 /* 891 * Wake up any processes waiting for _this_ buffer to 892 * become free. They are not allowed to grab it 893 * since it will be freed. But the only sleeper is 894 * getblk and it will restart the operation after 895 * sleep. 896 */ 897 if (ISSET(bp->b_flags, B_WANTED)) { 898 CLR(bp->b_flags, B_WANTED); 899 wakeup(bp); 900 } 901 buf_put(bp); 902 } else { 903 /* 904 * It has valid data. Put it on the end of the appropriate 905 * queue, so that it'll stick around for as long as possible. 906 */ 907 bufcache_release(bp); 908 909 /* Unlock the buffer. */ 910 CLR(bp->b_flags, (B_AGE | B_ASYNC | B_NOCACHE | B_DEFERRED)); 911 buf_release(bp); 912 913 /* Wake up any processes waiting for _this_ buffer to 914 * become free. */ 915 if (ISSET(bp->b_flags, B_WANTED)) { 916 CLR(bp->b_flags, B_WANTED); 917 wakeup(bp); 918 } 919 } 920 921 /* Wake up syncer and cleaner processes waiting for buffers. */ 922 if (nobuffers) { 923 nobuffers = 0; 924 wakeup(&nobuffers); 925 } 926 927 /* Wake up any processes waiting for any buffer to become free. */ 928 if (needbuffer && bcstats.dmapages < targetpages && 929 bcstats.kvaslots_avail > RESERVE_SLOTS) { 930 needbuffer = 0; 931 wakeup(&needbuffer); 932 } 933 934 splx(s); 935 } 936 937 /* 938 * Determine if a block is in the cache. Just look on what would be its hash 939 * chain. If it's there, return a pointer to it, unless it's marked invalid. 940 */ 941 struct buf * 942 incore(struct vnode *vp, daddr_t blkno) 943 { 944 struct buf *bp; 945 struct buf b; 946 int s; 947 948 s = splbio(); 949 950 /* Search buf lookup tree */ 951 b.b_lblkno = blkno; 952 bp = RBT_FIND(buf_rb_bufs, &vp->v_bufs_tree, &b); 953 if (bp != NULL && ISSET(bp->b_flags, B_INVAL)) 954 bp = NULL; 955 956 splx(s); 957 return (bp); 958 } 959 960 /* 961 * Get a block of requested size that is associated with 962 * a given vnode and block offset. If it is found in the 963 * block cache, mark it as having been found, make it busy 964 * and return it. Otherwise, return an empty block of the 965 * correct size. It is up to the caller to ensure that the 966 * cached blocks be of the correct size. 967 */ 968 struct buf * 969 getblk(struct vnode *vp, daddr_t blkno, int size, int slpflag, int slptimeo) 970 { 971 struct buf *bp; 972 struct buf b; 973 int s, error; 974 975 /* 976 * XXX 977 * The following is an inlined version of 'incore()', but with 978 * the 'invalid' test moved to after the 'busy' test. It's 979 * necessary because there are some cases in which the NFS 980 * code sets B_INVAL prior to writing data to the server, but 981 * in which the buffers actually contain valid data. In this 982 * case, we can't allow the system to allocate a new buffer for 983 * the block until the write is finished. 984 */ 985 start: 986 s = splbio(); 987 b.b_lblkno = blkno; 988 bp = RBT_FIND(buf_rb_bufs, &vp->v_bufs_tree, &b); 989 if (bp != NULL) { 990 if (ISSET(bp->b_flags, B_BUSY)) { 991 SET(bp->b_flags, B_WANTED); 992 error = tsleep(bp, slpflag | (PRIBIO + 1), "getblk", 993 slptimeo); 994 splx(s); 995 if (error) 996 return (NULL); 997 goto start; 998 } 999 1000 if (!ISSET(bp->b_flags, B_INVAL)) { 1001 bcstats.cachehits++; 1002 SET(bp->b_flags, B_CACHE); 1003 bufcache_take(bp); 1004 buf_acquire(bp); 1005 splx(s); 1006 return (bp); 1007 } 1008 } 1009 splx(s); 1010 1011 if ((bp = buf_get(vp, blkno, size)) == NULL) 1012 goto start; 1013 1014 return (bp); 1015 } 1016 1017 /* 1018 * Get an empty, disassociated buffer of given size. 1019 */ 1020 struct buf * 1021 geteblk(size_t size) 1022 { 1023 struct buf *bp; 1024 1025 while ((bp = buf_get(NULL, 0, size)) == NULL) 1026 continue; 1027 1028 return (bp); 1029 } 1030 1031 /* 1032 * Allocate a buffer. 1033 * If vp is given, put it into the buffer cache for that vnode. 1034 * If size != 0, allocate memory and call buf_map(). 1035 * If there is already a buffer for the given vnode/blkno, return NULL. 1036 */ 1037 struct buf * 1038 buf_get(struct vnode *vp, daddr_t blkno, size_t size) 1039 { 1040 struct buf *bp; 1041 int poolwait = size == 0 ? PR_NOWAIT : PR_WAITOK; 1042 int npages; 1043 int s; 1044 1045 s = splbio(); 1046 if (size) { 1047 /* 1048 * Wake up the cleaner if we have lots of dirty pages, 1049 * or if we are getting low on buffer cache kva. 1050 */ 1051 if (UNCLEAN_PAGES >= hidirtypages || 1052 bcstats.kvaslots_avail <= 2 * RESERVE_SLOTS) 1053 wakeup(&bd_req); 1054 1055 npages = atop(round_page(size)); 1056 1057 /* 1058 * if our cache has been previously shrunk, 1059 * allow it to grow again with use up to 1060 * bufhighpages (cachepercent) 1061 */ 1062 if (bufpages < bufhighpages) 1063 bufadjust(bufhighpages); 1064 1065 /* 1066 * If we would go over the page target with our 1067 * new allocation, free enough buffers first 1068 * to stay at the target with our new allocation. 1069 */ 1070 if (bcstats.dmapages + npages > targetpages) { 1071 (void) bufcache_recover_dmapages(0, npages); 1072 bufcache_adjust(); 1073 } 1074 1075 /* 1076 * If we get here, we tried to free the world down 1077 * above, and couldn't get down - Wake the cleaner 1078 * and wait for it to push some buffers out. 1079 */ 1080 if ((bcstats.dmapages + npages > targetpages || 1081 bcstats.kvaslots_avail <= RESERVE_SLOTS) && 1082 curproc != syncerproc && curproc != cleanerproc) { 1083 wakeup(&bd_req); 1084 needbuffer++; 1085 tsleep(&needbuffer, PRIBIO, "needbuffer", 0); 1086 splx(s); 1087 return (NULL); 1088 } 1089 if (bcstats.dmapages + npages > bufpages) { 1090 /* cleaner or syncer */ 1091 nobuffers = 1; 1092 tsleep(&nobuffers, PRIBIO, "nobuffers", 0); 1093 splx(s); 1094 return (NULL); 1095 } 1096 } 1097 1098 bp = pool_get(&bufpool, poolwait|PR_ZERO); 1099 1100 if (bp == NULL) { 1101 splx(s); 1102 return (NULL); 1103 } 1104 1105 bp->b_freelist.tqe_next = NOLIST; 1106 bp->b_dev = NODEV; 1107 LIST_INIT(&bp->b_dep); 1108 bp->b_bcount = size; 1109 1110 buf_acquire_nomap(bp); 1111 1112 if (vp != NULL) { 1113 /* 1114 * We insert the buffer into the hash with B_BUSY set 1115 * while we allocate pages for it. This way any getblk 1116 * that happens while we allocate pages will wait for 1117 * this buffer instead of starting its own buf_get. 1118 * 1119 * But first, we check if someone beat us to it. 1120 */ 1121 if (incore(vp, blkno)) { 1122 pool_put(&bufpool, bp); 1123 splx(s); 1124 return (NULL); 1125 } 1126 1127 bp->b_blkno = bp->b_lblkno = blkno; 1128 bgetvp(vp, bp); 1129 if (RBT_INSERT(buf_rb_bufs, &vp->v_bufs_tree, bp)) 1130 panic("buf_get: dup lblk vp %p bp %p", vp, bp); 1131 } else { 1132 bp->b_vnbufs.le_next = NOLIST; 1133 SET(bp->b_flags, B_INVAL); 1134 bp->b_vp = NULL; 1135 } 1136 1137 LIST_INSERT_HEAD(&bufhead, bp, b_list); 1138 bcstats.numbufs++; 1139 1140 if (size) { 1141 buf_alloc_pages(bp, round_page(size)); 1142 KASSERT(ISSET(bp->b_flags, B_DMA)); 1143 buf_map(bp); 1144 } 1145 1146 SET(bp->b_flags, B_BC); 1147 splx(s); 1148 1149 return (bp); 1150 } 1151 1152 /* 1153 * Buffer cleaning daemon. 1154 */ 1155 void 1156 buf_daemon(void *arg) 1157 { 1158 struct buf *bp = NULL; 1159 int s, pushed = 0; 1160 1161 s = splbio(); 1162 for (;;) { 1163 if (bp == NULL || (pushed >= 16 && 1164 UNCLEAN_PAGES < hidirtypages && 1165 bcstats.kvaslots_avail > 2 * RESERVE_SLOTS)){ 1166 pushed = 0; 1167 /* 1168 * Wake up anyone who was waiting for buffers 1169 * to be released. 1170 */ 1171 if (needbuffer) { 1172 needbuffer = 0; 1173 wakeup(&needbuffer); 1174 } 1175 tsleep(&bd_req, PRIBIO - 7, "cleaner", 0); 1176 } 1177 1178 while ((bp = bufcache_getdirtybuf())) { 1179 1180 if (UNCLEAN_PAGES < lodirtypages && 1181 bcstats.kvaslots_avail > 2 * RESERVE_SLOTS && 1182 pushed >= 16) 1183 break; 1184 1185 bufcache_take(bp); 1186 buf_acquire(bp); 1187 splx(s); 1188 1189 if (ISSET(bp->b_flags, B_INVAL)) { 1190 brelse(bp); 1191 s = splbio(); 1192 continue; 1193 } 1194 #ifdef DIAGNOSTIC 1195 if (!ISSET(bp->b_flags, B_DELWRI)) 1196 panic("Clean buffer on dirty queue"); 1197 #endif 1198 if (LIST_FIRST(&bp->b_dep) != NULL && 1199 !ISSET(bp->b_flags, B_DEFERRED) && 1200 buf_countdeps(bp, 0, 0)) { 1201 SET(bp->b_flags, B_DEFERRED); 1202 s = splbio(); 1203 bufcache_release(bp); 1204 buf_release(bp); 1205 continue; 1206 } 1207 1208 bawrite(bp); 1209 pushed++; 1210 1211 sched_pause(yield); 1212 1213 s = splbio(); 1214 } 1215 } 1216 } 1217 1218 /* 1219 * Wait for operations on the buffer to complete. 1220 * When they do, extract and return the I/O's error value. 1221 */ 1222 int 1223 biowait(struct buf *bp) 1224 { 1225 int s; 1226 1227 KASSERT(!(bp->b_flags & B_ASYNC)); 1228 1229 s = splbio(); 1230 while (!ISSET(bp->b_flags, B_DONE)) 1231 tsleep(bp, PRIBIO + 1, "biowait", 0); 1232 splx(s); 1233 1234 /* check for interruption of I/O (e.g. via NFS), then errors. */ 1235 if (ISSET(bp->b_flags, B_EINTR)) { 1236 CLR(bp->b_flags, B_EINTR); 1237 return (EINTR); 1238 } 1239 1240 if (ISSET(bp->b_flags, B_ERROR)) 1241 return (bp->b_error ? bp->b_error : EIO); 1242 else 1243 return (0); 1244 } 1245 1246 /* 1247 * Mark I/O complete on a buffer. 1248 * 1249 * If a callback has been requested, e.g. the pageout 1250 * daemon, do so. Otherwise, awaken waiting processes. 1251 * 1252 * [ Leffler, et al., says on p.247: 1253 * "This routine wakes up the blocked process, frees the buffer 1254 * for an asynchronous write, or, for a request by the pagedaemon 1255 * process, invokes a procedure specified in the buffer structure" ] 1256 * 1257 * In real life, the pagedaemon (or other system processes) wants 1258 * to do async stuff to, and doesn't want the buffer brelse()'d. 1259 * (for swap pager, that puts swap buffers on the free lists (!!!), 1260 * for the vn device, that puts malloc'd buffers on the free lists!) 1261 * 1262 * Must be called at splbio(). 1263 */ 1264 void 1265 biodone(struct buf *bp) 1266 { 1267 splassert(IPL_BIO); 1268 1269 if (ISSET(bp->b_flags, B_DONE)) 1270 panic("biodone already"); 1271 SET(bp->b_flags, B_DONE); /* note that it's done */ 1272 1273 if (bp->b_bq) 1274 bufq_done(bp->b_bq, bp); 1275 1276 if (LIST_FIRST(&bp->b_dep) != NULL) 1277 buf_complete(bp); 1278 1279 if (!ISSET(bp->b_flags, B_READ)) { 1280 CLR(bp->b_flags, B_WRITEINPROG); 1281 vwakeup(bp->b_vp); 1282 } 1283 if (bcstats.numbufs && 1284 (!(ISSET(bp->b_flags, B_RAW) || ISSET(bp->b_flags, B_PHYS)))) { 1285 if (!ISSET(bp->b_flags, B_READ)) { 1286 bcstats.pendingwrites--; 1287 } else 1288 bcstats.pendingreads--; 1289 } 1290 if (ISSET(bp->b_flags, B_CALL)) { /* if necessary, call out */ 1291 CLR(bp->b_flags, B_CALL); /* but note callout done */ 1292 (*bp->b_iodone)(bp); 1293 } else { 1294 if (ISSET(bp->b_flags, B_ASYNC)) {/* if async, release it */ 1295 brelse(bp); 1296 } else { /* or just wakeup the buffer */ 1297 CLR(bp->b_flags, B_WANTED); 1298 wakeup(bp); 1299 } 1300 } 1301 } 1302 1303 #ifdef DDB 1304 void bcstats_print(int (*)(const char *, ...) 1305 __attribute__((__format__(__kprintf__,1,2)))); 1306 /* 1307 * bcstats_print: ddb hook to print interesting buffer cache counters 1308 */ 1309 void 1310 bcstats_print( 1311 int (*pr)(const char *, ...) __attribute__((__format__(__kprintf__,1,2)))) 1312 { 1313 (*pr)("Current Buffer Cache status:\n"); 1314 (*pr)("numbufs %lld busymapped %lld, delwri %lld\n", 1315 bcstats.numbufs, bcstats.busymapped, bcstats.delwribufs); 1316 (*pr)("kvaslots %lld avail kva slots %lld\n", 1317 bcstats.kvaslots, bcstats.kvaslots_avail); 1318 (*pr)("bufpages %lld, dmapages %lld, dirtypages %lld\n", 1319 bcstats.numbufpages, bcstats.dmapages, bcstats.numdirtypages); 1320 (*pr)("pendingreads %lld, pendingwrites %lld\n", 1321 bcstats.pendingreads, bcstats.pendingwrites); 1322 (*pr)("highflips %lld, highflops %lld, dmaflips %lld\n", 1323 bcstats.highflips, bcstats.highflops, bcstats.dmaflips); 1324 } 1325 #endif 1326 1327 void 1328 buf_adjcnt(struct buf *bp, long ncount) 1329 { 1330 KASSERT(ncount <= bp->b_bufsize); 1331 bp->b_bcount = ncount; 1332 } 1333 1334 /* bufcache freelist code below */ 1335 /* 1336 * Copyright (c) 2014 Ted Unangst <tedu@openbsd.org> 1337 * 1338 * Permission to use, copy, modify, and distribute this software for any 1339 * purpose with or without fee is hereby granted, provided that the above 1340 * copyright notice and this permission notice appear in all copies. 1341 * 1342 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 1343 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 1344 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 1345 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 1346 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 1347 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 1348 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 1349 */ 1350 1351 /* 1352 * The code below implements a variant of the 2Q buffer cache algorithm by 1353 * Johnson and Shasha. 1354 * 1355 * General Outline 1356 * We divide the buffer cache into three working sets: current, previous, 1357 * and long term. Each list is itself LRU and buffers get promoted and moved 1358 * around between them. A buffer starts its life in the current working set. 1359 * As time passes and newer buffers push it out, it will turn into the previous 1360 * working set and is subject to recycling. But if it's accessed again from 1361 * the previous working set, that's an indication that it's actually in the 1362 * long term working set, so we promote it there. The separation of current 1363 * and previous working sets prevents us from promoting a buffer that's only 1364 * temporarily hot to the long term cache. 1365 * 1366 * The objective is to provide scan resistance by making the long term 1367 * working set ineligible for immediate recycling, even as the current 1368 * working set is rapidly turned over. 1369 * 1370 * Implementation 1371 * The code below identifies the current, previous, and long term sets as 1372 * hotqueue, coldqueue, and warmqueue. The hot and warm queues are capped at 1373 * 1/3 of the total clean pages, after which point they start pushing their 1374 * oldest buffers into coldqueue. 1375 * A buf always starts out with neither WARM or COLD flags set (implying HOT). 1376 * When released, it will be returned to the tail of the hotqueue list. 1377 * When the hotqueue gets too large, the oldest hot buf will be moved to the 1378 * coldqueue, with the B_COLD flag set. When a cold buf is released, we set 1379 * the B_WARM flag and put it onto the warmqueue. Warm bufs are also 1380 * directly returned to the end of the warmqueue. As with the hotqueue, when 1381 * the warmqueue grows too large, B_WARM bufs are moved onto the coldqueue. 1382 * 1383 * Note that this design does still support large working sets, greater 1384 * than the cap of hotqueue or warmqueue would imply. The coldqueue is still 1385 * cached and has no maximum length. The hot and warm queues form a Y feeding 1386 * into the coldqueue. Moving bufs between queues is constant time, so this 1387 * design decays to one long warm->cold queue. 1388 * 1389 * In the 2Q paper, hotqueue and coldqueue are A1in and A1out. The warmqueue 1390 * is Am. We always cache pages, as opposed to pointers to pages for A1. 1391 * 1392 * This implementation adds support for multiple 2q caches. 1393 * 1394 * If we have more than one 2q cache, as bufs fall off the cold queue 1395 * for recyclying, bufs that have been warm before (which retain the 1396 * B_WARM flag in addition to B_COLD) can be put into the hot queue of 1397 * a second level 2Q cache. buffers which are only B_COLD are 1398 * recycled. Bufs falling off the last cache's cold queue are always 1399 * recycled. 1400 * 1401 */ 1402 1403 /* 1404 * this function is called when a hot or warm queue may have exceeded its 1405 * size limit. it will move a buf to the coldqueue. 1406 */ 1407 int chillbufs(struct 1408 bufcache *cache, struct bufqueue *queue, int64_t *queuepages); 1409 1410 void 1411 bufcache_init(void) 1412 { 1413 int i; 1414 for (i=0; i < NUM_CACHES; i++) { 1415 TAILQ_INIT(&cleancache[i].hotqueue); 1416 TAILQ_INIT(&cleancache[i].coldqueue); 1417 TAILQ_INIT(&cleancache[i].warmqueue); 1418 } 1419 TAILQ_INIT(&dirtyqueue); 1420 } 1421 1422 /* 1423 * if the buffer caches have shrunk, we may need to rebalance our queues. 1424 */ 1425 void 1426 bufcache_adjust(void) 1427 { 1428 int i; 1429 for (i=0; i < NUM_CACHES; i++) { 1430 while (chillbufs(&cleancache[i], &cleancache[i].warmqueue, 1431 &cleancache[i].warmbufpages) || 1432 chillbufs(&cleancache[i], &cleancache[i].hotqueue, 1433 &cleancache[i].hotbufpages)) 1434 continue; 1435 } 1436 } 1437 1438 /* 1439 * Get a clean buffer from the cache. if "discard" is set do not promote 1440 * previously warm buffers as normal, because we are tossing everything 1441 * away such as in a hibernation 1442 */ 1443 struct buf * 1444 bufcache_getcleanbuf(int cachenum, int discard) 1445 { 1446 struct buf *bp = NULL; 1447 struct bufcache *cache = &cleancache[cachenum]; 1448 struct bufqueue * queue; 1449 1450 splassert(IPL_BIO); 1451 1452 /* try cold queue */ 1453 while ((bp = TAILQ_FIRST(&cache->coldqueue)) || 1454 (bp = TAILQ_FIRST(&cache->warmqueue)) || 1455 (bp = TAILQ_FIRST(&cache->hotqueue))) { 1456 if ((!discard) && cachenum < NUM_CACHES - 1) { 1457 int64_t pages = atop(bp->b_bufsize); 1458 struct bufcache *newcache; 1459 1460 KASSERT(bp->cache == cachenum); 1461 1462 /* 1463 * If this buffer was warm before, move it to 1464 * the hot queue in the next cache 1465 */ 1466 1467 if (fliphigh) { 1468 /* 1469 * If we are in the DMA cache, try to flip the 1470 * buffer up high to move it on to the other 1471 * caches. if we can't move the buffer to high 1472 * memory without sleeping, we give it up and 1473 * return it rather than fight for more memory 1474 * against non buffer cache competitors. 1475 */ 1476 SET(bp->b_flags, B_BUSY); 1477 if (bp->cache == 0 && buf_flip_high(bp) == -1) { 1478 CLR(bp->b_flags, B_BUSY); 1479 return bp; 1480 } 1481 CLR(bp->b_flags, B_BUSY); 1482 } 1483 1484 /* Move the buffer to the hot queue in the next cache */ 1485 if (ISSET(bp->b_flags, B_COLD)) { 1486 queue = &cache->coldqueue; 1487 } else if (ISSET(bp->b_flags, B_WARM)) { 1488 queue = &cache->warmqueue; 1489 cache->warmbufpages -= pages; 1490 } else { 1491 queue = &cache->hotqueue; 1492 cache->hotbufpages -= pages; 1493 } 1494 TAILQ_REMOVE(queue, bp, b_freelist); 1495 cache->cachepages -= pages; 1496 CLR(bp->b_flags, B_WARM); 1497 CLR(bp->b_flags, B_COLD); 1498 bp->cache++; 1499 newcache= &cleancache[bp->cache]; 1500 newcache->cachepages += pages; 1501 newcache->hotbufpages += pages; 1502 chillbufs(newcache, &newcache->hotqueue, 1503 &newcache->hotbufpages); 1504 TAILQ_INSERT_TAIL(&newcache->hotqueue, bp, b_freelist); 1505 } 1506 else 1507 /* Victim selected, give it up */ 1508 return bp; 1509 } 1510 return bp; 1511 } 1512 1513 1514 void 1515 discard_buffer(struct buf *bp) { 1516 bufcache_take(bp); 1517 if (bp->b_vp) { 1518 RBT_REMOVE(buf_rb_bufs, 1519 &bp->b_vp->v_bufs_tree, bp); 1520 brelvp(bp); 1521 } 1522 buf_put(bp); 1523 } 1524 1525 int64_t 1526 bufcache_recover_dmapages(int discard, int64_t howmany) 1527 { 1528 struct buf *bp = NULL; 1529 struct bufcache *cache = &cleancache[DMA_CACHE]; 1530 struct bufqueue * queue; 1531 int64_t recovered = 0; 1532 1533 splassert(IPL_BIO); 1534 1535 while ((recovered < howmany) && 1536 ((bp = TAILQ_FIRST(&cache->coldqueue)) || 1537 (bp = TAILQ_FIRST(&cache->warmqueue)) || 1538 (bp = TAILQ_FIRST(&cache->hotqueue)))) { 1539 if (!discard && DMA_CACHE < NUM_CACHES - 1) { 1540 int64_t pages = atop(bp->b_bufsize); 1541 struct bufcache *newcache; 1542 1543 KASSERT(bp->cache == DMA_CACHE); 1544 1545 /* 1546 * If this buffer was warm before, move it to 1547 * the hot queue in the next cache 1548 */ 1549 1550 /* 1551 * One way or another, the pages for this 1552 * buffer are leaving DMA memory 1553 */ 1554 recovered += pages; 1555 1556 if (fliphigh) { 1557 /* 1558 * If we are in the DMA cache, try to flip the 1559 * buffer up high to move it on to the other 1560 * caches. if we can't move the buffer to high 1561 * memory without sleeping, we give it up 1562 * now rather than fight for more memory 1563 * against non buffer cache competitors. 1564 */ 1565 SET(bp->b_flags, B_BUSY); 1566 if (bp->cache == 0 && buf_flip_high(bp) == -1) { 1567 CLR(bp->b_flags, B_BUSY); 1568 discard_buffer(bp); 1569 } else { 1570 CLR(bp->b_flags, B_BUSY); 1571 1572 /* 1573 * Move the buffer to the hot queue in 1574 * the next cache 1575 */ 1576 if (ISSET(bp->b_flags, B_COLD)) { 1577 queue = &cache->coldqueue; 1578 } else if (ISSET(bp->b_flags, B_WARM)) { 1579 queue = &cache->warmqueue; 1580 cache->warmbufpages -= pages; 1581 } else { 1582 queue = &cache->hotqueue; 1583 cache->hotbufpages -= pages; 1584 } 1585 TAILQ_REMOVE(queue, bp, b_freelist); 1586 cache->cachepages -= pages; 1587 CLR(bp->b_flags, B_WARM); 1588 CLR(bp->b_flags, B_COLD); 1589 bp->cache++; 1590 newcache= &cleancache[bp->cache]; 1591 newcache->cachepages += pages; 1592 newcache->hotbufpages += pages; 1593 chillbufs(newcache, &newcache->hotqueue, 1594 &newcache->hotbufpages); 1595 TAILQ_INSERT_TAIL(&newcache->hotqueue, 1596 bp, b_freelist); 1597 } 1598 } else 1599 discard_buffer(bp); 1600 } else 1601 discard_buffer(bp); 1602 } 1603 return recovered; 1604 } 1605 1606 struct buf * 1607 bufcache_getcleanbuf_range(int start, int end, int discard) 1608 { 1609 int i, j = start, q = end; 1610 struct buf *bp = NULL; 1611 1612 /* 1613 * XXX in theory we could promote warm buffers into a previous queue 1614 * so in the pathological case of where we go through all the caches 1615 * without getting a buffer we have to start at the beginning again. 1616 */ 1617 while (j <= q) { 1618 for (i = q; i >= j; i--) 1619 if ((bp = bufcache_getcleanbuf(i, discard))) 1620 return(bp); 1621 j++; 1622 } 1623 return bp; 1624 } 1625 1626 struct buf * 1627 bufcache_gethighcleanbuf(void) 1628 { 1629 if (!fliphigh) 1630 return NULL; 1631 return bufcache_getcleanbuf_range(DMA_CACHE + 1, NUM_CACHES - 1, 0); 1632 } 1633 1634 1635 struct buf * 1636 bufcache_getdmacleanbuf(void) 1637 { 1638 if (fliphigh) 1639 return bufcache_getcleanbuf_range(DMA_CACHE, DMA_CACHE, 0); 1640 return bufcache_getcleanbuf_range(DMA_CACHE, NUM_CACHES - 1, 0); 1641 } 1642 1643 1644 struct buf * 1645 bufcache_getdirtybuf(void) 1646 { 1647 return TAILQ_FIRST(&dirtyqueue); 1648 } 1649 1650 void 1651 bufcache_take(struct buf *bp) 1652 { 1653 struct bufqueue *queue; 1654 int64_t pages; 1655 1656 splassert(IPL_BIO); 1657 KASSERT(ISSET(bp->b_flags, B_BC)); 1658 KASSERT(bp->cache >= DMA_CACHE); 1659 KASSERT((bp->cache < NUM_CACHES)); 1660 1661 pages = atop(bp->b_bufsize); 1662 struct bufcache *cache = &cleancache[bp->cache]; 1663 if (!ISSET(bp->b_flags, B_DELWRI)) { 1664 if (ISSET(bp->b_flags, B_COLD)) { 1665 queue = &cache->coldqueue; 1666 } else if (ISSET(bp->b_flags, B_WARM)) { 1667 queue = &cache->warmqueue; 1668 cache->warmbufpages -= pages; 1669 } else { 1670 queue = &cache->hotqueue; 1671 cache->hotbufpages -= pages; 1672 } 1673 bcstats.numcleanpages -= pages; 1674 cache->cachepages -= pages; 1675 } else { 1676 queue = &dirtyqueue; 1677 bcstats.numdirtypages -= pages; 1678 bcstats.delwribufs--; 1679 } 1680 TAILQ_REMOVE(queue, bp, b_freelist); 1681 } 1682 1683 /* move buffers from a hot or warm queue to a cold queue in a cache */ 1684 int 1685 chillbufs(struct bufcache *cache, struct bufqueue *queue, int64_t *queuepages) 1686 { 1687 struct buf *bp; 1688 int64_t limit, pages; 1689 1690 /* 1691 * We limit the hot queue to be small, with a max of 4096 pages. 1692 * We limit the warm queue to half the cache size. 1693 * 1694 * We impose a minimum size of 96 to prevent too much "wobbling". 1695 */ 1696 if (queue == &cache->hotqueue) 1697 limit = min(cache->cachepages / 20, 4096); 1698 else if (queue == &cache->warmqueue) 1699 limit = (cache->cachepages / 2); 1700 else 1701 panic("chillbufs: invalid queue"); 1702 1703 if (*queuepages > 96 && *queuepages > limit) { 1704 bp = TAILQ_FIRST(queue); 1705 if (!bp) 1706 panic("inconsistent bufpage counts"); 1707 pages = atop(bp->b_bufsize); 1708 *queuepages -= pages; 1709 TAILQ_REMOVE(queue, bp, b_freelist); 1710 /* we do not clear B_WARM */ 1711 SET(bp->b_flags, B_COLD); 1712 TAILQ_INSERT_TAIL(&cache->coldqueue, bp, b_freelist); 1713 return 1; 1714 } 1715 return 0; 1716 } 1717 1718 void 1719 bufcache_release(struct buf *bp) 1720 { 1721 struct bufqueue *queue; 1722 int64_t pages; 1723 struct bufcache *cache = &cleancache[bp->cache]; 1724 1725 pages = atop(bp->b_bufsize); 1726 KASSERT(ISSET(bp->b_flags, B_BC)); 1727 if (fliphigh) { 1728 if (ISSET(bp->b_flags, B_DMA) && bp->cache > 0) 1729 panic("B_DMA buffer release from cache %d", 1730 bp->cache); 1731 else if ((!ISSET(bp->b_flags, B_DMA)) && bp->cache == 0) 1732 panic("Non B_DMA buffer release from cache %d", 1733 bp->cache); 1734 } 1735 1736 if (!ISSET(bp->b_flags, B_DELWRI)) { 1737 int64_t *queuepages; 1738 if (ISSET(bp->b_flags, B_WARM | B_COLD)) { 1739 SET(bp->b_flags, B_WARM); 1740 CLR(bp->b_flags, B_COLD); 1741 queue = &cache->warmqueue; 1742 queuepages = &cache->warmbufpages; 1743 } else { 1744 queue = &cache->hotqueue; 1745 queuepages = &cache->hotbufpages; 1746 } 1747 *queuepages += pages; 1748 bcstats.numcleanpages += pages; 1749 cache->cachepages += pages; 1750 chillbufs(cache, queue, queuepages); 1751 } else { 1752 queue = &dirtyqueue; 1753 bcstats.numdirtypages += pages; 1754 bcstats.delwribufs++; 1755 } 1756 TAILQ_INSERT_TAIL(queue, bp, b_freelist); 1757 } 1758 1759 #ifdef HIBERNATE 1760 /* 1761 * Nuke the buffer cache from orbit when hibernating. We do not want to save 1762 * any clean cache pages to swap and read them back. the original disk files 1763 * are just as good. 1764 */ 1765 void 1766 hibernate_suspend_bufcache(void) 1767 { 1768 struct buf *bp; 1769 int s; 1770 1771 s = splbio(); 1772 /* Chuck away all the cache pages.. discard bufs, do not promote */ 1773 while ((bp = bufcache_getcleanbuf_range(DMA_CACHE, NUM_CACHES - 1, 1))) { 1774 bufcache_take(bp); 1775 if (bp->b_vp) { 1776 RBT_REMOVE(buf_rb_bufs, &bp->b_vp->v_bufs_tree, bp); 1777 brelvp(bp); 1778 } 1779 buf_put(bp); 1780 } 1781 splx(s); 1782 } 1783 1784 void 1785 hibernate_resume_bufcache(void) 1786 { 1787 /* XXX Nothing needed here for now */ 1788 } 1789 #endif /* HIBERNATE */ 1790