1 /* $OpenBSD: vfs_bio.c,v 1.197 2020/03/04 13:27:10 anton Exp $ */ 2 /* $NetBSD: vfs_bio.c,v 1.44 1996/06/11 11:15:36 pk Exp $ */ 3 4 /* 5 * Copyright (c) 1994 Christopher G. Demetriou 6 * Copyright (c) 1982, 1986, 1989, 1993 7 * The Regents of the University of California. All rights reserved. 8 * (c) UNIX System Laboratories, Inc. 9 * All or some portions of this file are derived from material licensed 10 * to the University of California by American Telephone and Telegraph 11 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 12 * the permission of UNIX System Laboratories, Inc. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 3. Neither the name of the University nor the names of its contributors 23 * may be used to endorse or promote products derived from this software 24 * without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 29 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 36 * SUCH DAMAGE. 37 * 38 * @(#)vfs_bio.c 8.6 (Berkeley) 1/11/94 39 */ 40 41 /* 42 * Some references: 43 * Bach: The Design of the UNIX Operating System (Prentice Hall, 1986) 44 * Leffler, et al.: The Design and Implementation of the 4.3BSD 45 * UNIX Operating System (Addison Welley, 1989) 46 */ 47 48 #include <sys/param.h> 49 #include <sys/systm.h> 50 #include <sys/proc.h> 51 #include <sys/buf.h> 52 #include <sys/vnode.h> 53 #include <sys/mount.h> 54 #include <sys/malloc.h> 55 #include <sys/pool.h> 56 #include <sys/resourcevar.h> 57 #include <sys/conf.h> 58 #include <sys/kernel.h> 59 #include <sys/specdev.h> 60 #include <uvm/uvm_extern.h> 61 62 /* XXX Should really be in buf.h, but for uvm_constraint_range.. */ 63 int buf_realloc_pages(struct buf *, struct uvm_constraint_range *, int); 64 65 struct uvm_constraint_range high_constraint; 66 int fliphigh; 67 68 int nobuffers; 69 int needbuffer; 70 struct bio_ops bioops; 71 72 /* private bufcache functions */ 73 void bufcache_init(void); 74 void bufcache_adjust(void); 75 struct buf *bufcache_gethighcleanbuf(void); 76 struct buf *bufcache_getdmacleanbuf(void); 77 78 /* 79 * Buffer pool for I/O buffers. 80 */ 81 struct pool bufpool; 82 struct bufhead bufhead = LIST_HEAD_INITIALIZER(bufhead); 83 void buf_put(struct buf *); 84 85 struct buf *bio_doread(struct vnode *, daddr_t, int, int); 86 struct buf *buf_get(struct vnode *, daddr_t, size_t); 87 void bread_cluster_callback(struct buf *); 88 int64_t bufcache_recover_dmapages(int discard, int64_t howmany); 89 90 struct bcachestats bcstats; /* counters */ 91 long lodirtypages; /* dirty page count low water mark */ 92 long hidirtypages; /* dirty page count high water mark */ 93 long targetpages; /* target number of pages for cache size */ 94 long buflowpages; /* smallest size cache allowed */ 95 long bufhighpages; /* largest size cache allowed */ 96 long bufbackpages; /* minimum number of pages we shrink when asked to */ 97 98 vsize_t bufkvm; 99 100 struct proc *cleanerproc; 101 int bd_req; /* Sleep point for cleaner daemon. */ 102 103 #define NUM_CACHES 2 104 #define DMA_CACHE 0 105 struct bufcache cleancache[NUM_CACHES]; 106 struct bufqueue dirtyqueue; 107 108 void 109 buf_put(struct buf *bp) 110 { 111 splassert(IPL_BIO); 112 113 #ifdef DIAGNOSTIC 114 if (bp->b_pobj != NULL) 115 KASSERT(bp->b_bufsize > 0); 116 if (ISSET(bp->b_flags, B_DELWRI)) 117 panic("buf_put: releasing dirty buffer"); 118 if (bp->b_freelist.tqe_next != NOLIST && 119 bp->b_freelist.tqe_next != (void *)-1) 120 panic("buf_put: still on the free list"); 121 if (bp->b_vnbufs.le_next != NOLIST && 122 bp->b_vnbufs.le_next != (void *)-1) 123 panic("buf_put: still on the vnode list"); 124 if (!LIST_EMPTY(&bp->b_dep)) 125 panic("buf_put: b_dep is not empty"); 126 #endif 127 128 LIST_REMOVE(bp, b_list); 129 bcstats.numbufs--; 130 131 if (buf_dealloc_mem(bp) != 0) 132 return; 133 pool_put(&bufpool, bp); 134 } 135 136 /* 137 * Initialize buffers and hash links for buffers. 138 */ 139 void 140 bufinit(void) 141 { 142 u_int64_t dmapages; 143 u_int64_t highpages; 144 145 dmapages = uvm_pagecount(&dma_constraint); 146 /* take away a guess at how much of this the kernel will consume */ 147 dmapages -= (atop(physmem) - atop(uvmexp.free)); 148 149 /* See if we have memory above the dma accessible region. */ 150 high_constraint.ucr_low = dma_constraint.ucr_high; 151 high_constraint.ucr_high = no_constraint.ucr_high; 152 if (high_constraint.ucr_low != high_constraint.ucr_high) 153 high_constraint.ucr_low++; 154 highpages = uvm_pagecount(&high_constraint); 155 156 /* 157 * Do we have any significant amount of high memory above 158 * the DMA region? if so enable moving buffers there, if not, 159 * don't bother. 160 */ 161 if (highpages > dmapages / 4) 162 fliphigh = 1; 163 else 164 fliphigh = 0; 165 166 /* 167 * If MD code doesn't say otherwise, use up to 10% of DMA'able 168 * memory for buffers. 169 */ 170 if (bufcachepercent == 0) 171 bufcachepercent = 10; 172 173 /* 174 * XXX these values and their same use in kern_sysctl 175 * need to move into buf.h 176 */ 177 KASSERT(bufcachepercent <= 90); 178 KASSERT(bufcachepercent >= 5); 179 if (bufpages == 0) 180 bufpages = dmapages * bufcachepercent / 100; 181 if (bufpages < BCACHE_MIN) 182 bufpages = BCACHE_MIN; 183 KASSERT(bufpages < dmapages); 184 185 bufhighpages = bufpages; 186 187 /* 188 * Set the base backoff level for the buffer cache. We will 189 * not allow uvm to steal back more than this number of pages. 190 */ 191 buflowpages = dmapages * 5 / 100; 192 if (buflowpages < BCACHE_MIN) 193 buflowpages = BCACHE_MIN; 194 195 /* 196 * set bufbackpages to 100 pages, or 10 percent of the low water mark 197 * if we don't have that many pages. 198 */ 199 200 bufbackpages = buflowpages * 10 / 100; 201 if (bufbackpages > 100) 202 bufbackpages = 100; 203 204 /* 205 * If the MD code does not say otherwise, reserve 10% of kva 206 * space for mapping buffers. 207 */ 208 if (bufkvm == 0) 209 bufkvm = VM_KERNEL_SPACE_SIZE / 10; 210 211 /* 212 * Don't use more than twice the amount of bufpages for mappings. 213 * It's twice since we map things sparsely. 214 */ 215 if (bufkvm > bufpages * PAGE_SIZE) 216 bufkvm = bufpages * PAGE_SIZE; 217 /* 218 * Round bufkvm to MAXPHYS because we allocate chunks of va space 219 * in MAXPHYS chunks. 220 */ 221 bufkvm &= ~(MAXPHYS - 1); 222 223 pool_init(&bufpool, sizeof(struct buf), 0, IPL_BIO, 0, "bufpl", NULL); 224 225 bufcache_init(); 226 227 /* 228 * hmm - bufkvm is an argument because it's static, while 229 * bufpages is global because it can change while running. 230 */ 231 buf_mem_init(bufkvm); 232 233 /* 234 * Set the dirty page high water mark to be less than the low 235 * water mark for pages in the buffer cache. This ensures we 236 * can always back off by throwing away clean pages, and give 237 * ourselves a chance to write out the dirty pages eventually. 238 */ 239 hidirtypages = (buflowpages / 4) * 3; 240 lodirtypages = buflowpages / 2; 241 242 /* 243 * We are allowed to use up to the reserve. 244 */ 245 targetpages = bufpages - RESERVE_PAGES; 246 } 247 248 /* 249 * Change cachepct 250 */ 251 void 252 bufadjust(int newbufpages) 253 { 254 int s; 255 int64_t npages; 256 257 if (newbufpages < buflowpages) 258 newbufpages = buflowpages; 259 260 s = splbio(); 261 bufpages = newbufpages; 262 263 /* 264 * We are allowed to use up to the reserve 265 */ 266 targetpages = bufpages - RESERVE_PAGES; 267 268 npages = bcstats.dmapages - targetpages; 269 270 /* 271 * Shrinking the cache happens here only if someone has manually 272 * adjusted bufcachepercent - or the pagedaemon has told us 273 * to give back memory *now* - so we give it all back. 274 */ 275 if (bcstats.dmapages > targetpages) 276 (void) bufcache_recover_dmapages(0, bcstats.dmapages - targetpages); 277 bufcache_adjust(); 278 279 /* 280 * Wake up the cleaner if we have lots of dirty pages, 281 * or if we are getting low on buffer cache kva. 282 */ 283 if ((UNCLEAN_PAGES >= hidirtypages) || 284 bcstats.kvaslots_avail <= 2 * RESERVE_SLOTS) 285 wakeup(&bd_req); 286 287 splx(s); 288 } 289 290 /* 291 * Make the buffer cache back off from cachepct. 292 */ 293 int 294 bufbackoff(struct uvm_constraint_range *range, long size) 295 { 296 /* 297 * Back off "size" buffer cache pages. Called by the page 298 * daemon to consume buffer cache pages rather than scanning. 299 * 300 * It returns 0 to the pagedaemon to indicate that it has 301 * succeeded in freeing enough pages. It returns -1 to 302 * indicate that it could not and the pagedaemon should take 303 * other measures. 304 * 305 */ 306 long pdelta, oldbufpages; 307 308 /* 309 * If we will accept high memory for this backoff 310 * try to steal it from the high memory buffer cache. 311 */ 312 if (range->ucr_high > dma_constraint.ucr_high) { 313 struct buf *bp; 314 int64_t start = bcstats.numbufpages, recovered = 0; 315 int s = splbio(); 316 317 while ((recovered < size) && 318 (bp = bufcache_gethighcleanbuf())) { 319 bufcache_take(bp); 320 if (bp->b_vp) { 321 RBT_REMOVE(buf_rb_bufs, 322 &bp->b_vp->v_bufs_tree, bp); 323 brelvp(bp); 324 } 325 buf_put(bp); 326 recovered = start - bcstats.numbufpages; 327 } 328 bufcache_adjust(); 329 splx(s); 330 331 /* If we got enough, return success */ 332 if (recovered >= size) 333 return 0; 334 335 /* 336 * If we needed only memory above DMA, 337 * return failure 338 */ 339 if (range->ucr_low > dma_constraint.ucr_high) 340 return -1; 341 342 /* Otherwise get the rest from DMA */ 343 size -= recovered; 344 } 345 346 /* 347 * XXX Otherwise do the dma memory cache dance. this needs 348 * refactoring later to get rid of 'bufpages' 349 */ 350 351 /* 352 * Back off by at least bufbackpages. If the page daemon gave us 353 * a larger size, back off by that much. 354 */ 355 pdelta = (size > bufbackpages) ? size : bufbackpages; 356 357 if (bufpages <= buflowpages) 358 return(-1); 359 if (bufpages - pdelta < buflowpages) 360 pdelta = bufpages - buflowpages; 361 oldbufpages = bufpages; 362 bufadjust(bufpages - pdelta); 363 if (oldbufpages - bufpages < size) 364 return (-1); /* we did not free what we were asked */ 365 else 366 return(0); 367 } 368 369 370 /* 371 * Opportunistically flip a buffer into high memory. Will move the buffer 372 * if memory is available without sleeping, and return 0, otherwise will 373 * fail and return -1 with the buffer unchanged. 374 */ 375 376 int 377 buf_flip_high(struct buf *bp) 378 { 379 int s; 380 int ret = -1; 381 382 KASSERT(ISSET(bp->b_flags, B_BC)); 383 KASSERT(ISSET(bp->b_flags, B_DMA)); 384 KASSERT(bp->cache == DMA_CACHE); 385 KASSERT(fliphigh); 386 387 /* Attempt to move the buffer to high memory if we can */ 388 s = splbio(); 389 if (buf_realloc_pages(bp, &high_constraint, UVM_PLA_NOWAIT) == 0) { 390 KASSERT(!ISSET(bp->b_flags, B_DMA)); 391 bcstats.highflips++; 392 ret = 0; 393 } else 394 bcstats.highflops++; 395 splx(s); 396 397 return ret; 398 } 399 400 /* 401 * Flip a buffer to dma reachable memory, when we need it there for 402 * I/O. This can sleep since it will wait for memory allocation in the 403 * DMA reachable area since we have to have the buffer there to proceed. 404 */ 405 void 406 buf_flip_dma(struct buf *bp) 407 { 408 KASSERT(ISSET(bp->b_flags, B_BC)); 409 KASSERT(ISSET(bp->b_flags, B_BUSY)); 410 KASSERT(bp->cache < NUM_CACHES); 411 412 if (!ISSET(bp->b_flags, B_DMA)) { 413 int s = splbio(); 414 415 /* move buf to dma reachable memory */ 416 (void) buf_realloc_pages(bp, &dma_constraint, UVM_PLA_WAITOK); 417 KASSERT(ISSET(bp->b_flags, B_DMA)); 418 bcstats.dmaflips++; 419 splx(s); 420 } 421 422 if (bp->cache > DMA_CACHE) { 423 CLR(bp->b_flags, B_COLD); 424 CLR(bp->b_flags, B_WARM); 425 bp->cache = DMA_CACHE; 426 } 427 } 428 429 struct buf * 430 bio_doread(struct vnode *vp, daddr_t blkno, int size, int async) 431 { 432 struct buf *bp; 433 struct mount *mp; 434 435 bp = getblk(vp, blkno, size, 0, INFSLP); 436 437 /* 438 * If buffer does not have valid data, start a read. 439 * Note that if buffer is B_INVAL, getblk() won't return it. 440 * Therefore, it's valid if its I/O has completed or been delayed. 441 */ 442 if (!ISSET(bp->b_flags, (B_DONE | B_DELWRI))) { 443 SET(bp->b_flags, B_READ | async); 444 bcstats.pendingreads++; 445 bcstats.numreads++; 446 VOP_STRATEGY(bp); 447 /* Pay for the read. */ 448 curproc->p_ru.ru_inblock++; /* XXX */ 449 } else if (async) { 450 brelse(bp); 451 } 452 453 mp = vp->v_type == VBLK ? vp->v_specmountpoint : vp->v_mount; 454 455 /* 456 * Collect statistics on synchronous and asynchronous reads. 457 * Reads from block devices are charged to their associated 458 * filesystem (if any). 459 */ 460 if (mp != NULL) { 461 if (async == 0) 462 mp->mnt_stat.f_syncreads++; 463 else 464 mp->mnt_stat.f_asyncreads++; 465 } 466 467 return (bp); 468 } 469 470 /* 471 * Read a disk block. 472 * This algorithm described in Bach (p.54). 473 */ 474 int 475 bread(struct vnode *vp, daddr_t blkno, int size, struct buf **bpp) 476 { 477 struct buf *bp; 478 479 /* Get buffer for block. */ 480 bp = *bpp = bio_doread(vp, blkno, size, 0); 481 482 /* Wait for the read to complete, and return result. */ 483 return (biowait(bp)); 484 } 485 486 /* 487 * Read-ahead multiple disk blocks. The first is sync, the rest async. 488 * Trivial modification to the breada algorithm presented in Bach (p.55). 489 */ 490 int 491 breadn(struct vnode *vp, daddr_t blkno, int size, daddr_t rablks[], 492 int rasizes[], int nrablks, struct buf **bpp) 493 { 494 struct buf *bp; 495 int i; 496 497 bp = *bpp = bio_doread(vp, blkno, size, 0); 498 499 /* 500 * For each of the read-ahead blocks, start a read, if necessary. 501 */ 502 for (i = 0; i < nrablks; i++) { 503 /* If it's in the cache, just go on to next one. */ 504 if (incore(vp, rablks[i])) 505 continue; 506 507 /* Get a buffer for the read-ahead block */ 508 (void) bio_doread(vp, rablks[i], rasizes[i], B_ASYNC); 509 } 510 511 /* Otherwise, we had to start a read for it; wait until it's valid. */ 512 return (biowait(bp)); 513 } 514 515 /* 516 * Called from interrupt context. 517 */ 518 void 519 bread_cluster_callback(struct buf *bp) 520 { 521 struct buf **xbpp = bp->b_saveaddr; 522 int i; 523 524 if (xbpp[1] != NULL) { 525 size_t newsize = xbpp[1]->b_bufsize; 526 527 /* 528 * Shrink this buffer's mapping to only cover its part of 529 * the total I/O. 530 */ 531 buf_fix_mapping(bp, newsize); 532 bp->b_bcount = newsize; 533 } 534 535 /* Invalidate read-ahead buffers if read short */ 536 if (bp->b_resid > 0) { 537 for (i = 1; xbpp[i] != NULL; i++) 538 continue; 539 for (i = i - 1; i != 0; i--) { 540 if (xbpp[i]->b_bufsize <= bp->b_resid) { 541 bp->b_resid -= xbpp[i]->b_bufsize; 542 SET(xbpp[i]->b_flags, B_INVAL); 543 } else if (bp->b_resid > 0) { 544 bp->b_resid = 0; 545 SET(xbpp[i]->b_flags, B_INVAL); 546 } else 547 break; 548 } 549 } 550 551 for (i = 1; xbpp[i] != NULL; i++) { 552 if (ISSET(bp->b_flags, B_ERROR)) 553 SET(xbpp[i]->b_flags, B_INVAL | B_ERROR); 554 /* 555 * Move the pages from the master buffer's uvm object 556 * into the individual buffer's uvm objects. 557 */ 558 struct uvm_object *newobj = &xbpp[i]->b_uobj; 559 struct uvm_object *oldobj = &bp->b_uobj; 560 int page; 561 562 uvm_objinit(newobj, NULL, 1); 563 for (page = 0; page < atop(xbpp[i]->b_bufsize); page++) { 564 struct vm_page *pg = uvm_pagelookup(oldobj, 565 xbpp[i]->b_poffs + ptoa(page)); 566 KASSERT(pg != NULL); 567 KASSERT(pg->wire_count == 1); 568 uvm_pagerealloc(pg, newobj, xbpp[i]->b_poffs + ptoa(page)); 569 } 570 xbpp[i]->b_pobj = newobj; 571 572 biodone(xbpp[i]); 573 } 574 575 free(xbpp, M_TEMP, (i + 1) * sizeof(*xbpp)); 576 577 if (ISSET(bp->b_flags, B_ASYNC)) { 578 brelse(bp); 579 } else { 580 CLR(bp->b_flags, B_WANTED); 581 wakeup(bp); 582 } 583 } 584 585 /* 586 * Read-ahead multiple disk blocks, but make sure only one (big) I/O 587 * request is sent to the disk. 588 * XXX This should probably be dropped and breadn should instead be optimized 589 * XXX to do fewer I/O requests. 590 */ 591 int 592 bread_cluster(struct vnode *vp, daddr_t blkno, int size, struct buf **rbpp) 593 { 594 struct buf *bp, **xbpp; 595 int howmany, maxra, i, inc; 596 daddr_t sblkno; 597 598 *rbpp = bio_doread(vp, blkno, size, 0); 599 600 /* 601 * If the buffer is in the cache skip any I/O operation. 602 */ 603 if (ISSET((*rbpp)->b_flags, B_CACHE)) 604 goto out; 605 606 if (size != round_page(size)) 607 goto out; 608 609 if (VOP_BMAP(vp, blkno + 1, NULL, &sblkno, &maxra)) 610 goto out; 611 612 maxra++; 613 if (sblkno == -1 || maxra < 2) 614 goto out; 615 616 howmany = MAXPHYS / size; 617 if (howmany > maxra) 618 howmany = maxra; 619 620 xbpp = mallocarray(howmany + 1, sizeof(*xbpp), M_TEMP, M_NOWAIT); 621 if (xbpp == NULL) 622 goto out; 623 624 for (i = howmany - 1; i >= 0; i--) { 625 size_t sz; 626 627 /* 628 * First buffer allocates big enough size to cover what 629 * all the other buffers need. 630 */ 631 sz = i == 0 ? howmany * size : 0; 632 633 xbpp[i] = buf_get(vp, blkno + i + 1, sz); 634 if (xbpp[i] == NULL) { 635 for (++i; i < howmany; i++) { 636 SET(xbpp[i]->b_flags, B_INVAL); 637 brelse(xbpp[i]); 638 } 639 free(xbpp, M_TEMP, (howmany + 1) * sizeof(*xbpp)); 640 goto out; 641 } 642 } 643 644 bp = xbpp[0]; 645 646 xbpp[howmany] = NULL; 647 648 inc = btodb(size); 649 650 for (i = 1; i < howmany; i++) { 651 bcstats.pendingreads++; 652 bcstats.numreads++; 653 /* 654 * We set B_DMA here because bp above will be B_DMA, 655 * and we are playing buffer slice-n-dice games from 656 * the memory allocated in bp. 657 */ 658 SET(xbpp[i]->b_flags, B_DMA | B_READ | B_ASYNC); 659 xbpp[i]->b_blkno = sblkno + (i * inc); 660 xbpp[i]->b_bufsize = xbpp[i]->b_bcount = size; 661 xbpp[i]->b_data = NULL; 662 xbpp[i]->b_pobj = bp->b_pobj; 663 xbpp[i]->b_poffs = bp->b_poffs + (i * size); 664 } 665 666 KASSERT(bp->b_lblkno == blkno + 1); 667 KASSERT(bp->b_vp == vp); 668 669 bp->b_blkno = sblkno; 670 SET(bp->b_flags, B_READ | B_ASYNC | B_CALL); 671 672 bp->b_saveaddr = (void *)xbpp; 673 bp->b_iodone = bread_cluster_callback; 674 675 bcstats.pendingreads++; 676 bcstats.numreads++; 677 VOP_STRATEGY(bp); 678 curproc->p_ru.ru_inblock++; 679 680 out: 681 return (biowait(*rbpp)); 682 } 683 684 /* 685 * Block write. Described in Bach (p.56) 686 */ 687 int 688 bwrite(struct buf *bp) 689 { 690 int rv, async, wasdelayed, s; 691 struct vnode *vp; 692 struct mount *mp; 693 694 vp = bp->b_vp; 695 if (vp != NULL) 696 mp = vp->v_type == VBLK? vp->v_specmountpoint : vp->v_mount; 697 else 698 mp = NULL; 699 700 /* 701 * Remember buffer type, to switch on it later. If the write was 702 * synchronous, but the file system was mounted with MNT_ASYNC, 703 * convert it to a delayed write. 704 * XXX note that this relies on delayed tape writes being converted 705 * to async, not sync writes (which is safe, but ugly). 706 */ 707 async = ISSET(bp->b_flags, B_ASYNC); 708 if (!async && mp && ISSET(mp->mnt_flag, MNT_ASYNC)) { 709 bdwrite(bp); 710 return (0); 711 } 712 713 /* 714 * Collect statistics on synchronous and asynchronous writes. 715 * Writes to block devices are charged to their associated 716 * filesystem (if any). 717 */ 718 if (mp != NULL) { 719 if (async) 720 mp->mnt_stat.f_asyncwrites++; 721 else 722 mp->mnt_stat.f_syncwrites++; 723 } 724 bcstats.pendingwrites++; 725 bcstats.numwrites++; 726 727 wasdelayed = ISSET(bp->b_flags, B_DELWRI); 728 CLR(bp->b_flags, (B_READ | B_DONE | B_ERROR | B_DELWRI)); 729 730 s = splbio(); 731 732 /* 733 * If not synchronous, pay for the I/O operation and make 734 * sure the buf is on the correct vnode queue. We have 735 * to do this now, because if we don't, the vnode may not 736 * be properly notified that its I/O has completed. 737 */ 738 if (wasdelayed) { 739 reassignbuf(bp); 740 } else 741 curproc->p_ru.ru_oublock++; 742 743 744 /* Initiate disk write. Make sure the appropriate party is charged. */ 745 bp->b_vp->v_numoutput++; 746 splx(s); 747 buf_flip_dma(bp); 748 SET(bp->b_flags, B_WRITEINPROG); 749 VOP_STRATEGY(bp); 750 751 /* 752 * If the queue is above the high water mark, wait till 753 * the number of outstanding write bufs drops below the low 754 * water mark. 755 */ 756 if (bp->b_bq) 757 bufq_wait(bp->b_bq); 758 759 if (async) 760 return (0); 761 762 /* 763 * If I/O was synchronous, wait for it to complete. 764 */ 765 rv = biowait(bp); 766 767 /* Release the buffer. */ 768 brelse(bp); 769 770 return (rv); 771 } 772 773 774 /* 775 * Delayed write. 776 * 777 * The buffer is marked dirty, but is not queued for I/O. 778 * This routine should be used when the buffer is expected 779 * to be modified again soon, typically a small write that 780 * partially fills a buffer. 781 * 782 * NB: magnetic tapes cannot be delayed; they must be 783 * written in the order that the writes are requested. 784 * 785 * Described in Leffler, et al. (pp. 208-213). 786 */ 787 void 788 bdwrite(struct buf *bp) 789 { 790 int s; 791 792 /* 793 * If the block hasn't been seen before: 794 * (1) Mark it as having been seen, 795 * (2) Charge for the write. 796 * (3) Make sure it's on its vnode's correct block list, 797 * (4) If a buffer is rewritten, move it to end of dirty list 798 */ 799 if (!ISSET(bp->b_flags, B_DELWRI)) { 800 SET(bp->b_flags, B_DELWRI); 801 s = splbio(); 802 buf_flip_dma(bp); 803 reassignbuf(bp); 804 splx(s); 805 curproc->p_ru.ru_oublock++; /* XXX */ 806 } 807 808 /* The "write" is done, so mark and release the buffer. */ 809 CLR(bp->b_flags, B_NEEDCOMMIT); 810 SET(bp->b_flags, B_DONE); 811 brelse(bp); 812 } 813 814 /* 815 * Asynchronous block write; just an asynchronous bwrite(). 816 */ 817 void 818 bawrite(struct buf *bp) 819 { 820 821 SET(bp->b_flags, B_ASYNC); 822 VOP_BWRITE(bp); 823 } 824 825 /* 826 * Must be called at splbio() 827 */ 828 void 829 buf_dirty(struct buf *bp) 830 { 831 splassert(IPL_BIO); 832 833 #ifdef DIAGNOSTIC 834 if (!ISSET(bp->b_flags, B_BUSY)) 835 panic("Trying to dirty buffer on freelist!"); 836 #endif 837 838 if (ISSET(bp->b_flags, B_DELWRI) == 0) { 839 SET(bp->b_flags, B_DELWRI); 840 buf_flip_dma(bp); 841 reassignbuf(bp); 842 } 843 } 844 845 /* 846 * Must be called at splbio() 847 */ 848 void 849 buf_undirty(struct buf *bp) 850 { 851 splassert(IPL_BIO); 852 853 #ifdef DIAGNOSTIC 854 if (!ISSET(bp->b_flags, B_BUSY)) 855 panic("Trying to undirty buffer on freelist!"); 856 #endif 857 if (ISSET(bp->b_flags, B_DELWRI)) { 858 CLR(bp->b_flags, B_DELWRI); 859 reassignbuf(bp); 860 } 861 } 862 863 /* 864 * Release a buffer on to the free lists. 865 * Described in Bach (p. 46). 866 */ 867 void 868 brelse(struct buf *bp) 869 { 870 int s; 871 872 s = splbio(); 873 874 if (bp->b_data != NULL) 875 KASSERT(bp->b_bufsize > 0); 876 877 /* 878 * softdep is basically incompatible with not cacheing buffers 879 * that have dependencies, so this buffer must be cached 880 */ 881 if (LIST_FIRST(&bp->b_dep) != NULL) 882 CLR(bp->b_flags, B_NOCACHE); 883 884 /* 885 * Determine which queue the buffer should be on, then put it there. 886 */ 887 888 /* If it's not cacheable, or an error, mark it invalid. */ 889 if (ISSET(bp->b_flags, (B_NOCACHE|B_ERROR))) 890 SET(bp->b_flags, B_INVAL); 891 /* If it's a write error, also mark the vnode as damaged. */ 892 if (ISSET(bp->b_flags, B_ERROR) && !ISSET(bp->b_flags, B_READ)) { 893 if (bp->b_vp && bp->b_vp->v_type == VREG) 894 SET(bp->b_vp->v_bioflag, VBIOERROR); 895 } 896 897 if (ISSET(bp->b_flags, B_INVAL)) { 898 /* 899 * If the buffer is invalid, free it now rather than leaving 900 * it in a queue and wasting memory. 901 */ 902 if (LIST_FIRST(&bp->b_dep) != NULL) 903 buf_deallocate(bp); 904 905 if (ISSET(bp->b_flags, B_DELWRI)) { 906 CLR(bp->b_flags, B_DELWRI); 907 } 908 909 if (bp->b_vp) { 910 RBT_REMOVE(buf_rb_bufs, &bp->b_vp->v_bufs_tree, bp); 911 brelvp(bp); 912 } 913 bp->b_vp = NULL; 914 915 /* 916 * Wake up any processes waiting for _this_ buffer to 917 * become free. They are not allowed to grab it 918 * since it will be freed. But the only sleeper is 919 * getblk and it will restart the operation after 920 * sleep. 921 */ 922 if (ISSET(bp->b_flags, B_WANTED)) { 923 CLR(bp->b_flags, B_WANTED); 924 wakeup(bp); 925 } 926 buf_put(bp); 927 } else { 928 /* 929 * It has valid data. Put it on the end of the appropriate 930 * queue, so that it'll stick around for as long as possible. 931 */ 932 bufcache_release(bp); 933 934 /* Unlock the buffer. */ 935 CLR(bp->b_flags, (B_AGE | B_ASYNC | B_NOCACHE | B_DEFERRED)); 936 buf_release(bp); 937 938 /* Wake up any processes waiting for _this_ buffer to 939 * become free. */ 940 if (ISSET(bp->b_flags, B_WANTED)) { 941 CLR(bp->b_flags, B_WANTED); 942 wakeup(bp); 943 } 944 } 945 946 /* Wake up syncer and cleaner processes waiting for buffers. */ 947 if (nobuffers) { 948 nobuffers = 0; 949 wakeup(&nobuffers); 950 } 951 952 /* Wake up any processes waiting for any buffer to become free. */ 953 if (needbuffer && bcstats.dmapages < targetpages && 954 bcstats.kvaslots_avail > RESERVE_SLOTS) { 955 needbuffer = 0; 956 wakeup(&needbuffer); 957 } 958 959 splx(s); 960 } 961 962 /* 963 * Determine if a block is in the cache. Just look on what would be its hash 964 * chain. If it's there, return a pointer to it, unless it's marked invalid. 965 */ 966 struct buf * 967 incore(struct vnode *vp, daddr_t blkno) 968 { 969 struct buf *bp; 970 struct buf b; 971 int s; 972 973 s = splbio(); 974 975 /* Search buf lookup tree */ 976 b.b_lblkno = blkno; 977 bp = RBT_FIND(buf_rb_bufs, &vp->v_bufs_tree, &b); 978 if (bp != NULL && ISSET(bp->b_flags, B_INVAL)) 979 bp = NULL; 980 981 splx(s); 982 return (bp); 983 } 984 985 /* 986 * Get a block of requested size that is associated with 987 * a given vnode and block offset. If it is found in the 988 * block cache, mark it as having been found, make it busy 989 * and return it. Otherwise, return an empty block of the 990 * correct size. It is up to the caller to ensure that the 991 * cached blocks be of the correct size. 992 */ 993 struct buf * 994 getblk(struct vnode *vp, daddr_t blkno, int size, int slpflag, 995 uint64_t slptimeo) 996 { 997 struct buf *bp; 998 struct buf b; 999 int s, error; 1000 1001 /* 1002 * XXX 1003 * The following is an inlined version of 'incore()', but with 1004 * the 'invalid' test moved to after the 'busy' test. It's 1005 * necessary because there are some cases in which the NFS 1006 * code sets B_INVAL prior to writing data to the server, but 1007 * in which the buffers actually contain valid data. In this 1008 * case, we can't allow the system to allocate a new buffer for 1009 * the block until the write is finished. 1010 */ 1011 start: 1012 s = splbio(); 1013 b.b_lblkno = blkno; 1014 bp = RBT_FIND(buf_rb_bufs, &vp->v_bufs_tree, &b); 1015 if (bp != NULL) { 1016 if (ISSET(bp->b_flags, B_BUSY)) { 1017 SET(bp->b_flags, B_WANTED); 1018 error = tsleep_nsec(bp, slpflag | (PRIBIO + 1), 1019 "getblk", slptimeo); 1020 splx(s); 1021 if (error) 1022 return (NULL); 1023 goto start; 1024 } 1025 1026 if (!ISSET(bp->b_flags, B_INVAL)) { 1027 bcstats.cachehits++; 1028 SET(bp->b_flags, B_CACHE); 1029 bufcache_take(bp); 1030 buf_acquire(bp); 1031 splx(s); 1032 return (bp); 1033 } 1034 } 1035 splx(s); 1036 1037 if ((bp = buf_get(vp, blkno, size)) == NULL) 1038 goto start; 1039 1040 return (bp); 1041 } 1042 1043 /* 1044 * Get an empty, disassociated buffer of given size. 1045 */ 1046 struct buf * 1047 geteblk(size_t size) 1048 { 1049 struct buf *bp; 1050 1051 while ((bp = buf_get(NULL, 0, size)) == NULL) 1052 continue; 1053 1054 return (bp); 1055 } 1056 1057 /* 1058 * Allocate a buffer. 1059 * If vp is given, put it into the buffer cache for that vnode. 1060 * If size != 0, allocate memory and call buf_map(). 1061 * If there is already a buffer for the given vnode/blkno, return NULL. 1062 */ 1063 struct buf * 1064 buf_get(struct vnode *vp, daddr_t blkno, size_t size) 1065 { 1066 struct buf *bp; 1067 int poolwait = size == 0 ? PR_NOWAIT : PR_WAITOK; 1068 int npages; 1069 int s; 1070 1071 s = splbio(); 1072 if (size) { 1073 /* 1074 * Wake up the cleaner if we have lots of dirty pages, 1075 * or if we are getting low on buffer cache kva. 1076 */ 1077 if (UNCLEAN_PAGES >= hidirtypages || 1078 bcstats.kvaslots_avail <= 2 * RESERVE_SLOTS) 1079 wakeup(&bd_req); 1080 1081 npages = atop(round_page(size)); 1082 1083 /* 1084 * if our cache has been previously shrunk, 1085 * allow it to grow again with use up to 1086 * bufhighpages (cachepercent) 1087 */ 1088 if (bufpages < bufhighpages) 1089 bufadjust(bufhighpages); 1090 1091 /* 1092 * If we would go over the page target with our 1093 * new allocation, free enough buffers first 1094 * to stay at the target with our new allocation. 1095 */ 1096 if (bcstats.dmapages + npages > targetpages) { 1097 (void) bufcache_recover_dmapages(0, npages); 1098 bufcache_adjust(); 1099 } 1100 1101 /* 1102 * If we get here, we tried to free the world down 1103 * above, and couldn't get down - Wake the cleaner 1104 * and wait for it to push some buffers out. 1105 */ 1106 if ((bcstats.dmapages + npages > targetpages || 1107 bcstats.kvaslots_avail <= RESERVE_SLOTS) && 1108 curproc != syncerproc && curproc != cleanerproc) { 1109 wakeup(&bd_req); 1110 needbuffer++; 1111 tsleep_nsec(&needbuffer, PRIBIO, "needbuffer", INFSLP); 1112 splx(s); 1113 return (NULL); 1114 } 1115 if (bcstats.dmapages + npages > bufpages) { 1116 /* cleaner or syncer */ 1117 nobuffers = 1; 1118 tsleep_nsec(&nobuffers, PRIBIO, "nobuffers", INFSLP); 1119 splx(s); 1120 return (NULL); 1121 } 1122 } 1123 1124 bp = pool_get(&bufpool, poolwait|PR_ZERO); 1125 1126 if (bp == NULL) { 1127 splx(s); 1128 return (NULL); 1129 } 1130 1131 bp->b_freelist.tqe_next = NOLIST; 1132 bp->b_dev = NODEV; 1133 LIST_INIT(&bp->b_dep); 1134 bp->b_bcount = size; 1135 1136 buf_acquire_nomap(bp); 1137 1138 if (vp != NULL) { 1139 /* 1140 * We insert the buffer into the hash with B_BUSY set 1141 * while we allocate pages for it. This way any getblk 1142 * that happens while we allocate pages will wait for 1143 * this buffer instead of starting its own buf_get. 1144 * 1145 * But first, we check if someone beat us to it. 1146 */ 1147 if (incore(vp, blkno)) { 1148 pool_put(&bufpool, bp); 1149 splx(s); 1150 return (NULL); 1151 } 1152 1153 bp->b_blkno = bp->b_lblkno = blkno; 1154 bgetvp(vp, bp); 1155 if (RBT_INSERT(buf_rb_bufs, &vp->v_bufs_tree, bp)) 1156 panic("buf_get: dup lblk vp %p bp %p", vp, bp); 1157 } else { 1158 bp->b_vnbufs.le_next = NOLIST; 1159 SET(bp->b_flags, B_INVAL); 1160 bp->b_vp = NULL; 1161 } 1162 1163 LIST_INSERT_HEAD(&bufhead, bp, b_list); 1164 bcstats.numbufs++; 1165 1166 if (size) { 1167 buf_alloc_pages(bp, round_page(size)); 1168 KASSERT(ISSET(bp->b_flags, B_DMA)); 1169 buf_map(bp); 1170 } 1171 1172 SET(bp->b_flags, B_BC); 1173 splx(s); 1174 1175 return (bp); 1176 } 1177 1178 /* 1179 * Buffer cleaning daemon. 1180 */ 1181 void 1182 buf_daemon(void *arg) 1183 { 1184 struct buf *bp = NULL; 1185 int s, pushed = 0; 1186 1187 s = splbio(); 1188 for (;;) { 1189 if (bp == NULL || (pushed >= 16 && 1190 UNCLEAN_PAGES < hidirtypages && 1191 bcstats.kvaslots_avail > 2 * RESERVE_SLOTS)){ 1192 pushed = 0; 1193 /* 1194 * Wake up anyone who was waiting for buffers 1195 * to be released. 1196 */ 1197 if (needbuffer) { 1198 needbuffer = 0; 1199 wakeup(&needbuffer); 1200 } 1201 tsleep_nsec(&bd_req, PRIBIO - 7, "cleaner", INFSLP); 1202 } 1203 1204 while ((bp = bufcache_getdirtybuf())) { 1205 if (UNCLEAN_PAGES < lodirtypages && 1206 bcstats.kvaslots_avail > 2 * RESERVE_SLOTS && 1207 pushed >= 16) 1208 break; 1209 1210 bufcache_take(bp); 1211 buf_acquire(bp); 1212 splx(s); 1213 1214 if (ISSET(bp->b_flags, B_INVAL)) { 1215 brelse(bp); 1216 s = splbio(); 1217 continue; 1218 } 1219 #ifdef DIAGNOSTIC 1220 if (!ISSET(bp->b_flags, B_DELWRI)) 1221 panic("Clean buffer on dirty queue"); 1222 #endif 1223 if (LIST_FIRST(&bp->b_dep) != NULL && 1224 !ISSET(bp->b_flags, B_DEFERRED) && 1225 buf_countdeps(bp, 0, 0)) { 1226 SET(bp->b_flags, B_DEFERRED); 1227 s = splbio(); 1228 bufcache_release(bp); 1229 buf_release(bp); 1230 continue; 1231 } 1232 1233 bawrite(bp); 1234 pushed++; 1235 1236 sched_pause(yield); 1237 1238 s = splbio(); 1239 } 1240 } 1241 } 1242 1243 /* 1244 * Wait for operations on the buffer to complete. 1245 * When they do, extract and return the I/O's error value. 1246 */ 1247 int 1248 biowait(struct buf *bp) 1249 { 1250 int s; 1251 1252 KASSERT(!(bp->b_flags & B_ASYNC)); 1253 1254 s = splbio(); 1255 while (!ISSET(bp->b_flags, B_DONE)) 1256 tsleep_nsec(bp, PRIBIO + 1, "biowait", INFSLP); 1257 splx(s); 1258 1259 /* check for interruption of I/O (e.g. via NFS), then errors. */ 1260 if (ISSET(bp->b_flags, B_EINTR)) { 1261 CLR(bp->b_flags, B_EINTR); 1262 return (EINTR); 1263 } 1264 1265 if (ISSET(bp->b_flags, B_ERROR)) 1266 return (bp->b_error ? bp->b_error : EIO); 1267 else 1268 return (0); 1269 } 1270 1271 /* 1272 * Mark I/O complete on a buffer. 1273 * 1274 * If a callback has been requested, e.g. the pageout 1275 * daemon, do so. Otherwise, awaken waiting processes. 1276 * 1277 * [ Leffler, et al., says on p.247: 1278 * "This routine wakes up the blocked process, frees the buffer 1279 * for an asynchronous write, or, for a request by the pagedaemon 1280 * process, invokes a procedure specified in the buffer structure" ] 1281 * 1282 * In real life, the pagedaemon (or other system processes) wants 1283 * to do async stuff to, and doesn't want the buffer brelse()'d. 1284 * (for swap pager, that puts swap buffers on the free lists (!!!), 1285 * for the vn device, that puts malloc'd buffers on the free lists!) 1286 * 1287 * Must be called at splbio(). 1288 */ 1289 void 1290 biodone(struct buf *bp) 1291 { 1292 splassert(IPL_BIO); 1293 1294 if (ISSET(bp->b_flags, B_DONE)) 1295 panic("biodone already"); 1296 SET(bp->b_flags, B_DONE); /* note that it's done */ 1297 1298 if (bp->b_bq) 1299 bufq_done(bp->b_bq, bp); 1300 1301 if (LIST_FIRST(&bp->b_dep) != NULL) 1302 buf_complete(bp); 1303 1304 if (!ISSET(bp->b_flags, B_READ)) { 1305 CLR(bp->b_flags, B_WRITEINPROG); 1306 vwakeup(bp->b_vp); 1307 } 1308 if (bcstats.numbufs && 1309 (!(ISSET(bp->b_flags, B_RAW) || ISSET(bp->b_flags, B_PHYS)))) { 1310 if (!ISSET(bp->b_flags, B_READ)) { 1311 bcstats.pendingwrites--; 1312 } else 1313 bcstats.pendingreads--; 1314 } 1315 if (ISSET(bp->b_flags, B_CALL)) { /* if necessary, call out */ 1316 CLR(bp->b_flags, B_CALL); /* but note callout done */ 1317 (*bp->b_iodone)(bp); 1318 } else { 1319 if (ISSET(bp->b_flags, B_ASYNC)) {/* if async, release it */ 1320 brelse(bp); 1321 } else { /* or just wakeup the buffer */ 1322 CLR(bp->b_flags, B_WANTED); 1323 wakeup(bp); 1324 } 1325 } 1326 } 1327 1328 #ifdef DDB 1329 void bcstats_print(int (*)(const char *, ...) 1330 __attribute__((__format__(__kprintf__,1,2)))); 1331 /* 1332 * bcstats_print: ddb hook to print interesting buffer cache counters 1333 */ 1334 void 1335 bcstats_print( 1336 int (*pr)(const char *, ...) __attribute__((__format__(__kprintf__,1,2)))) 1337 { 1338 (*pr)("Current Buffer Cache status:\n"); 1339 (*pr)("numbufs %lld busymapped %lld, delwri %lld\n", 1340 bcstats.numbufs, bcstats.busymapped, bcstats.delwribufs); 1341 (*pr)("kvaslots %lld avail kva slots %lld\n", 1342 bcstats.kvaslots, bcstats.kvaslots_avail); 1343 (*pr)("bufpages %lld, dmapages %lld, dirtypages %lld\n", 1344 bcstats.numbufpages, bcstats.dmapages, bcstats.numdirtypages); 1345 (*pr)("pendingreads %lld, pendingwrites %lld\n", 1346 bcstats.pendingreads, bcstats.pendingwrites); 1347 (*pr)("highflips %lld, highflops %lld, dmaflips %lld\n", 1348 bcstats.highflips, bcstats.highflops, bcstats.dmaflips); 1349 } 1350 #endif 1351 1352 void 1353 buf_adjcnt(struct buf *bp, long ncount) 1354 { 1355 KASSERT(ncount <= bp->b_bufsize); 1356 bp->b_bcount = ncount; 1357 } 1358 1359 /* bufcache freelist code below */ 1360 /* 1361 * Copyright (c) 2014 Ted Unangst <tedu@openbsd.org> 1362 * 1363 * Permission to use, copy, modify, and distribute this software for any 1364 * purpose with or without fee is hereby granted, provided that the above 1365 * copyright notice and this permission notice appear in all copies. 1366 * 1367 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 1368 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 1369 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 1370 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 1371 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 1372 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 1373 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 1374 */ 1375 1376 /* 1377 * The code below implements a variant of the 2Q buffer cache algorithm by 1378 * Johnson and Shasha. 1379 * 1380 * General Outline 1381 * We divide the buffer cache into three working sets: current, previous, 1382 * and long term. Each list is itself LRU and buffers get promoted and moved 1383 * around between them. A buffer starts its life in the current working set. 1384 * As time passes and newer buffers push it out, it will turn into the previous 1385 * working set and is subject to recycling. But if it's accessed again from 1386 * the previous working set, that's an indication that it's actually in the 1387 * long term working set, so we promote it there. The separation of current 1388 * and previous working sets prevents us from promoting a buffer that's only 1389 * temporarily hot to the long term cache. 1390 * 1391 * The objective is to provide scan resistance by making the long term 1392 * working set ineligible for immediate recycling, even as the current 1393 * working set is rapidly turned over. 1394 * 1395 * Implementation 1396 * The code below identifies the current, previous, and long term sets as 1397 * hotqueue, coldqueue, and warmqueue. The hot and warm queues are capped at 1398 * 1/3 of the total clean pages, after which point they start pushing their 1399 * oldest buffers into coldqueue. 1400 * A buf always starts out with neither WARM or COLD flags set (implying HOT). 1401 * When released, it will be returned to the tail of the hotqueue list. 1402 * When the hotqueue gets too large, the oldest hot buf will be moved to the 1403 * coldqueue, with the B_COLD flag set. When a cold buf is released, we set 1404 * the B_WARM flag and put it onto the warmqueue. Warm bufs are also 1405 * directly returned to the end of the warmqueue. As with the hotqueue, when 1406 * the warmqueue grows too large, B_WARM bufs are moved onto the coldqueue. 1407 * 1408 * Note that this design does still support large working sets, greater 1409 * than the cap of hotqueue or warmqueue would imply. The coldqueue is still 1410 * cached and has no maximum length. The hot and warm queues form a Y feeding 1411 * into the coldqueue. Moving bufs between queues is constant time, so this 1412 * design decays to one long warm->cold queue. 1413 * 1414 * In the 2Q paper, hotqueue and coldqueue are A1in and A1out. The warmqueue 1415 * is Am. We always cache pages, as opposed to pointers to pages for A1. 1416 * 1417 * This implementation adds support for multiple 2q caches. 1418 * 1419 * If we have more than one 2q cache, as bufs fall off the cold queue 1420 * for recyclying, bufs that have been warm before (which retain the 1421 * B_WARM flag in addition to B_COLD) can be put into the hot queue of 1422 * a second level 2Q cache. buffers which are only B_COLD are 1423 * recycled. Bufs falling off the last cache's cold queue are always 1424 * recycled. 1425 * 1426 */ 1427 1428 /* 1429 * this function is called when a hot or warm queue may have exceeded its 1430 * size limit. it will move a buf to the coldqueue. 1431 */ 1432 int chillbufs(struct 1433 bufcache *cache, struct bufqueue *queue, int64_t *queuepages); 1434 1435 void 1436 bufcache_init(void) 1437 { 1438 int i; 1439 1440 for (i = 0; i < NUM_CACHES; i++) { 1441 TAILQ_INIT(&cleancache[i].hotqueue); 1442 TAILQ_INIT(&cleancache[i].coldqueue); 1443 TAILQ_INIT(&cleancache[i].warmqueue); 1444 } 1445 TAILQ_INIT(&dirtyqueue); 1446 } 1447 1448 /* 1449 * if the buffer caches have shrunk, we may need to rebalance our queues. 1450 */ 1451 void 1452 bufcache_adjust(void) 1453 { 1454 int i; 1455 1456 for (i = 0; i < NUM_CACHES; i++) { 1457 while (chillbufs(&cleancache[i], &cleancache[i].warmqueue, 1458 &cleancache[i].warmbufpages) || 1459 chillbufs(&cleancache[i], &cleancache[i].hotqueue, 1460 &cleancache[i].hotbufpages)) 1461 continue; 1462 } 1463 } 1464 1465 /* 1466 * Get a clean buffer from the cache. if "discard" is set do not promote 1467 * previously warm buffers as normal, because we are tossing everything 1468 * away such as in a hibernation 1469 */ 1470 struct buf * 1471 bufcache_getcleanbuf(int cachenum, int discard) 1472 { 1473 struct buf *bp = NULL; 1474 struct bufcache *cache = &cleancache[cachenum]; 1475 struct bufqueue * queue; 1476 1477 splassert(IPL_BIO); 1478 1479 /* try cold queue */ 1480 while ((bp = TAILQ_FIRST(&cache->coldqueue)) || 1481 (bp = TAILQ_FIRST(&cache->warmqueue)) || 1482 (bp = TAILQ_FIRST(&cache->hotqueue))) { 1483 int64_t pages = atop(bp->b_bufsize); 1484 struct bufcache *newcache; 1485 1486 if (discard || cachenum >= NUM_CACHES - 1) { 1487 /* Victim selected, give it up */ 1488 return bp; 1489 } 1490 KASSERT(bp->cache == cachenum); 1491 1492 /* 1493 * If this buffer was warm before, move it to 1494 * the hot queue in the next cache 1495 */ 1496 1497 if (fliphigh) { 1498 /* 1499 * If we are in the DMA cache, try to flip the 1500 * buffer up high to move it on to the other 1501 * caches. if we can't move the buffer to high 1502 * memory without sleeping, we give it up and 1503 * return it rather than fight for more memory 1504 * against non buffer cache competitors. 1505 */ 1506 SET(bp->b_flags, B_BUSY); 1507 if (bp->cache == 0 && buf_flip_high(bp) == -1) { 1508 CLR(bp->b_flags, B_BUSY); 1509 return bp; 1510 } 1511 CLR(bp->b_flags, B_BUSY); 1512 } 1513 1514 /* Move the buffer to the hot queue in the next cache */ 1515 if (ISSET(bp->b_flags, B_COLD)) { 1516 queue = &cache->coldqueue; 1517 } else if (ISSET(bp->b_flags, B_WARM)) { 1518 queue = &cache->warmqueue; 1519 cache->warmbufpages -= pages; 1520 } else { 1521 queue = &cache->hotqueue; 1522 cache->hotbufpages -= pages; 1523 } 1524 TAILQ_REMOVE(queue, bp, b_freelist); 1525 cache->cachepages -= pages; 1526 CLR(bp->b_flags, B_WARM); 1527 CLR(bp->b_flags, B_COLD); 1528 bp->cache++; 1529 newcache= &cleancache[bp->cache]; 1530 newcache->cachepages += pages; 1531 newcache->hotbufpages += pages; 1532 chillbufs(newcache, &newcache->hotqueue, 1533 &newcache->hotbufpages); 1534 TAILQ_INSERT_TAIL(&newcache->hotqueue, bp, b_freelist); 1535 } 1536 return bp; 1537 } 1538 1539 1540 void 1541 discard_buffer(struct buf *bp) { 1542 bufcache_take(bp); 1543 if (bp->b_vp) { 1544 RBT_REMOVE(buf_rb_bufs, 1545 &bp->b_vp->v_bufs_tree, bp); 1546 brelvp(bp); 1547 } 1548 buf_put(bp); 1549 } 1550 1551 int64_t 1552 bufcache_recover_dmapages(int discard, int64_t howmany) 1553 { 1554 struct buf *bp = NULL; 1555 struct bufcache *cache = &cleancache[DMA_CACHE]; 1556 struct bufqueue * queue; 1557 int64_t recovered = 0; 1558 1559 splassert(IPL_BIO); 1560 1561 while ((recovered < howmany) && 1562 ((bp = TAILQ_FIRST(&cache->coldqueue)) || 1563 (bp = TAILQ_FIRST(&cache->warmqueue)) || 1564 (bp = TAILQ_FIRST(&cache->hotqueue)))) { 1565 int64_t pages = atop(bp->b_bufsize); 1566 struct bufcache *newcache; 1567 1568 if (discard || DMA_CACHE >= NUM_CACHES - 1) { 1569 discard_buffer(bp); 1570 continue; 1571 } 1572 KASSERT(bp->cache == DMA_CACHE); 1573 1574 /* 1575 * If this buffer was warm before, move it to 1576 * the hot queue in the next cache 1577 */ 1578 1579 /* 1580 * One way or another, the pages for this 1581 * buffer are leaving DMA memory 1582 */ 1583 recovered += pages; 1584 1585 if (!fliphigh) { 1586 discard_buffer(bp); 1587 continue; 1588 } 1589 1590 /* 1591 * If we are in the DMA cache, try to flip the 1592 * buffer up high to move it on to the other 1593 * caches. if we can't move the buffer to high 1594 * memory without sleeping, we give it up 1595 * now rather than fight for more memory 1596 * against non buffer cache competitors. 1597 */ 1598 SET(bp->b_flags, B_BUSY); 1599 if (bp->cache == 0 && buf_flip_high(bp) == -1) { 1600 CLR(bp->b_flags, B_BUSY); 1601 discard_buffer(bp); 1602 continue; 1603 } 1604 CLR(bp->b_flags, B_BUSY); 1605 1606 /* 1607 * Move the buffer to the hot queue in the next cache 1608 */ 1609 if (ISSET(bp->b_flags, B_COLD)) { 1610 queue = &cache->coldqueue; 1611 } else if (ISSET(bp->b_flags, B_WARM)) { 1612 queue = &cache->warmqueue; 1613 cache->warmbufpages -= pages; 1614 } else { 1615 queue = &cache->hotqueue; 1616 cache->hotbufpages -= pages; 1617 } 1618 TAILQ_REMOVE(queue, bp, b_freelist); 1619 cache->cachepages -= pages; 1620 CLR(bp->b_flags, B_WARM); 1621 CLR(bp->b_flags, B_COLD); 1622 bp->cache++; 1623 newcache= &cleancache[bp->cache]; 1624 newcache->cachepages += pages; 1625 newcache->hotbufpages += pages; 1626 chillbufs(newcache, &newcache->hotqueue, 1627 &newcache->hotbufpages); 1628 TAILQ_INSERT_TAIL(&newcache->hotqueue, bp, b_freelist); 1629 } 1630 return recovered; 1631 } 1632 1633 struct buf * 1634 bufcache_getcleanbuf_range(int start, int end, int discard) 1635 { 1636 int i, j = start, q = end; 1637 struct buf *bp = NULL; 1638 1639 /* 1640 * XXX in theory we could promote warm buffers into a previous queue 1641 * so in the pathological case of where we go through all the caches 1642 * without getting a buffer we have to start at the beginning again. 1643 */ 1644 while (j <= q) { 1645 for (i = q; i >= j; i--) 1646 if ((bp = bufcache_getcleanbuf(i, discard))) 1647 return (bp); 1648 j++; 1649 } 1650 return bp; 1651 } 1652 1653 struct buf * 1654 bufcache_gethighcleanbuf(void) 1655 { 1656 if (!fliphigh) 1657 return NULL; 1658 return bufcache_getcleanbuf_range(DMA_CACHE + 1, NUM_CACHES - 1, 0); 1659 } 1660 1661 1662 struct buf * 1663 bufcache_getdmacleanbuf(void) 1664 { 1665 if (fliphigh) 1666 return bufcache_getcleanbuf_range(DMA_CACHE, DMA_CACHE, 0); 1667 return bufcache_getcleanbuf_range(DMA_CACHE, NUM_CACHES - 1, 0); 1668 } 1669 1670 1671 struct buf * 1672 bufcache_getdirtybuf(void) 1673 { 1674 return TAILQ_FIRST(&dirtyqueue); 1675 } 1676 1677 void 1678 bufcache_take(struct buf *bp) 1679 { 1680 struct bufqueue *queue; 1681 int64_t pages; 1682 1683 splassert(IPL_BIO); 1684 KASSERT(ISSET(bp->b_flags, B_BC)); 1685 KASSERT(bp->cache >= DMA_CACHE); 1686 KASSERT((bp->cache < NUM_CACHES)); 1687 1688 pages = atop(bp->b_bufsize); 1689 struct bufcache *cache = &cleancache[bp->cache]; 1690 if (!ISSET(bp->b_flags, B_DELWRI)) { 1691 if (ISSET(bp->b_flags, B_COLD)) { 1692 queue = &cache->coldqueue; 1693 } else if (ISSET(bp->b_flags, B_WARM)) { 1694 queue = &cache->warmqueue; 1695 cache->warmbufpages -= pages; 1696 } else { 1697 queue = &cache->hotqueue; 1698 cache->hotbufpages -= pages; 1699 } 1700 bcstats.numcleanpages -= pages; 1701 cache->cachepages -= pages; 1702 } else { 1703 queue = &dirtyqueue; 1704 bcstats.numdirtypages -= pages; 1705 bcstats.delwribufs--; 1706 } 1707 TAILQ_REMOVE(queue, bp, b_freelist); 1708 } 1709 1710 /* move buffers from a hot or warm queue to a cold queue in a cache */ 1711 int 1712 chillbufs(struct bufcache *cache, struct bufqueue *queue, int64_t *queuepages) 1713 { 1714 struct buf *bp; 1715 int64_t limit, pages; 1716 1717 /* 1718 * We limit the hot queue to be small, with a max of 4096 pages. 1719 * We limit the warm queue to half the cache size. 1720 * 1721 * We impose a minimum size of 96 to prevent too much "wobbling". 1722 */ 1723 if (queue == &cache->hotqueue) 1724 limit = min(cache->cachepages / 20, 4096); 1725 else if (queue == &cache->warmqueue) 1726 limit = (cache->cachepages / 2); 1727 else 1728 panic("chillbufs: invalid queue"); 1729 1730 if (*queuepages > 96 && *queuepages > limit) { 1731 bp = TAILQ_FIRST(queue); 1732 if (!bp) 1733 panic("inconsistent bufpage counts"); 1734 pages = atop(bp->b_bufsize); 1735 *queuepages -= pages; 1736 TAILQ_REMOVE(queue, bp, b_freelist); 1737 /* we do not clear B_WARM */ 1738 SET(bp->b_flags, B_COLD); 1739 TAILQ_INSERT_TAIL(&cache->coldqueue, bp, b_freelist); 1740 return 1; 1741 } 1742 return 0; 1743 } 1744 1745 void 1746 bufcache_release(struct buf *bp) 1747 { 1748 struct bufqueue *queue; 1749 int64_t pages; 1750 struct bufcache *cache = &cleancache[bp->cache]; 1751 1752 pages = atop(bp->b_bufsize); 1753 KASSERT(ISSET(bp->b_flags, B_BC)); 1754 if (fliphigh) { 1755 if (ISSET(bp->b_flags, B_DMA) && bp->cache > 0) 1756 panic("B_DMA buffer release from cache %d", 1757 bp->cache); 1758 else if ((!ISSET(bp->b_flags, B_DMA)) && bp->cache == 0) 1759 panic("Non B_DMA buffer release from cache %d", 1760 bp->cache); 1761 } 1762 1763 if (!ISSET(bp->b_flags, B_DELWRI)) { 1764 int64_t *queuepages; 1765 if (ISSET(bp->b_flags, B_WARM | B_COLD)) { 1766 SET(bp->b_flags, B_WARM); 1767 CLR(bp->b_flags, B_COLD); 1768 queue = &cache->warmqueue; 1769 queuepages = &cache->warmbufpages; 1770 } else { 1771 queue = &cache->hotqueue; 1772 queuepages = &cache->hotbufpages; 1773 } 1774 *queuepages += pages; 1775 bcstats.numcleanpages += pages; 1776 cache->cachepages += pages; 1777 chillbufs(cache, queue, queuepages); 1778 } else { 1779 queue = &dirtyqueue; 1780 bcstats.numdirtypages += pages; 1781 bcstats.delwribufs++; 1782 } 1783 TAILQ_INSERT_TAIL(queue, bp, b_freelist); 1784 } 1785 1786 #ifdef HIBERNATE 1787 /* 1788 * Nuke the buffer cache from orbit when hibernating. We do not want to save 1789 * any clean cache pages to swap and read them back. the original disk files 1790 * are just as good. 1791 */ 1792 void 1793 hibernate_suspend_bufcache(void) 1794 { 1795 struct buf *bp; 1796 int s; 1797 1798 s = splbio(); 1799 /* Chuck away all the cache pages.. discard bufs, do not promote */ 1800 while ((bp = bufcache_getcleanbuf_range(DMA_CACHE, NUM_CACHES - 1, 1))) { 1801 bufcache_take(bp); 1802 if (bp->b_vp) { 1803 RBT_REMOVE(buf_rb_bufs, &bp->b_vp->v_bufs_tree, bp); 1804 brelvp(bp); 1805 } 1806 buf_put(bp); 1807 } 1808 splx(s); 1809 } 1810 1811 void 1812 hibernate_resume_bufcache(void) 1813 { 1814 /* XXX Nothing needed here for now */ 1815 } 1816 #endif /* HIBERNATE */ 1817